From 4e52cfa0a7ff6245d3583cd97a6725a1546fa0bc Mon Sep 17 00:00:00 2001 From: fargito Date: Mon, 27 Jul 2026 16:51:07 +0200 Subject: [PATCH] feat(run-environment): add CircleCI provider for GitHub repositories Support running benchmarks from CircleCI on GitHub-hosted repositories. The build runs on CircleCI while commits and pull requests live on GitHub, so the run reports against the GitHub commit and PR. Unlike GitHub Actions, CircleCI checks out the associated commit itself rather than a synthetic merge ref, so the default git HEAD read already yields CIRCLE_SHA1 and no commit hash override is needed. CircleCI exposes no base branch variable either: baseRef is left empty and CodSpeed resolves the base branch from the pull request. CircleCI also builds Bitbucket and GitLab repositories, so the domain of CIRCLE_REPOSITORY_URL is validated and anything but github.com is rejected with an explicit error. That domain is the only signal available at runtime: the explicit pipeline.project.type is a pipeline value, interpolated when the config is compiled, so it never reaches the job as a variable. Uploads authenticate with a static CODSPEED_TOKEN. Run part data is already reported, so runs group correctly once multi-part upload support lands: CIRCLE_WORKFLOW_ID is shared by every job and every parallel container of a workflow, while CIRCLE_JOB and CIRCLE_NODE_INDEX identify a single part. Refs COD-2977 Refs COD-2995 Refs COD-3262 --- src/run_environment/circleci/logger.rs | 105 +++++ src/run_environment/circleci/mod.rs | 4 + src/run_environment/circleci/provider.rs | 424 ++++++++++++++++++ ...ll_request_run_environment_metadata-2.snap | 13 + ...pull_request_run_environment_metadata.snap | 17 + src/run_environment/interfaces.rs | 1 + src/run_environment/mod.rs | 5 + src/upload/uploader.rs | 3 +- 8 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 src/run_environment/circleci/logger.rs create mode 100644 src/run_environment/circleci/mod.rs create mode 100644 src/run_environment/circleci/provider.rs create mode 100644 src/run_environment/circleci/snapshots/codspeed_runner__run_environment__circleci__provider__tests__pull_request_run_environment_metadata-2.snap create mode 100644 src/run_environment/circleci/snapshots/codspeed_runner__run_environment__circleci__provider__tests__pull_request_run_environment_metadata.snap diff --git a/src/run_environment/circleci/logger.rs b/src/run_environment/circleci/logger.rs new file mode 100644 index 000000000..95c62d1f4 --- /dev/null +++ b/src/run_environment/circleci/logger.rs @@ -0,0 +1,105 @@ +use console::style; +use log::*; +use simplelog::SharedLogger; +use std::{env, io::Write}; + +use crate::{ + logger::{GroupEvent, get_announcement_event, get_group_event, get_json_event}, + run_environment::logger::should_provider_logger_handle_record, +}; + +/// A logger that prints logs in the format expected by CircleCI +/// +/// CircleCI has no collapsible section markers, so groups are printed as plain +/// headers. +pub struct CircleCILogger { + log_level: LevelFilter, +} + +impl CircleCILogger { + pub fn new() -> Self { + // force activation of colors: CircleCI renders ANSI sequences in its UI, but + // the output is not a TTY so colors would be disabled by default. + console::set_colors_enabled(true); + + let log_level = env::var("CODSPEED_LOG") + .ok() + .and_then(|log_level| log_level.parse::().ok()) + .unwrap_or(log::LevelFilter::Info); + Self { log_level } + } +} + +impl Log for CircleCILogger { + fn enabled(&self, _metadata: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + if !should_provider_logger_handle_record(record) { + return; + } + + let level = record.level(); + let message = record.args(); + + if let Some(group_event) = get_group_event(record) { + match group_event { + GroupEvent::Start(name) | GroupEvent::StartOpened(name) => { + println!("{}", style(name).cyan().bold()); + } + GroupEvent::End => {} + } + return; + } + + if get_json_event(record).is_some() { + return; + } + + if let Some(announcement) = get_announcement_event(record) { + println!("{}", style(announcement).green()); + return; + } + + if level > self.log_level { + return; + } + + match level { + Level::Error => { + println!("{}", style(message).red()); + } + Level::Warn => { + println!("{}", style(message).yellow()); + } + Level::Info => { + println!("{message}"); + } + Level::Debug => { + println!("{}", style(message).cyan()); + } + Level::Trace => { + println!("{}", style(message).magenta()); + } + } + } + + fn flush(&self) { + std::io::stdout().flush().unwrap(); + } +} + +impl SharedLogger for CircleCILogger { + fn level(&self) -> LevelFilter { + self.log_level + } + + fn config(&self) -> Option<&simplelog::Config> { + None + } + + fn as_log(self: Box) -> Box { + Box::new(*self) + } +} diff --git a/src/run_environment/circleci/mod.rs b/src/run_environment/circleci/mod.rs new file mode 100644 index 000000000..0297b42af --- /dev/null +++ b/src/run_environment/circleci/mod.rs @@ -0,0 +1,4 @@ +mod logger; +mod provider; + +pub use provider::CircleCIProvider; diff --git a/src/run_environment/circleci/provider.rs b/src/run_environment/circleci/provider.rs new file mode 100644 index 000000000..0fae4d67f --- /dev/null +++ b/src/run_environment/circleci/provider.rs @@ -0,0 +1,424 @@ +use std::collections::BTreeMap; +use std::env; + +use async_trait::async_trait; +use serde_json::Value; +use simplelog::SharedLogger; + +use crate::api_client::CodSpeedAPIClient; +use crate::cli::run::helpers::{ + GitRemote, find_repository_root, get_env_variable, parse_git_remote, +}; +use crate::executor::config::OrchestratorConfig; +use crate::prelude::*; +use crate::run_environment::interfaces::{RepositoryProvider, RunEnvironmentMetadata, RunEvent}; +use crate::run_environment::provider::{RunEnvironmentDetector, RunEnvironmentProvider}; +use crate::run_environment::{RunEnvironment, RunPart}; + +use super::logger::CircleCILogger; + +#[derive(Debug)] +pub struct CircleCIProvider { + owner: String, + repository: String, + ref_: String, + head_ref: Option, + event: RunEvent, + repository_root_path: String, + workflow_id: String, + job_name: String, + /// Index of this container within a job running with `parallelism`, `0` when unset. + node_index: u32, + /// Number of containers the job runs on, `1` when unset. + node_total: u32, +} + +/// Returns the number of the pull request the build runs on, if any. +/// +/// `CIRCLE_PULL_REQUEST` holds the URL of the pull request (`.../pull/22`) and is +/// only set for pull request builds. `CIRCLE_PR_NUMBER` is a fallback that is only +/// populated for pull requests opened from a fork. +fn get_pr_number() -> Option { + let from_url = env::var("CIRCLE_PULL_REQUEST") + .ok() + .and_then(|url| url.rsplit('/').next()?.parse().ok()); + + from_url.or_else(|| { + env::var("CIRCLE_PR_NUMBER") + .ok() + .and_then(|number| number.parse().ok()) + }) +} + +fn get_ref(pr_number: Option) -> Result { + match pr_number { + Some(pr_number) => Ok(format!("refs/pull/{pr_number}/merge")), + None => Ok(format!("refs/heads/{}", get_env_variable("CIRCLE_BRANCH")?)), + } +} + +fn get_env_number(name: &str, default: u32) -> u32 { + env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +impl TryFrom<&OrchestratorConfig> for CircleCIProvider { + type Error = Error; + fn try_from(config: &OrchestratorConfig) -> Result { + if config.repository_override.is_some() { + bail!("Specifying owner and repository from CLI is not supported for CircleCI"); + } + + let pr_number = get_pr_number(); + let repository_url = get_env_variable("CIRCLE_REPOSITORY_URL")?; + let GitRemote { + owner, + repository, + domain, + } = parse_git_remote(&repository_url)?; + + // CircleCI also builds Bitbucket and GitLab repositories, which CodSpeed does not + // support here. The domain of the remote is the only signal available at runtime: + // the explicit `pipeline.project.type` is a pipeline value, interpolated when the + // config is compiled, so it never reaches the job as an environment variable. + // https://circleci.com/docs/reference/variables/ + if domain != "github.com" { + bail!( + "CodSpeed only supports GitHub repositories on CircleCI, but \ + CIRCLE_REPOSITORY_URL points to {domain} ({repository_url})" + ); + } + + let repository_root_path = match find_repository_root(&std::env::current_dir()?) { + Some(mut path) => { + // Add a trailing slash to the path + path.push(""); + path.to_string_lossy().to_string() + } + None => { + // Fallback to the working directory of the job, where CircleCI checks the + // repository out. Its value mirrors the `working_directory` key of the job, + // so it can start with a `~`. + // https://circleci.com/docs/variables/#built-in-environment-variables + let working_directory = get_env_variable("CIRCLE_WORKING_DIRECTORY")?; + format!("{}/", shellexpand::tilde(&working_directory)) + } + }; + + Ok(Self { + owner, + repository, + ref_: get_ref(pr_number)?, + head_ref: if pr_number.is_some() { + Some(get_env_variable("CIRCLE_BRANCH")?) + } else { + None + }, + event: if pr_number.is_some() { + RunEvent::PullRequest + } else { + RunEvent::Push + }, + repository_root_path, + workflow_id: get_env_variable("CIRCLE_WORKFLOW_ID")?, + job_name: get_env_variable("CIRCLE_JOB")?, + node_index: get_env_number("CIRCLE_NODE_INDEX", 0), + node_total: get_env_number("CIRCLE_NODE_TOTAL", 1), + }) + } +} + +impl RunEnvironmentDetector for CircleCIProvider { + fn detect() -> bool { + env::var("CIRCLECI") == Ok("true".into()) + } +} + +#[async_trait(?Send)] +impl RunEnvironmentProvider for CircleCIProvider { + fn get_repository_provider(&self) -> RepositoryProvider { + RepositoryProvider::GitHub + } + + fn get_logger(&self) -> Box { + Box::new(CircleCILogger::new()) + } + + fn get_run_environment(&self) -> RunEnvironment { + RunEnvironment::Circleci + } + + fn get_run_environment_metadata(&self) -> Result { + Ok(RunEnvironmentMetadata { + // CircleCI exposes no base branch variable. CodSpeed resolves it from the pull request. + base_ref: None, + head_ref: self.head_ref.clone(), + event: self.event.clone(), + owner: self.owner.clone(), + repository: self.repository.clone(), + ref_: self.ref_.clone(), + repository_root_path: self.repository_root_path.clone(), + gh_data: None, + gl_data: None, + local_data: None, + sender: None, + }) + } + + /// `CIRCLE_WORKFLOW_ID` is shared by every job and every parallel container of a + /// workflow, which makes it the key that groups run parts together. The per-job + /// `CIRCLE_WORKFLOW_JOB_ID` would instead split one workflow into unrelated runs. + /// + /// A workflow fans out along two axes, so the part id has to cover both: its jobs + /// (`CIRCLE_JOB`) and, within a job declaring `parallelism`, its containers + /// (`CIRCLE_NODE_INDEX`). + fn get_run_provider_run_part(&self) -> Option { + Some(RunPart { + run_id: self.workflow_id.clone(), + run_part_id: format!("{}-{}", self.job_name, self.node_index), + job_name: self.job_name.clone(), + metadata: BTreeMap::from([ + ("node-index".to_string(), Value::from(self.node_index)), + ("node-total".to_string(), Value::from(self.node_total)), + ]), + }) + } + + /// CircleCI requires a static `CODSPEED_TOKEN`. We don't yet support OIDC + /// tokens here (could be added via `CIRCLE_OIDC_TOKEN_V2`: + /// ), so this just enforces + /// token presence. + fn check_oidc_configuration(&mut self, api_client: &CodSpeedAPIClient) -> Result<()> { + if api_client.token().is_none() { + bail!("Token authentication is required for CircleCI"); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use insta::assert_json_snapshot; + use temp_env::{with_var, with_vars}; + + use super::*; + + #[test] + fn test_detect() { + with_var("CIRCLECI", Some("true"), || { + assert!(CircleCIProvider::detect()); + }); + } + + #[test] + fn test_try_from_push_main() { + with_vars( + [ + ("CIRCLECI", Some("true")), + ("CIRCLE_BRANCH", Some("main")), + ( + "CIRCLE_REPOSITORY_URL", + Some("git@github.com:my-org/adrien-python-test.git"), + ), + ("CIRCLE_WORKING_DIRECTORY", Some("/home/circleci/project")), + ( + "CIRCLE_WORKFLOW_ID", + Some("8d8f0b2a-1f3e-4b6a-9c2d-0f1e2a3b4c5d"), + ), + ("CIRCLE_JOB", Some("benchmarks")), + ], + || { + let config = OrchestratorConfig { + ..OrchestratorConfig::test() + }; + let provider = CircleCIProvider::try_from(&config).unwrap(); + + assert_eq!(provider.owner, "my-org"); + assert_eq!(provider.repository, "adrien-python-test"); + assert_eq!(provider.ref_, "refs/heads/main"); + assert_eq!(provider.head_ref, None); + assert_eq!(provider.event, RunEvent::Push); + assert_eq!(provider.repository_root_path, "/home/circleci/project/"); + }, + ); + } + + #[test] + fn test_try_from_pull_request() { + with_vars( + [ + ("CIRCLECI", Some("true")), + ("CIRCLE_BRANCH", Some("feat/codspeed-runner")), + ( + "CIRCLE_PULL_REQUEST", + Some("https://github.com/my-org/adrien-python-test/pull/22"), + ), + ( + "CIRCLE_REPOSITORY_URL", + Some("https://github.com/my-org/adrien-python-test"), + ), + ("CIRCLE_WORKING_DIRECTORY", Some("/home/circleci/project")), + ( + "CIRCLE_WORKFLOW_ID", + Some("8d8f0b2a-1f3e-4b6a-9c2d-0f1e2a3b4c5d"), + ), + ("CIRCLE_JOB", Some("benchmarks")), + ], + || { + let config = OrchestratorConfig { + ..OrchestratorConfig::test() + }; + let provider = CircleCIProvider::try_from(&config).unwrap(); + + assert_eq!(provider.owner, "my-org"); + assert_eq!(provider.repository, "adrien-python-test"); + assert_eq!(provider.ref_, "refs/pull/22/merge"); + assert_eq!(provider.head_ref, Some("feat/codspeed-runner".into())); + assert_eq!(provider.event, RunEvent::PullRequest); + }, + ); + } + + /// On pull requests opened from a fork, `CIRCLE_PULL_REQUEST` is not set and + /// `CIRCLE_BRANCH` is the `pull/{number}` ref CircleCI checks out. + #[test] + fn test_try_from_fork_pull_request() { + with_vars( + [ + ("CIRCLECI", Some("true")), + ("CIRCLE_BRANCH", Some("pull/22")), + ("CIRCLE_PR_NUMBER", Some("22")), + ( + "CIRCLE_REPOSITORY_URL", + Some("git@github.com:my-org/adrien-python-test.git"), + ), + ("CIRCLE_WORKING_DIRECTORY", Some("/home/circleci/project")), + ( + "CIRCLE_WORKFLOW_ID", + Some("8d8f0b2a-1f3e-4b6a-9c2d-0f1e2a3b4c5d"), + ), + ("CIRCLE_JOB", Some("benchmarks")), + ], + || { + let config = OrchestratorConfig { + ..OrchestratorConfig::test() + }; + let provider = CircleCIProvider::try_from(&config).unwrap(); + + assert_eq!(provider.ref_, "refs/pull/22/merge"); + assert_eq!(provider.event, RunEvent::PullRequest); + }, + ); + } + + #[test] + fn test_try_from_non_github_repository() { + with_vars( + [ + ("CIRCLECI", Some("true")), + ("CIRCLE_BRANCH", Some("main")), + ( + "CIRCLE_REPOSITORY_URL", + Some("git@bitbucket.org:my-org/adrien-python-test.git"), + ), + ("CIRCLE_WORKING_DIRECTORY", Some("/home/circleci/project")), + ( + "CIRCLE_WORKFLOW_ID", + Some("8d8f0b2a-1f3e-4b6a-9c2d-0f1e2a3b4c5d"), + ), + ("CIRCLE_JOB", Some("benchmarks")), + ], + || { + let config = OrchestratorConfig { + ..OrchestratorConfig::test() + }; + let error = CircleCIProvider::try_from(&config).unwrap_err(); + + assert_eq!( + error.to_string(), + "CodSpeed only supports GitHub repositories on CircleCI, but \ + CIRCLE_REPOSITORY_URL points to bitbucket.org \ + (git@bitbucket.org:my-org/adrien-python-test.git)" + ); + }, + ); + } + + #[test] + fn test_pull_request_run_environment_metadata() { + with_vars( + [ + ("CIRCLECI", Some("true")), + ("CIRCLE_BRANCH", Some("feat/codspeed-runner")), + ( + "CIRCLE_PULL_REQUEST", + Some("https://github.com/my-org/adrien-python-test/pull/22"), + ), + ( + "CIRCLE_REPOSITORY_URL", + Some("git@github.com:my-org/adrien-python-test.git"), + ), + ("CIRCLE_WORKING_DIRECTORY", Some("/home/circleci/project")), + ( + "CIRCLE_WORKFLOW_ID", + Some("8d8f0b2a-1f3e-4b6a-9c2d-0f1e2a3b4c5d"), + ), + ("CIRCLE_JOB", Some("benchmarks")), + ], + || { + let config = OrchestratorConfig { + ..OrchestratorConfig::test() + }; + let provider = CircleCIProvider::try_from(&config).unwrap(); + let run_environment_metadata = provider.get_run_environment_metadata().unwrap(); + let run_part = provider.get_run_provider_run_part().unwrap(); + + assert_json_snapshot!(run_environment_metadata); + assert_json_snapshot!(run_part); + }, + ); + } + + /// A job declaring `parallelism` runs on several containers that share + /// `CIRCLE_JOB`, so the node index is what keeps their part ids apart. + #[test] + fn test_run_part_of_parallel_job() { + with_vars( + [ + ("CIRCLECI", Some("true")), + ("CIRCLE_BRANCH", Some("main")), + ( + "CIRCLE_REPOSITORY_URL", + Some("git@github.com:my-org/adrien-python-test.git"), + ), + ("CIRCLE_WORKING_DIRECTORY", Some("/home/circleci/project")), + ( + "CIRCLE_WORKFLOW_ID", + Some("8d8f0b2a-1f3e-4b6a-9c2d-0f1e2a3b4c5d"), + ), + ("CIRCLE_JOB", Some("benchmarks")), + ("CIRCLE_NODE_INDEX", Some("2")), + ("CIRCLE_NODE_TOTAL", Some("4")), + ], + || { + let config = OrchestratorConfig { + ..OrchestratorConfig::test() + }; + let provider = CircleCIProvider::try_from(&config).unwrap(); + let run_part = provider.get_run_provider_run_part().unwrap(); + + assert_eq!(run_part.run_id, "8d8f0b2a-1f3e-4b6a-9c2d-0f1e2a3b4c5d"); + assert_eq!(run_part.job_name, "benchmarks"); + assert_eq!(run_part.run_part_id, "benchmarks-2"); + assert_json_snapshot!(run_part.metadata, @r#" + { + "node-index": 2, + "node-total": 4 + } + "#); + }, + ); + } +} diff --git a/src/run_environment/circleci/snapshots/codspeed_runner__run_environment__circleci__provider__tests__pull_request_run_environment_metadata-2.snap b/src/run_environment/circleci/snapshots/codspeed_runner__run_environment__circleci__provider__tests__pull_request_run_environment_metadata-2.snap new file mode 100644 index 000000000..58144a2c1 --- /dev/null +++ b/src/run_environment/circleci/snapshots/codspeed_runner__run_environment__circleci__provider__tests__pull_request_run_environment_metadata-2.snap @@ -0,0 +1,13 @@ +--- +source: src/run_environment/circleci/provider.rs +expression: run_part +--- +{ + "runId": "8d8f0b2a-1f3e-4b6a-9c2d-0f1e2a3b4c5d", + "runPartId": "benchmarks-0", + "jobName": "benchmarks", + "metadata": { + "node-index": 0, + "node-total": 1 + } +} diff --git a/src/run_environment/circleci/snapshots/codspeed_runner__run_environment__circleci__provider__tests__pull_request_run_environment_metadata.snap b/src/run_environment/circleci/snapshots/codspeed_runner__run_environment__circleci__provider__tests__pull_request_run_environment_metadata.snap new file mode 100644 index 000000000..ab99e3aad --- /dev/null +++ b/src/run_environment/circleci/snapshots/codspeed_runner__run_environment__circleci__provider__tests__pull_request_run_environment_metadata.snap @@ -0,0 +1,17 @@ +--- +source: src/run_environment/circleci/provider.rs +expression: run_environment_metadata +--- +{ + "ref": "refs/pull/22/merge", + "headRef": "feat/codspeed-runner", + "baseRef": null, + "owner": "my-org", + "repository": "adrien-python-test", + "event": "pull_request", + "sender": null, + "ghData": null, + "glData": null, + "localData": null, + "repositoryRootPath": "/home/circleci/project/" +} diff --git a/src/run_environment/interfaces.rs b/src/run_environment/interfaces.rs index 2d2e83383..a3c8d9289 100644 --- a/src/run_environment/interfaces.rs +++ b/src/run_environment/interfaces.rs @@ -29,6 +29,7 @@ pub enum RunEnvironment { GithubActions, GitlabCi, Buildkite, + Circleci, Local, } diff --git a/src/run_environment/mod.rs b/src/run_environment/mod.rs index ddebd46fe..77ba7faa0 100644 --- a/src/run_environment/mod.rs +++ b/src/run_environment/mod.rs @@ -3,6 +3,7 @@ pub mod logger; mod provider; use buildkite::BuildkiteProvider; +use circleci::CircleCIProvider; use github_actions::GitHubActionsProvider; use gitlab_ci::GitLabCIProvider; use local::LocalProvider; @@ -17,6 +18,7 @@ pub use self::provider::RunEnvironmentProvider; // RunEnvironment Provider implementations mod buildkite; +mod circleci; mod github_actions; mod gitlab_ci; mod local; @@ -29,6 +31,9 @@ pub async fn get_provider( if BuildkiteProvider::detect() { let provider = BuildkiteProvider::try_from(config)?; Box::new(provider) + } else if CircleCIProvider::detect() { + let provider = CircleCIProvider::try_from(config)?; + Box::new(provider) } else if GitHubActionsProvider::detect() { let provider = GitHubActionsProvider::try_from(config)?; Box::new(provider) diff --git a/src/upload/uploader.rs b/src/upload/uploader.rs index c24bafe3a..9bcb4c0e1 100644 --- a/src/upload/uploader.rs +++ b/src/upload/uploader.rs @@ -155,7 +155,8 @@ async fn retrieve_upload_data( RunEnvironment::GitlabCi => { "Check that the CI job is correctly authenticated. View more at https://codspeed.io/docs/integrations/ci/gitlab-ci/configuration#authentication" } - RunEnvironment::Buildkite => { + // TODO: support OIDC for CircleCI + RunEnvironment::Buildkite | RunEnvironment::Circleci => { "Check that CODSPEED_TOKEN is set and has the correct value" } RunEnvironment::Local => {