feat: Add --max-concurrency flag to bt eval

README.md

@@ -210,6 +210,8 @@ bt eval foo.eval.ts -- --description "Prod" --shard=1/4
 - `bt eval --sample 20 --sample-seed 7 qa.eval.ts` — run a deterministic random sample and clearly label the summary as a non-final smoke run.
 - If you do not pass a sampling flag, `bt eval` runs the full dataset and marks the summary as final.
 
+Use `--max-concurrency <n>` to limit how many evaluators run at once. This does not change the concurrency configured inside an individual evaluator.
+
 ## `bt datasets`
 
 - `bt datasets` works directly against remote Braintrust datasets — no local `bt sync` artifact flow is required.

scripts/eval-runner-impl.ts

@@ -121,6 +121,7 @@ type RunnerConfig = {
   jsonl: boolean;
   list: boolean;
   terminateOnFailure: boolean;
+  maxConcurrency: number | null;
   autoInstrumentation: boolean;
   filters: EvalFilter[];
   first: number | null;
@@ -421,6 +422,7 @@ function readRunnerConfig(): RunnerConfig {
     jsonl: envFlag("BT_EVAL_JSONL"),
     list: envFlag("BT_EVAL_LIST"),
     terminateOnFailure: envFlag("BT_EVAL_TERMINATE_ON_FAILURE"),
+    maxConcurrency: parsePositiveIntegerEnv("BT_EVAL_MAX_CONCURRENCY"),
     autoInstrumentation: !envFlag("BT_EVAL_NO_AUTO_INSTRUMENTATION"),
     filters: parseSerializedFilters(process.env.BT_EVAL_FILTER_PARSED),
     first: parsePositiveIntegerEnv("BT_EVAL_FIRST"),
@@ -2269,8 +2271,8 @@ async function createEvalRunner(
   sse: SseWriter | null,
 ): Promise<EvalRunner> {
   const braintrust = await loadBraintrust();
-  const Eval = braintrust.Eval;
-  if (typeof Eval !== "function") {
+  const sdkEval = braintrust.Eval;
+  if (typeof sdkEval !== "function") {
     throw new Error("Unable to load Eval() from braintrust package.");
   }
   const login = braintrust.login;
@@ -2281,6 +2283,32 @@ async function createEvalRunner(
   const parseParent = loadBraintrustUtilParseParent();
   const getState = extractGlobalStateGetter(braintrust);
 
+  let availableEvalSlots = config.maxConcurrency ?? 0;
+  const evalSlotWaiters: Array<() => void> = [];
+  const withEvalSlot = async <T>(run: () => Promise<T>): Promise<T> => {
+    if (config.maxConcurrency !== null) {
+      if (availableEvalSlots > 0) {
+        availableEvalSlots -= 1;
+      } else {
+        await new Promise<void>((resolve) => evalSlotWaiters.push(resolve));
+      }
+    }
+
+    try {
+      return await run();
+    } finally {
+      if (config.maxConcurrency !== null) {
+        const next = evalSlotWaiters.shift();
+        if (next) {
+          next();
+        } else {
+          availableEvalSlots += 1;
+        }
+      }
+    }
+  };
+  const Eval: EvalFunction = (...args) => withEvalSlot(() => sdkEval(...args));
+
   const makeEvalOptions = (
     evaluatorName: string,
     overrides?: EvalOptions,
@@ -2313,7 +2341,7 @@ async function createEvalRunner(
     return mergeEvalOptions(base, overrides);
   };
 
-  const runEval = async (
+  const runEvalUnbounded = async (
     projectName: string,
     evaluator: Record<string, unknown>,
     options?: EvalOptions,
@@ -2344,7 +2372,7 @@ async function createEvalRunner(
       ...evaluator,
       data: sampledData,
     });
-    const result = await Eval(projectName, wrappedEvaluator, opts);
+    const result = await sdkEval(projectName, wrappedEvaluator, opts);
     const summary = attachSamplingSummary(result.summary, config);
     const failingResults = result.results.filter(
       (r: { error?: unknown }) => r.error !== undefined,
@@ -2362,6 +2390,8 @@ async function createEvalRunner(
     }
     return result;
   };
+  const runEval: EvalRunner["runEval"] = (...args) =>
+    withEvalSlot(() => runEvalUnbounded(...args));
 
   const runRegisteredEvals = async (evaluators: EvaluatorEntry[]) => {
     if (sse) {

scripts/eval-runner.py

@@ -82,6 +82,7 @@ class RunnerConfig:
     jsonl: bool
     list_only: bool
     terminate_on_failure: bool
+    max_concurrency: int | None
     num_workers: int | None
     filters: list[EvalFilter]
     first: int | None
@@ -270,6 +271,7 @@ def read_runner_config() -> RunnerConfig:
         jsonl=env_flag("BT_EVAL_JSONL"),
         list_only=env_flag("BT_EVAL_LIST"),
         terminate_on_failure=env_flag("BT_EVAL_TERMINATE_ON_FAILURE"),
+        max_concurrency=parse_positive_int_env("BT_EVAL_MAX_CONCURRENCY"),
         num_workers=num_workers,
         filters=parse_serialized_filters(os.getenv("BT_EVAL_FILTER_PARSED")),
         first=parse_positive_int_env("BT_EVAL_FIRST"),
@@ -1324,6 +1326,11 @@ async def run_once(
         sse.send("processing", {"evaluators": len(evaluators)})
 
     progress_mode = run_evaluator_progress_mode()
+    eval_semaphore = (
+        asyncio.Semaphore(config.max_concurrency)
+        if config.max_concurrency is not None
+        else None
+    )
 
     async def run_single_evaluator(
         idx: int, evaluator_instance: EvaluatorInstance
@@ -1350,7 +1357,11 @@ async def run_single_evaluator(
             # command running multiple evaluators doesn't fail on unrelated params.
             filtered_params = filter_params_for_evaluator(effective_params, evaluator.parameters)
             evaluator.parameters = validate_parameters(filtered_params, evaluator.parameters)
+        acquired_eval_slot = False
         try:
+            if eval_semaphore is not None:
+                await eval_semaphore.acquire()
+                acquired_eval_slot = True
             result = await run_evaluator_task(
                 evaluator_instance.evaluator,
                 idx,
@@ -1363,6 +1374,9 @@ async def run_single_evaluator(
         except Exception as exc:
             err = serialize_error(str(exc), traceback.format_exc())
             return evaluator_instance, resolved_reporter, None, err
+        finally:
+            if acquired_eval_slot and eval_semaphore is not None:
+                eval_semaphore.release()
 
         return evaluator_instance, resolved_reporter, result, None
 

src/eval.rs

@@ -312,6 +312,17 @@ pub struct EvalArgs {
     )]
     pub terminate_on_failure: bool,
 
+    /// Maximum number of evaluators to run concurrently.
+    #[arg(
+        long,
+        visible_alias = "maxConcurrency",
+        env = "BT_EVAL_MAX_CONCURRENCY",
+        value_name = "COUNT",
+        value_parser = parse_positive_usize,
+        conflicts_with = "dev"
+    )]
+    pub max_concurrency: Option<usize>,
+
     /// Number of worker threads for Python eval execution.
     #[arg(long, env = "BT_EVAL_NUM_WORKERS", value_name = "COUNT")]
     pub num_workers: Option<usize>,
@@ -454,6 +465,7 @@ enum EvalSamplingMode {
 struct EvalRunOptions {
     jsonl: bool,
     terminate_on_failure: bool,
+    max_concurrency: Option<usize>,
     num_workers: Option<usize>,
     list: bool,
     filter: Vec<String>,
@@ -506,6 +518,7 @@ pub async fn run(base: BaseArgs, args: EvalArgs) -> Result<()> {
     let options = EvalRunOptions {
         jsonl: args.jsonl,
         terminate_on_failure: args.terminate_on_failure,
+        max_concurrency: args.max_concurrency,
         num_workers: args.num_workers,
         list: args.list,
         filter: args.filter,
@@ -867,6 +880,9 @@ async fn spawn_eval_runner(
     if options.terminate_on_failure {
         cmd.env("BT_EVAL_TERMINATE_ON_FAILURE", "1");
     }
+    if let Some(max_concurrency) = options.max_concurrency {
+        cmd.env("BT_EVAL_MAX_CONCURRENCY", max_concurrency.to_string());
+    }
     if options.list {
         cmd.env("BT_EVAL_LIST", "1");
     }
@@ -5098,6 +5114,50 @@ mod tests {
         }
     }
 
+    #[test]
+    fn eval_args_parse_max_concurrency_flag_and_alias() {
+        let _guard = env_test_lock()
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+        let previous = clear_env_var("BT_EVAL_MAX_CONCURRENCY");
+
+        let parsed =
+            EvalArgsHarness::try_parse_from(["bt", "--max-concurrency", "2", "sample.eval.ts"])
+                .expect("max-concurrency flag should parse");
+        assert_eq!(parsed.eval.max_concurrency, Some(2));
+
+        let parsed =
+            EvalArgsHarness::try_parse_from(["bt", "--maxConcurrency=3", "sample.eval.ts"])
+                .expect("maxConcurrency alias should parse");
+        assert_eq!(parsed.eval.max_concurrency, Some(3));
+
+        let err =
+            EvalArgsHarness::try_parse_from(["bt", "--max-concurrency", "0", "sample.eval.ts"])
+                .expect_err("zero max concurrency should fail");
+        assert!(err.to_string().contains("greater than 0"));
+
+        let err = EvalArgsHarness::try_parse_from([
+            "bt",
+            "--dev",
+            "--max-concurrency",
+            "2",
+            "sample.eval.ts",
+        ])
+        .expect_err("max concurrency with dev mode should fail");
+        let message = err.to_string();
+        assert!(message.contains("--max-concurrency"));
+        assert!(message.contains("--dev"));
+
+        set_env_var("BT_EVAL_MAX_CONCURRENCY", "2");
+        let err = EvalArgsHarness::try_parse_from(["bt", "--dev", "sample.eval.ts"])
+            .expect_err("max concurrency env var with dev mode should fail");
+        let message = err.to_string();
+        assert!(message.contains("--max-concurrency"));
+        assert!(message.contains("--dev"));
+
+        restore_env_var("BT_EVAL_MAX_CONCURRENCY", previous);
+    }
+
     #[test]
     fn eval_args_from_env_populates_supported_fields() {
         let _guard = env_test_lock()
@@ -5106,6 +5166,7 @@ mod tests {
         let keys = [
             "BT_EVAL_JSONL",
             "BT_EVAL_TERMINATE_ON_FAILURE",
+            "BT_EVAL_MAX_CONCURRENCY",
             "BT_EVAL_NUM_WORKERS",
             "BT_EVAL_LIST",
             "BT_EVAL_FILTER",
@@ -5123,6 +5184,7 @@ mod tests {
             keys.iter().map(|key| (*key, clear_env_var(key))).collect();
         set_env_var("BT_EVAL_JSONL", "true");
         set_env_var("BT_EVAL_TERMINATE_ON_FAILURE", "1");
+        set_env_var("BT_EVAL_MAX_CONCURRENCY", "3");
         set_env_var("BT_EVAL_NUM_WORKERS", "4");
         set_env_var("BT_EVAL_LIST", "yes");
         set_env_var("BT_EVAL_FILTER", "metadata.case=smoke.*,metadata.kind=fast");
@@ -5137,6 +5199,7 @@ mod tests {
             .expect("env vars should parse into eval args");
         assert!(parsed.eval.jsonl);
         assert!(parsed.eval.terminate_on_failure);
+        assert_eq!(parsed.eval.max_concurrency, Some(3));
         assert_eq!(parsed.eval.num_workers, Some(4));
         assert!(parsed.eval.list);
         assert_eq!(