diff --git a/benchmarks/scripts/run-swe-headless.py b/benchmarks/scripts/run-swe-headless.py index aa92d96..3464ae3 100644 --- a/benchmarks/scripts/run-swe-headless.py +++ b/benchmarks/scripts/run-swe-headless.py @@ -33,6 +33,7 @@ import shutil import subprocess # nosec B404 - used with list args, no shell, hardcoded command import sys +import tempfile import threading import time from concurrent.futures import ThreadPoolExecutor, as_completed @@ -125,6 +126,16 @@ def _skill_path(config: RunnerConfig) -> Path: # diverge without a silent coupling. OMP_PROVIDER_VLLM = "vllm" OMP_PROVIDER_BEDROCK = "amazon-bedrock" +# omp's --model is a FUZZY match: a Bedrock id missing from the catalog omp has +# loaded silently resolves to the nearest one it knows (omp 18.1.15 sent every +# us.anthropic.claude-haiku-5-5 request to Haiku 4.5). In a fresh agent dir +# `omp -p` sees only the snapshot bundled in the binary; `omp models` fetches the +# full catalog and caches it in that dir. So each Bedrock task first lists the +# catalog in its own agent dir (which both warms it and proves the exact id is +# there), and every task checks the model each assistant message names. A +# mismatch is a distinct, non-retryable subtype. +OMP_CATALOG_TIMEOUT_SECONDS = 60 +OMP_MODEL_MISMATCH_SUBTYPE = "error_model_mismatch" # kiro-cli binary and the parsers for the one-line summary it prints on stderr at # the end of a non-interactive run, e.g. "▸ Credits: 0.21 • Time: 17s". kiro-cli @@ -755,11 +766,128 @@ def _build_omp_cmd(config: RunnerConfig, prompt: str) -> list[str]: return [*cmd, "--", full_prompt] +def _omp_bedrock_catalog(env: dict[str, str]) -> set[str]: + """List omp's Amazon Bedrock model ids, caching the catalog in the agent dir. + + Runs ``omp models amazon-bedrock --json`` with ``env`` (which pins + ``PI_CODING_AGENT_DIR``). Besides returning the ids, this writes the fetched + catalog into that dir's ``models.db``, so a following ``omp -p`` there resolves + ids the bundled snapshot lacks (see ``OMP_CATALOG_TIMEOUT_SECONDS``). + + Args: + env: Environment for the subprocess, as built by ``_build_omp_env``. + + Returns: + The set of model ids omp lists under its ``amazon-bedrock`` provider. + + Raises: + RuntimeError: If omp is missing, fails, times out, or prints bad JSON. + """ + cmd = ["omp", "models", OMP_PROVIDER_BEDROCK, "--json"] + try: + proc = subprocess.run( # nosec B603 B607 - hardcoded command, list args + cmd, + env=env, + capture_output=True, + text=True, + check=True, + timeout=OMP_CATALOG_TIMEOUT_SECONDS, + stdin=subprocess.DEVNULL, + ) + catalog = json.loads(proc.stdout) + except FileNotFoundError as exc: + raise RuntimeError("omp CLI not found on PATH; see docs/omp-setup.md") from exc + except subprocess.TimeoutExpired as exc: + raise RuntimeError( + f"'{' '.join(cmd)}' timed out after {OMP_CATALOG_TIMEOUT_SECONDS}s" + ) from exc + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f"'{' '.join(cmd)}' failed (exit {exc.returncode}): {exc.stderr[:500]}" + ) from exc + except json.JSONDecodeError as exc: + raise RuntimeError(f"'{' '.join(cmd)}' printed invalid JSON: {exc}") from exc + models = catalog.get("models") if isinstance(catalog, dict) else None + if not isinstance(models, list): + raise RuntimeError(f"'{' '.join(cmd)}' printed no 'models' list") + return {m["id"] for m in models if isinstance(m, dict) and m.get("id")} + + +def _require_omp_bedrock_model(config: RunnerConfig, env: dict[str, str]) -> None: + """Fail unless omp, in ``env``'s agent dir, lists the exact Bedrock model id. + + Call this before every omp Bedrock task: listing the catalog also warms it, + without which a fresh agent dir fuzzy-matches ids the bundled snapshot lacks. + + Args: + config: The runner config (model). + env: The omp subprocess environment for this task's agent dir. + + Raises: + RuntimeError: If the catalog cannot be read or lacks the model id. + """ + wire_id = model_to_wire_id(config.model or "") + if wire_id in _omp_bedrock_catalog(env): + return + raise RuntimeError( + f"omp's Amazon Bedrock catalog does not list '{wire_id}', so " + f"'--model {OMP_PROVIDER_BEDROCK}/{wire_id}' would fuzzy-match a different " + "model. Upgrade omp ('omp update'), confirm with " + f"'omp models find {wire_id}', and check the id against " + "'aws bedrock list-inference-profiles'." + ) + + +def _omp_models_used(events: list[dict[str, Any]]) -> list[str]: + """Return the sorted model ids named by the assistant messages in an omp stream. + + Args: + events: The parsed JSON-lines events omp emitted. + + Returns: + Every distinct ``message.model`` on an assistant ``message_end`` event. + """ + models: set[str] = set() + for event in events: + if event.get("type") != "message_end": + continue + message = event.get("message") or {} + if message.get("role") == "assistant" and message.get("model"): + models.add(message["model"]) + return sorted(models) + + +def _flag_omp_model_mismatch( + result: dict[str, Any], + expected_model: str, + models_used: list[str], +) -> None: + """Mark ``result`` failed when omp answered with a model other than requested. + + Args: + result: The normalized result dict, updated in place. + expected_model: The model id the harness asked omp for. + models_used: The model ids the stream's assistant messages named. + """ + unexpected = [m for m in models_used if m != expected_model] + if not unexpected: + return + message = ( + f"omp answered with {unexpected} instead of the requested " + f"'{expected_model}': its --model fuzzy-matched a different model" + ) + logger.error(message) + result["is_error"] = True + result["subtype"] = OMP_MODEL_MISMATCH_SUBTYPE + result["result"] = message + + def _run_omp( cmd: list[str], env: dict[str, str], timeout: int, stream_log: Path | None = None, + expected_model: str | None = None, ) -> dict[str, Any]: """Run ``omp -p --mode json`` and normalize its events to a result dict. @@ -843,6 +971,9 @@ def _run_omp( ) result = _pi_result_from_events(events, elapsed) result["_elapsed_seconds"] = round(elapsed, 1) + result["models_used"] = _omp_models_used(events) + if expected_model: + _flag_omp_model_mismatch(result, expected_model, result["models_used"]) return result @@ -1266,6 +1397,10 @@ def _metrics_from_result(result: dict[str, Any], elapsed: float) -> dict[str, An # cost is auditable back to what the CLI actually charged. if result.get("kiro_credits") is not None: metrics["kiro_credits"] = result["kiro_credits"] + # agent=omp only: the model ids omp actually answered with, so a fuzzy-matched + # substitute is visible in the recorded metrics (see _flag_omp_model_mismatch). + if result.get("models_used"): + metrics["models_used"] = result["models_used"] # Capture the error message so failures are diagnosable from metrics.json # without re-running the task by hand. if is_error: @@ -3275,6 +3410,10 @@ def _run_task( _write_omp_config(config, omp_agent_dir) cmd = _build_omp_cmd(config, prompt) env = _build_omp_env(config, omp_agent_dir) + if config.is_bedrock: + # Warm this task's catalog and prove the exact id is in it; a + # fresh agent dir otherwise fuzzy-matches (see _omp_bedrock_catalog). + _require_omp_bedrock_model(config, env) omp_cap = ( f"max-time {config.agent_max_time_seconds}s" if config.agent_max_time_seconds @@ -3367,6 +3506,11 @@ def _run_task( config.timeout_seconds, stream_log=_artifact_dir(config, dataset, task) / "omp-stream.jsonl", + expected_model=( + model_to_wire_id(config.model or "") + if config.is_bedrock + else config.model + ), ) elif config.is_kiro: # kiro-cli streams ANSI text and prints a Credits/Time summary on @@ -3598,6 +3742,9 @@ def _summary_is_retryable(summary: dict[str, Any]) -> bool: metrics = summary.get("metrics") or {} if metrics.get("result_subtype") == "error_max_turns": return False + # omp answered with the wrong model: a retry resolves the same id the same way. + if metrics.get("result_subtype") == OMP_MODEL_MISMATCH_SUBTYPE: + return False # Defensive fallback: no subtype recorded but the run clearly ran the turn # budget dry (e.g. an older claude that omits the subtype). Treat a run that # burned >=95% of max_turns without finishing the design as turn exhaustion. @@ -4269,6 +4416,16 @@ def main() -> None: if args.dry_run: _dry_run(config, dataset, tasks) return + if config.is_omp and config.is_bedrock: + # Fail before the first task rather than once per task (with retries). + try: + with tempfile.TemporaryDirectory() as agent_dir: + _require_omp_bedrock_model( + config, _build_omp_env(config, Path(agent_dir)) + ) + except RuntimeError as exc: + logger.error("%s", exc) + sys.exit(1) if config.is_strands: try: _require_strands_sdk(harness=config.is_strands_harness) diff --git a/benchmarks/tests/test_run_swe_headless.py b/benchmarks/tests/test_run_swe_headless.py index 7107a23..eac98b0 100644 --- a/benchmarks/tests/test_run_swe_headless.py +++ b/benchmarks/tests/test_run_swe_headless.py @@ -2312,3 +2312,90 @@ def test_a_run_with_no_input_count_keeps_its_shape(self) -> None: metrics = {"input_tokens": 0, "output_tokens": 0} s = harness._summary_metrics(metrics, self._vllm(1000, 800), 0.0, agent="codex") self.assertEqual(s["input_tokens"], 0) + + +class OmpModelResolutionTest(unittest.TestCase): + """omp fuzzy-matches --model, so the harness must catch a substituted model.""" + + _HAIKU_5_5 = "us.anthropic.claude-haiku-5-5" + _HAIKU_4_5 = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + + def _bedrock_config(self) -> RunnerConfig: + return _config( + agent="omp", provider="bedrock", endpoint=None, model=self._HAIKU_5_5 + ) + + def _events(self, *models: str) -> list[dict[str, object]]: + events: list[dict[str, object]] = [ + {"type": "message_end", "message": {"role": "user"}} + ] + for model in models: + events.append( + { + "type": "message_end", + "message": {"role": "assistant", "model": model}, + } + ) + return events + + def test_models_used_lists_assistant_models_once(self) -> None: + events = self._events(self._HAIKU_4_5, self._HAIKU_4_5) + self.assertEqual(harness._omp_models_used(events), [self._HAIKU_4_5]) + + def test_mismatch_marks_result_failed(self) -> None: + result: dict[str, object] = {"is_error": False, "subtype": "success"} + harness._flag_omp_model_mismatch(result, self._HAIKU_5_5, [self._HAIKU_4_5]) + self.assertEqual( + (result["is_error"], result["subtype"]), + (True, harness.OMP_MODEL_MISMATCH_SUBTYPE), + ) + + def test_matching_model_leaves_result_alone(self) -> None: + result: dict[str, object] = {"is_error": False, "subtype": "success"} + harness._flag_omp_model_mismatch(result, self._HAIKU_5_5, [self._HAIKU_5_5]) + self.assertEqual(result, {"is_error": False, "subtype": "success"}) + + def test_mismatch_is_not_retried(self) -> None: + summary = { + "ok": False, + "metrics": {"result_subtype": harness.OMP_MODEL_MISMATCH_SUBTYPE}, + } + self.assertFalse(harness._summary_is_retryable(summary)) + + def test_metrics_record_models_used(self) -> None: + result = {"usage": {}, "models_used": [self._HAIKU_4_5]} + metrics = harness._metrics_from_result(result, 1.0) + self.assertEqual(metrics["models_used"], [self._HAIKU_4_5]) + + def test_require_model_raises_when_catalog_lacks_id(self) -> None: + with mock.patch.object( + harness, "_omp_bedrock_catalog", return_value={self._HAIKU_4_5} + ): + with self.assertRaisesRegex(RuntimeError, "fuzzy-match"): + harness._require_omp_bedrock_model(self._bedrock_config(), {}) + + def test_require_model_passes_when_catalog_lists_id(self) -> None: + with mock.patch.object( + harness, "_omp_bedrock_catalog", return_value={self._HAIKU_5_5} + ): + self.assertIsNone( + harness._require_omp_bedrock_model(self._bedrock_config(), {}) + ) + + def test_catalog_parses_model_ids(self) -> None: + stdout = json.dumps({"models": [{"id": self._HAIKU_5_5}, {"name": "no id"}]}) + completed = mock.Mock(stdout=stdout) + with mock.patch.object(harness.subprocess, "run", return_value=completed): + self.assertEqual(harness._omp_bedrock_catalog({}), {self._HAIKU_5_5}) + + def test_catalog_without_models_list_raises_runtime_error(self) -> None: + completed = mock.Mock(stdout=json.dumps([{"id": self._HAIKU_5_5}])) + with mock.patch.object(harness.subprocess, "run", return_value=completed): + with self.assertRaisesRegex(RuntimeError, "no 'models' list"): + harness._omp_bedrock_catalog({}) + + def test_catalog_timeout_raises_runtime_error(self) -> None: + timeout = harness.subprocess.TimeoutExpired(cmd="omp", timeout=60) + with mock.patch.object(harness.subprocess, "run", side_effect=timeout): + with self.assertRaisesRegex(RuntimeError, "timed out"): + harness._omp_bedrock_catalog({})