From 449f949ce8589a42f279c4a1eb70b7920e1e479a Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 28 Oct 2026 08:58:26 -0700 Subject: [PATCH 01/12] refactor(arms): move the run's shared checks to dokimi/eval/arms_run.py resolve_doc_library_id, validate_expected_skills, probe_sampling and validate_supported_matrix_harnesses read no parsed arguments, and judge_reliability imported the probe from another command. Move them, unchanged, into a new evaluation-layer module under public names; the commands and the tests import them from there. validate_expected_skills collects the offered skill names in a helper of its own, so the moved function stays under the complexity cap and its growth baseline entry goes. Refs #4 Signed-off-by: Vladislav Perevezentsev Signed-off-by: Nikolay Petrov --- dokimi/docs/evaluating-treatments.md | 1 + dokimi/dokimi/commands/arms.py | 124 ++---------------- dokimi/dokimi/commands/judge_reliability.py | 4 +- dokimi/dokimi/eval/arms_run.py | 123 +++++++++++++++++ dokimi/scripts/growth_baseline.json | 1 - dokimi/tests/test_arm_runner.py | 22 ++-- dokimi/tests/test_empty_question_rule.py | 6 +- .../test_judge_temperature_entry_points.py | 6 +- dokimi/tests/test_question_ids.py | 6 +- dokimi/tests/test_session_runner.py | 2 +- dokimi/tests/test_skill_selection.py | 6 +- dokimi/tests/test_treatments.py | 12 +- 12 files changed, 168 insertions(+), 145 deletions(-) create mode 100644 dokimi/dokimi/eval/arms_run.py diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 691ab695a..588612114 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -2,6 +2,7 @@ depends_on: - dokimi/dokimi/treatments/** - dokimi/dokimi/commands/arms.py + - dokimi/dokimi/eval/arms_run.py - dokimi/dokimi/schemas/arms.v1.json - dokimi/dokimi/skills/loader.py - dokimi/dokimi/agent_profiles/loader.py diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 0198feb0d..f9ee36253 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -6,7 +6,6 @@ from __future__ import annotations import argparse -import functools from collections import Counter import sys from pathlib import Path @@ -108,6 +107,7 @@ def cmd_arms_run(args: argparse.Namespace) -> None: def cmd_arms_matrix_run(args: argparse.Namespace) -> None: """Run every cell in a matrix.cells descriptor and write a rollup.""" + from dokimi.eval.arms_run import validate_supported_matrix_harnesses from dokimi.eval.cells import load_matrix_cells from dokimi.eval.matrix_rollup import build_matrix_rollup, strip_internal_payloads from dokimi.report.matrix_rollup_report import render_matrix_rollup_report @@ -117,7 +117,7 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: specs = _parse_arm_specs(args.arms) questions = _load_questions(args.questions) cells = load_matrix_cells(args.matrix_cells) - _validate_supported_matrix_harnesses(cells) + validate_supported_matrix_harnesses(cells) _validate_unique_cell_filenames(cells) for cell in cells: cell_args = argparse.Namespace(**vars(args)) @@ -337,37 +337,6 @@ def register(sub, positive_int) -> None: plugin_delta_p.set_defaults(func=cmd_arms_plugin_delta) -def _resolve_doc_library_id(treatments, product: str, explicit_id=None, registry=None): - """Resolve a library id for the first doc arm, before plugin wrapping. - - Same three-step order as the ``answers``/``questions`` commands: explicit - flag, then the product's ``context7_id`` in ``products.yaml``, then the - client's own resolution. When no id can be named the ``ValueError`` is - raised to the caller, which stops the run: substituting the product name - would query an id that does not exist and record an empty-retrieval doc - arm as a measurement. Other failures (an unreachable MCP server, say) still - fall back to the product name and are reported by the arm itself. - """ - if explicit_id is not None: - return explicit_id - - from dokimi.mcp.factory import resolve_library_id_for_product - from dokimi.treatments.arms import DocTreatment - - for treatment in treatments: - unwrapped = getattr(treatment, "inner", treatment) - if isinstance(unwrapped, DocTreatment): - try: - return resolve_library_id_for_product( - product, unwrapped.mcp_client, registry=registry - ) - except ValueError: - raise - except Exception: - return product - return None - - def _add_common_run_args(parser, positive_int, *, matrix_required: bool = False) -> None: parser.add_argument("--product", required=True, help="Product name (e.g., oneTBB)") parser.add_argument("--questions", required=True, help="Path to questions JSON file") @@ -503,51 +472,15 @@ def _refuse_duplicate_ids(ids: list, path: str) -> None: raise ValueError(f"duplicate question ids in {path}: {', '.join(map(repr, dupes[:5]))}") -def _validate_expected_skills(questions: list[dict], treatments) -> None: - """Refuse ``expected_skill`` labels the selection summary cannot score. - - Checked before any call is spent: a label of the wrong type crashes the - summary after the run, and a name no skill arm offers silently scores as a - miss. Names are checked only when some arm offers skills. - """ - from dokimi.treatments import SkillAgentTreatment, SkillCatalogTreatment - - def leaves(t): - t = getattr(t, "inner", t) - for sub in getattr(t, "treatments", None) or [t]: - yield sub - - known: set = set() - for t in treatments: - for sub in leaves(t): - if isinstance(sub, SkillCatalogTreatment): - known.update(s.name for s in sub.skills) - elif isinstance(sub, SkillAgentTreatment): - known.add(sub.skill.name) - for i, q in enumerate(questions): - if "expected_skill" not in q: - continue - label = q["expected_skill"] - qid = question_id(q) - qid = f"#{i}" if qid is None else qid - names = label if isinstance(label, list) else [label] if label is not None else [] - if not all(isinstance(n, str) and n for n in names): - raise ValueError(f"question {qid}: expected_skill must be a skill name, a list of " - f"names, or null; got {label!r}") - unknown = [n for n in names if known and n not in known] - if unknown: - raise ValueError(f"question {qid}: expected_skill names no offered skill: " - f"{', '.join(unknown)}") - - def _apply_named_matrix_cell(args: argparse.Namespace) -> None: + from dokimi.eval.arms_run import validate_supported_matrix_harnesses from dokimi.eval.cells import load_matrix_cells, select_matrix_cell if args.matrix_cells or args.matrix_cell: if not args.matrix_cells or not args.matrix_cell: raise ValueError("--matrix-cells and --matrix-cell must be passed together") matrix_cell = select_matrix_cell(load_matrix_cells(args.matrix_cells), args.matrix_cell) - _validate_supported_matrix_harnesses([matrix_cell]) + validate_supported_matrix_harnesses([matrix_cell]) _apply_matrix_cell(args, matrix_cell) @@ -561,6 +494,7 @@ def _apply_matrix_cell(args: argparse.Namespace, matrix_cell) -> None: def _check_sampling_support(args: argparse.Namespace) -> None: """Reject sampling flags the answer provider would refuse on every answer.""" + from dokimi.eval.arms_run import probe_sampling from dokimi.llm import unsupported_sampling_params bad = unsupported_sampling_params( @@ -574,26 +508,8 @@ def _check_sampling_support(args: argparse.Namespace) -> None: f"model '{args.model}'. Drop the flag(s); every answer would " "otherwise fail and the report would show null scores." ) - _probe_sampling(args.model, args.provider, - getattr(args, "temperature", None), getattr(args, "seed", None)) - - -@functools.lru_cache(maxsize=None) -def _probe_sampling(model: str, provider: str, temperature, seed) -> None: - """One short real call with the requested sampling, so a value the provider - rejects (gpt-5 accepts only temperature=1) fails before the run, not per answer.""" - if temperature is None and seed is None: - return - from dokimi.llm import llm_call_with_usage - - try: - llm_call_with_usage("Reply with OK.", model=model, provider=provider, - max_tokens=16, temperature=temperature, seed=seed) - except Exception as exc: - raise ValueError( - f"Probe call to '{provider}/{model}' with temperature={temperature}, " - f"seed={seed} failed, so every call in the run would too: {exc}" - ) from exc + probe_sampling(args.model, args.provider, + getattr(args, "temperature", None), getattr(args, "seed", None)) def _refuse_judged_session(args: argparse.Namespace) -> None: @@ -606,6 +522,7 @@ def _build_arms_output(args: argparse.Namespace, specs: list[str], questions: li from dokimi.eval import Judge, JudgePanel from dokimi.eval.panel import JUDGE_ROLES, JudgeConfig from dokimi.eval.arm_runner import ArmRunner + from dokimi.eval.arms_run import probe_sampling, resolve_doc_library_id, validate_expected_skills from dokimi.plugins import ( create_plugins, plugin_set_metadata, @@ -616,9 +533,9 @@ def _build_arms_output(args: argparse.Namespace, specs: list[str], questions: li _refuse_judged_session(args) treatments = create_treatments(specs, top_k=args.top_k, rerank_threshold=args.rerank_threshold) - _validate_expected_skills(questions, treatments) + validate_expected_skills(questions, treatments) try: - library_id = _resolve_doc_library_id(treatments, args.product, args.context7_id) + library_id = resolve_doc_library_id(treatments, args.product, args.context7_id) except ValueError as exc: print(f"Error: {exc}", file=sys.stderr) raise SystemExit(1) from exc @@ -657,8 +574,8 @@ def _build_arms_output(args: argparse.Namespace, specs: list[str], questions: li f"{', '.join(bad_judge)} for model '{args.judge_model}'. Drop the " "flag; every score would otherwise be null." ) - _probe_sampling(judge_model, args.judge_provider, - getattr(args, "judge_temperature", None), None) + probe_sampling(judge_model, args.judge_provider, + getattr(args, "judge_temperature", None), None) # Built before any answer is paid for: an unsupported judge provider raises here. judge: Judge | JudgePanel | None = None @@ -773,23 +690,6 @@ def _validate_unique_cell_filenames(cells) -> None: seen[safe_name] = cell.name -def _validate_supported_matrix_harnesses(cells) -> None: - """Reject matrix harness labels that the in-process runner cannot execute.""" - from dokimi.eval.cells import HARNESS_ARMS_RUNNER - - unsupported = [ - cell.harness - for cell in cells - if cell.harness != HARNESS_ARMS_RUNNER - ] - if unsupported: - raise ValueError( - "matrix cells only support harness='arms-runner' until other runners are wired; " - "unsupported harnesses: " - + ", ".join(sorted(set(unsupported))) - ) - - def _safe_name(name: str) -> str: safe = "".join(c if c.isalnum() or c in ("-", "_", ".") else "-" for c in name) return safe.strip("-") or "cell" diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py index 96ccfe1c1..bcb752827 100644 --- a/dokimi/dokimi/commands/judge_reliability.py +++ b/dokimi/dokimi/commands/judge_reliability.py @@ -118,7 +118,7 @@ def _check_judge_temperature(judges: list[tuple[str, str]], temperature) -> None """Exit 2 before any judgement when a (model, provider) judge rejects ``temperature``.""" if temperature is None: return - from dokimi.commands.arms import _probe_sampling + from dokimi.eval.arms_run import probe_sampling from dokimi.llm import unsupported_sampling_params from dokimi.utils import bedrock_litellm_model @@ -128,7 +128,7 @@ def _check_judge_temperature(judges: list[tuple[str, str]], temperature) -> None try: if unsupported_sampling_params(resolved, provider, temperature=temperature): raise ValueError(f"provider '{provider}' does not support temperature for model '{model}'") - _probe_sampling(resolved, provider, temperature, None) + probe_sampling(resolved, provider, temperature, None) except ValueError as exc: print(f"Error: --judge-temperature {temperature} is rejected by judge {provider}/{model}: {exc}", file=sys.stderr) diff --git a/dokimi/dokimi/eval/arms_run.py b/dokimi/dokimi/eval/arms_run.py new file mode 100644 index 000000000..4dfca1359 --- /dev/null +++ b/dokimi/dokimi/eval/arms_run.py @@ -0,0 +1,123 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""What an arms run checks and resolves before the first answer is paid for. + +The arm-run commands (``arms run``, ``arms matrix-run``, and ``subjects run`` through +``matrix-run``) and ``eval`` judge-reliability share these: the doc arm's library id, the +``expected_skill`` labels, the harnesses a matrix cell may name, and the sampling probe. +""" + +from __future__ import annotations + +import functools + +from dokimi.eval.cells import HARNESS_ARMS_RUNNER +from dokimi.utils import question_id + + +def validate_supported_matrix_harnesses(cells) -> None: + """Reject matrix harness labels that the in-process runner cannot execute.""" + unsupported = [ + cell.harness + for cell in cells + if cell.harness != HARNESS_ARMS_RUNNER + ] + if unsupported: + raise ValueError( + "matrix cells only support harness='arms-runner' until other runners are wired; " + "unsupported harnesses: " + + ", ".join(sorted(set(unsupported))) + ) + + +def resolve_doc_library_id(treatments, product: str, explicit_id=None, registry=None): + """Resolve a library id for the first doc arm, before plugin wrapping. + + Same three-step order as the ``answers``/``questions`` commands: explicit + flag, then the product's ``context7_id`` in ``products.yaml``, then the + client's own resolution. When no id can be named the ``ValueError`` is + raised to the caller, which stops the run: substituting the product name + would query an id that does not exist and record an empty-retrieval doc + arm as a measurement. Other failures (an unreachable MCP server, say) still + fall back to the product name and are reported by the arm itself. + """ + if explicit_id is not None: + return explicit_id + + from dokimi.mcp.factory import resolve_library_id_for_product + from dokimi.treatments.arms import DocTreatment + + for treatment in treatments: + unwrapped = getattr(treatment, "inner", treatment) + if isinstance(unwrapped, DocTreatment): + try: + return resolve_library_id_for_product( + product, unwrapped.mcp_client, registry=registry + ) + except ValueError: + raise + except Exception: + return product + return None + + +def _offered_skill_names(treatments) -> set: + """The names of the skills any arm in *treatments* offers, past plugins and stacks.""" + from dokimi.treatments import SkillAgentTreatment, SkillCatalogTreatment + + def leaves(t): + t = getattr(t, "inner", t) + for sub in getattr(t, "treatments", None) or [t]: + yield sub + + known: set = set() + for t in treatments: + for sub in leaves(t): + if isinstance(sub, SkillCatalogTreatment): + known.update(s.name for s in sub.skills) + elif isinstance(sub, SkillAgentTreatment): + known.add(sub.skill.name) + return known + + +def validate_expected_skills(questions: list[dict], treatments) -> None: + """Refuse ``expected_skill`` labels the selection summary cannot score. + + Checked before any call is spent: a label of the wrong type crashes the + summary after the run, and a name no skill arm offers silently scores as a + miss. Names are checked only when some arm offers skills. + """ + known = _offered_skill_names(treatments) + for i, q in enumerate(questions): + if "expected_skill" not in q: + continue + label = q["expected_skill"] + qid = question_id(q) + qid = f"#{i}" if qid is None else qid + names = label if isinstance(label, list) else [label] if label is not None else [] + if not all(isinstance(n, str) and n for n in names): + raise ValueError(f"question {qid}: expected_skill must be a skill name, a list of " + f"names, or null; got {label!r}") + unknown = [n for n in names if known and n not in known] + if unknown: + raise ValueError(f"question {qid}: expected_skill names no offered skill: " + f"{', '.join(unknown)}") + + +@functools.lru_cache(maxsize=None) +def probe_sampling(model: str, provider: str, temperature, seed) -> None: + """One short real call with the requested sampling, so a value the provider + rejects (gpt-5 accepts only temperature=1) fails before the run, not per answer.""" + if temperature is None and seed is None: + return + from dokimi.llm import llm_call_with_usage + + try: + llm_call_with_usage("Reply with OK.", model=model, provider=provider, + max_tokens=16, temperature=temperature, seed=seed) + except Exception as exc: + raise ValueError( + f"Probe call to '{provider}/{model}' with temperature={temperature}, " + f"seed={seed} failed, so every call in the run would too: {exc}" + ) from exc diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 7ad83253b..b8a876b70 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -17,7 +17,6 @@ }, "functions": { "dokimi/commands/arms.py::_build_arms_output": {"complexity": 13}, - "dokimi/commands/arms.py::_validate_expected_skills": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 21}, "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 15}, diff --git a/dokimi/tests/test_arm_runner.py b/dokimi/tests/test_arm_runner.py index 7fdd6af32..5babbb31c 100644 --- a/dokimi/tests/test_arm_runner.py +++ b/dokimi/tests/test_arm_runner.py @@ -513,7 +513,7 @@ def test_matrix_run_checks_every_cell_before_running_the_first(monkeypatch, tmp_ monkeypatch.setattr("dokimi.llm.unsupported_sampling_params", lambda model, provider, **kw: ["seed"] if model == "m-bad" else []) - monkeypatch.setattr(arms_mod, "_probe_sampling", lambda *a: None) + monkeypatch.setattr("dokimi.eval.arms_run.probe_sampling", lambda *a: None) ran = [] monkeypatch.setattr(arms_mod, "_build_arms_output", lambda *a, **kw: ran.append(1)) args = build_parser().parse_args( @@ -526,30 +526,30 @@ def test_matrix_run_checks_every_cell_before_running_the_first(monkeypatch, tmp_ def test_probe_rejects_a_sampling_value_the_provider_refuses(monkeypatch): - import dokimi.commands.arms as arms_mod + import dokimi.eval.arms_run as arms_run import dokimi.llm as llm_mod def refuse(prompt, model, provider, **kw): raise Exception("UnsupportedParamsError: gpt-5 models don't support temperature=0.0") monkeypatch.setattr(llm_mod, "llm_call_with_usage", refuse) - arms_mod._probe_sampling.cache_clear() + arms_run.probe_sampling.cache_clear() with pytest.raises(ValueError, match="temperature=0.0"): - arms_mod._probe_sampling("gpt-5", "openai", 0.0, None) + arms_run.probe_sampling("gpt-5", "openai", 0.0, None) def test_probe_makes_no_call_without_sampling_flags_and_one_per_configuration(monkeypatch): - import dokimi.commands.arms as arms_mod + import dokimi.eval.arms_run as arms_run import dokimi.llm as llm_mod calls = [] monkeypatch.setattr(llm_mod, "llm_call_with_usage", lambda prompt, model, provider, **kw: calls.append(kw) or ("OK", None)) - arms_mod._probe_sampling.cache_clear() - arms_mod._probe_sampling("m", "openai", None, None) + arms_run.probe_sampling.cache_clear() + arms_run.probe_sampling("m", "openai", None, None) assert calls == [] - arms_mod._probe_sampling("m", "openai", 0.0, None) - arms_mod._probe_sampling("m", "openai", 0.0, None) + arms_run.probe_sampling("m", "openai", 0.0, None) + arms_run.probe_sampling("m", "openai", 0.0, None) assert len(calls) == 1 and calls[0]["temperature"] == 0.0 @@ -947,7 +947,7 @@ def _litellm_resp(text): def test_a_bare_bedrock_judge_name_is_checked_as_the_judge_will_call_it(monkeypatch): import litellm - import dokimi.commands.arms as arms_mod + import dokimi.eval.arms_run as arms_run from dokimi.commands.arms import _build_arms_output from dokimi.utils import get_llm @@ -968,7 +968,7 @@ def completion(model, messages, **kw): monkeypatch.setattr(litellm, "get_supported_openai_params", supported) monkeypatch.setattr(litellm, "completion", completion) monkeypatch.setattr(ArmRunner, "run", lambda self, *a, **kw: []) - arms_mod._probe_sampling.cache_clear() + arms_run.probe_sampling.cache_clear() args = _parse_arms_run(["--judge", "--judge-provider", "amazon-bedrock", "--judge-model", "claude-sonnet-4-5", "--judge-temperature", "0"]) _build_arms_output(args, ["baseline"], QUESTIONS[:1]) diff --git a/dokimi/tests/test_empty_question_rule.py b/dokimi/tests/test_empty_question_rule.py index 8b088a041..f2b5f95a9 100644 --- a/dokimi/tests/test_empty_question_rule.py +++ b/dokimi/tests/test_empty_question_rule.py @@ -176,7 +176,7 @@ def test_cli_reports_the_record_and_exits_2(tmp_path, monkeypatch, capsys): def test_arms_run_refuses_a_blank_question_before_any_answer_is_paid_for(tmp_path, monkeypatch, capsys): """The question-set reader accepts a whitespace-only question; the judge would refuse it only after every arm had answered, and the run's output is never written.""" - import dokimi.commands.arms as arms_mod + import dokimi.eval.arms_run as arms_run from dokimi import cli questions = tmp_path / "q.json" @@ -185,7 +185,7 @@ def test_arms_run_refuses_a_blank_question_before_any_answer_is_paid_for(tmp_pat answered = [] monkeypatch.setattr(ArmRunner, "run", lambda self, library_name, questions, **kw: answered.append(1) or [ _arms_record(q["id"], q["question"]) for q in questions]) - monkeypatch.setattr(arms_mod, "_probe_sampling", lambda *a: None) + monkeypatch.setattr(arms_run, "probe_sampling", lambda *a: None) out = tmp_path / "arms.json" monkeypatch.setattr(sys, "argv", ["dokimi", "arms", "run", "--product", "lib", "--questions", str(questions), "--arms", "baseline", "--judge", "--out-json", str(out)]) @@ -325,7 +325,7 @@ def test_no_probe_call_before_the_refusal(cmd, tmp_path, monkeypatch, capsys): {"id": "q1", "question": "How?", "expected_topics": ["fit"]}, {"id": "q2", "question": "Why?", "expected_topics": ["predict"]}]}), encoding="utf-8") calls = [] - monkeypatch.setattr("dokimi.commands.arms._probe_sampling", lambda *a: calls.append(a)) + monkeypatch.setattr("dokimi.eval.arms_run.probe_sampling", lambda *a: calls.append(a)) monkeypatch.setattr("dokimi.llm.unsupported_sampling_params", lambda *a, **k: []) argv = [a.format(questions=questions) for a in cmd] monkeypatch.setattr(sys, "argv", ["dokimi", *argv, "--answers", str(answers), "--judge-temperature", "0.2"]) diff --git a/dokimi/tests/test_judge_temperature_entry_points.py b/dokimi/tests/test_judge_temperature_entry_points.py index ad424a860..fef94e5b8 100644 --- a/dokimi/tests/test_judge_temperature_entry_points.py +++ b/dokimi/tests/test_judge_temperature_entry_points.py @@ -112,9 +112,9 @@ def routed(monkeypatch): """Records every request; replies so each command's parser accepts the reply.""" import litellm - from dokimi.commands.arms import _probe_sampling + from dokimi.eval.arms_run import probe_sampling - _probe_sampling.cache_clear() + probe_sampling.cache_clear() sent = [] def completion(**kw): @@ -128,7 +128,7 @@ def completion(**kw): monkeypatch.setattr(litellm, "completion", completion) yield sent - _probe_sampling.cache_clear() + probe_sampling.cache_clear() def _is_probe(request): diff --git a/dokimi/tests/test_question_ids.py b/dokimi/tests/test_question_ids.py index 0074927ed..705fb696a 100644 --- a/dokimi/tests/test_question_ids.py +++ b/dokimi/tests/test_question_ids.py @@ -131,9 +131,9 @@ def test_answerer_keeps_question_zero(): def test_expected_skill_error_names_question_zero(): - from dokimi.commands.arms import _validate_expected_skills + from dokimi.eval.arms_run import validate_expected_skills with pytest.raises(ValueError, match="question 0: expected_skill"): - _validate_expected_skills([{"id": 0, "question": "q?", "expected_skill": 5}], []) + validate_expected_skills([{"id": 0, "question": "q?", "expected_skill": 5}], []) with pytest.raises(ValueError, match="question #0: expected_skill"): - _validate_expected_skills([{"question": "q?", "expected_skill": 5}], []) + validate_expected_skills([{"question": "q?", "expected_skill": 5}], []) diff --git a/dokimi/tests/test_session_runner.py b/dokimi/tests/test_session_runner.py index 32ace6bcb..ad45078fd 100644 --- a/dokimi/tests/test_session_runner.py +++ b/dokimi/tests/test_session_runner.py @@ -506,7 +506,7 @@ def test_a_session_artifact_is_refused_by_every_judge_command(cmd, fake_llm, tmp encoding="utf-8") questions = tmp_path / "q.json" questions.write_text(json.dumps({"questions": [q]}), encoding="utf-8") - monkeypatch.setattr("dokimi.commands.arms._probe_sampling", lambda *a: None) + monkeypatch.setattr("dokimi.eval.arms_run.probe_sampling", lambda *a: None) argv = [a.format(questions=questions) for a in cmd] monkeypatch.setattr(sys, "argv", ["dokimi", *argv, "--answers", str(answers)]) diff --git a/dokimi/tests/test_skill_selection.py b/dokimi/tests/test_skill_selection.py index f720d9d2c..c6fdbc98a 100644 --- a/dokimi/tests/test_skill_selection.py +++ b/dokimi/tests/test_skill_selection.py @@ -619,13 +619,13 @@ def no_calls(*a, **kw): def test_arms_run_accepts_labels_from_any_skill_arm(catalog): - from dokimi.commands.arms import _validate_expected_skills + from dokimi.eval.arms_run import validate_expected_skills arms = [create_treatment(f"skill-agent:{catalog / 'linux-perf'}+skill-agent:{catalog / 'dpnp-io'}")] - _validate_expected_skills([{"expected_skill": ["dpnp-io", "linux-perf"]}, + validate_expected_skills([{"expected_skill": ["dpnp-io", "linux-perf"]}, {"expected_skill": None}], arms) # Without a skill arm only the type is checked: the label is not scored. - _validate_expected_skills([{"expected_skill": "anything"}], [create_treatment("baseline")]) + validate_expected_skills([{"expected_skill": "anything"}], [create_treatment("baseline")]) def test_catalog_arm_judge_reference_names_the_catalog_entries(monkeypatch, catalog): diff --git a/dokimi/tests/test_treatments.py b/dokimi/tests/test_treatments.py index e606f1992..624b482a9 100644 --- a/dokimi/tests/test_treatments.py +++ b/dokimi/tests/test_treatments.py @@ -25,7 +25,7 @@ validate_plugins_for_harness, wrap_treatments, ) -from dokimi.commands.arms import _resolve_doc_library_id +from dokimi.eval.arms_run import resolve_doc_library_id # ── frontmatter ───────────────────────────────────────────────────────── @@ -206,19 +206,19 @@ def test_doc_library_id_resolves_before_or_after_plugin_wrapping(tmp_path): wrapped = wrap_treatments([t], [create_plugin("plugin:caveman")]) # Unregistered product: the doc client resolves it, wrapped or not. - assert _resolve_doc_library_id([t], "notaproduct", registry=registry) == "fake/notaproduct" - assert _resolve_doc_library_id(wrapped, "notaproduct", registry=registry) == "fake/notaproduct" + assert resolve_doc_library_id([t], "notaproduct", registry=registry) == "fake/notaproduct" + assert resolve_doc_library_id(wrapped, "notaproduct", registry=registry) == "fake/notaproduct" # Explicit flag wins over everything. assert ( - _resolve_doc_library_id( + resolve_doc_library_id( wrapped, "notaproduct", explicit_id="manual/id", registry=registry ) == "manual/id" ) # A registered product's declared context7_id wins over client resolution. - assert _resolve_doc_library_id(wrapped, "demo", registry=registry) == "declared/id" + assert resolve_doc_library_id(wrapped, "demo", registry=registry) == "declared/id" # Default registry (the committed products.yaml) is used when none is given. - assert _resolve_doc_library_id([t], "notaproduct") == "fake/notaproduct" + assert resolve_doc_library_id([t], "notaproduct") == "fake/notaproduct" # ── factory ───────────────────────────────────────────────────────────── From 59247cecb3cbd1f56cb5848c61788368a1d586b1 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 28 Oct 2026 09:04:56 -0700 Subject: [PATCH 02/12] refactor(arms): run arms run and matrix-run from a typed ArmsRunConfig _build_arms_output read the run configuration off the parsed Namespace, most fields with a getattr default, and matrix-run copied the Namespace per cell and mutated the copy. Add ArmsRunConfig to dokimi/eval/arms_run.py: frozen, the field names are the flags' dest names, and each default is what arms run runs with when the flag is left out. run_arms is the body of _build_arms_output, split under the complexity cap, and cell_config puts a matrix cell into a config with dataclasses.replace. run_config_from_args is the one place parsed flags become a config. A doc arm with no library id raises LibraryIdError instead of exiting inside the run; both commands print the same message and exit 1. The regex test over getattr(args, ...) goes with the getattr defaults. subjects run still hands matrix-run a Namespace; the next commit changes that. No flag changes; stdout, exit codes and arms.v1 are unchanged. Refs #4 Refs #229 Signed-off-by: Vladislav Perevezentsev Signed-off-by: Nikolay Petrov --- dokimi/docs/architecture.md | 4 +- dokimi/dokimi/commands/arms.py | 232 ++++++-------------------- dokimi/dokimi/eval/arms_run.py | 233 ++++++++++++++++++++++++++- dokimi/scripts/growth_baseline.json | 3 +- dokimi/tests/test_arm_runner.py | 19 ++- dokimi/tests/test_arms_run_config.py | 70 ++++++++ dokimi/tests/test_session_runner.py | 23 ++- dokimi/tests/test_subjects.py | 17 -- 8 files changed, 375 insertions(+), 226 deletions(-) create mode 100644 dokimi/tests/test_arms_run_config.py diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 6dd46dc4f..97d3e30e7 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -152,7 +152,9 @@ where the model decides to call a doc-search or skill tool (`agent:`, The spec grammar — what an `--arms` entry means, where `+` splits a combined spec, and what each arm is *named* — is `treatments/specs.py`. `eval/arm_runner.py` generates and judges answers for every arm and reports -per-arm deltas vs a baseline. `dokimi/plugins.py` adds runtime plugins that +per-arm deltas vs a baseline. `eval/arms_run.py` runs it from an `ArmsRunConfig`, +the one typed run configuration `arms run`, `arms matrix-run` (one per cell) and +`subjects run` build from their flags. `dokimi/plugins.py` adds runtime plugins that modify every arm inside one run (`plugin:caveman`): a plugin changes the system prompt before the model runs or reshapes the answer after it. Entry point: `python cli.py arms run`. Details in diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index f9ee36253..8c5b73cee 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -9,6 +9,7 @@ from collections import Counter import sys from pathlib import Path +from typing import TYPE_CHECKING from dokimi.utils import question_id from dokimi.eval.judge import JUDGE_PROVIDERS @@ -19,13 +20,17 @@ JUDGE_MODEL_FOR_PROVIDER, JUDGE_MODEL_HELP, ) +from dokimi.eval.question_text import EmptyQuestionError + +if TYPE_CHECKING: + from dokimi.eval.arms_run import ArmsRunConfig # Run-configuration fields the arms commands read off parsed args, shared with # any other command that drives an arms run by constructing its own Namespace. # Kept in one place so a new flag reaches every caller: enumerating these at each # call site is what lets a flag silently fall back to its default, since -# _build_arms_output reads them with getattr defaults and cannot tell a caller -# that omitted a field from a user who chose the default. +# run_config_from_args gives a field the Namespace lacks the config's default and +# cannot tell a caller that omitted it from a user who chose the default. FORWARDED_RUN_CONFIG_FIELDS = ( "model", "provider", @@ -50,13 +55,26 @@ ) -from dokimi.eval.question_text import EmptyQuestionError +def run_config_from_args(args: argparse.Namespace, **overrides) -> ArmsRunConfig: + """The run configuration in parsed *args*: the one place flags become an ``ArmsRunConfig``. + + A config field the command has no flag for takes the config's default, the value + ``arms run`` runs with when that flag is left out. Pass *overrides* for fields the + command sets itself. + """ + from dataclasses import fields + + from dokimi.eval.arms_run import ArmsRunConfig + + values = {f.name: getattr(args, f.name) for f in fields(ArmsRunConfig) if hasattr(args, f.name)} + return ArmsRunConfig(**{**values, **overrides}) + def forwarded_run_config(args: argparse.Namespace, **overrides) -> dict: """Copy the shared run-configuration fields off *args*. Fields absent from *args* are omitted rather than defaulted, so - ``_build_arms_output`` applies its own default instead of this helper + ``run_config_from_args`` applies the config's default instead of this helper inventing one. Pass *overrides* for values the caller sets itself. """ forwarded = { @@ -80,13 +98,18 @@ def judge_panel_roles(value: str) -> str: def cmd_arms_run(args: argparse.Namespace) -> None: """Run an N-arm treatment comparison and optionally judge it.""" + from dokimi.eval.arms_run import LibraryIdError, run_arms + try: specs = _parse_arm_specs(args.arms) questions = _load_questions(args.questions) - _apply_named_matrix_cell(args) - output = _build_arms_output(args, specs, questions) + config = _with_named_cell(run_config_from_args(args), args.matrix_cells, args.matrix_cell) + output = run_arms(config, specs, questions) except EmptyQuestionError: raise # a bad input record, not an arm setup error: cli.main reports it (exit 2) + except LibraryIdError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) except (ValueError, OSError, RecursionError) as exc: print(f"Error building arms/plugins: {exc}", file=sys.stderr) sys.exit(1) @@ -107,12 +130,19 @@ def cmd_arms_run(args: argparse.Namespace) -> None: def cmd_arms_matrix_run(args: argparse.Namespace) -> None: """Run every cell in a matrix.cells descriptor and write a rollup.""" - from dokimi.eval.arms_run import validate_supported_matrix_harnesses + from dokimi.eval.arms_run import ( + LibraryIdError, + cell_config, + check_sampling_support, + run_arms, + validate_supported_matrix_harnesses, + ) from dokimi.eval.cells import load_matrix_cells from dokimi.eval.matrix_rollup import build_matrix_rollup, strip_internal_payloads from dokimi.report.matrix_rollup_report import render_matrix_rollup_report from dokimi.report.plugin_delta_report import render_plugin_delta_report + config = run_config_from_args(args) try: specs = _parse_arm_specs(args.arms) questions = _load_questions(args.questions) @@ -120,9 +150,7 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: validate_supported_matrix_harnesses(cells) _validate_unique_cell_filenames(cells) for cell in cells: - cell_args = argparse.Namespace(**vars(args)) - _apply_matrix_cell(cell_args, cell) - _check_sampling_support(cell_args) + check_sampling_support(cell_config(config, cell)) except (ValueError, OSError, RecursionError) as exc: print(f"Error loading matrix run inputs: {exc}", file=sys.stderr) sys.exit(1) @@ -136,15 +164,16 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: sys.exit(1) cell_artifacts = [] for cell in cells: - cell_args = argparse.Namespace(**vars(args)) - _apply_matrix_cell(cell_args, cell) safe_name = _safe_name(cell.name) out_json = cells_dir / f"{safe_name}.json" out_md = cells_dir / f"{safe_name}.md" try: - output = _build_arms_output(cell_args, specs, questions) + output = run_arms(cell_config(config, cell), specs, questions) except EmptyQuestionError: raise # a bad input record, not a cell error: cli.main reports it (exit 2) + except LibraryIdError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) except (ValueError, OSError, RecursionError) as exc: print(f"Error running matrix cell {cell.name!r}: {exc}", file=sys.stderr) sys.exit(1) @@ -472,174 +501,19 @@ def _refuse_duplicate_ids(ids: list, path: str) -> None: raise ValueError(f"duplicate question ids in {path}: {', '.join(map(repr, dupes[:5]))}") -def _apply_named_matrix_cell(args: argparse.Namespace) -> None: - from dokimi.eval.arms_run import validate_supported_matrix_harnesses +def _with_named_cell(config: ArmsRunConfig, matrix_cells: str | None, + matrix_cell: str | None) -> ArmsRunConfig: + """*config* as the cell named by ``--matrix-cell`` in ``--matrix-cells`` runs it, if one is.""" + from dokimi.eval.arms_run import cell_config, validate_supported_matrix_harnesses from dokimi.eval.cells import load_matrix_cells, select_matrix_cell - if args.matrix_cells or args.matrix_cell: - if not args.matrix_cells or not args.matrix_cell: - raise ValueError("--matrix-cells and --matrix-cell must be passed together") - matrix_cell = select_matrix_cell(load_matrix_cells(args.matrix_cells), args.matrix_cell) - validate_supported_matrix_harnesses([matrix_cell]) - _apply_matrix_cell(args, matrix_cell) - - -def _apply_matrix_cell(args: argparse.Namespace, matrix_cell) -> None: - args.matrix_cell = matrix_cell.name - args.model = matrix_cell.model - args.provider = matrix_cell.provider - args.harness = matrix_cell.harness - args.plugins = ",".join(matrix_cell.plugin_specs) - - -def _check_sampling_support(args: argparse.Namespace) -> None: - """Reject sampling flags the answer provider would refuse on every answer.""" - from dokimi.eval.arms_run import probe_sampling - from dokimi.llm import unsupported_sampling_params - - bad = unsupported_sampling_params( - args.model, args.provider, - temperature=getattr(args, "temperature", None), - seed=getattr(args, "seed", None), - ) - if bad: - raise ValueError( - f"Provider '{args.provider}' does not support {', '.join(bad)} for " - f"model '{args.model}'. Drop the flag(s); every answer would " - "otherwise fail and the report would show null scores." - ) - probe_sampling(args.model, args.provider, - getattr(args, "temperature", None), getattr(args, "seed", None)) - - -def _refuse_judged_session(args: argparse.Namespace) -> None: - """Refused before any answer is paid for: a session is measured, not judged.""" - if getattr(args, "session_turns", None) is not None and args.judge: - raise ValueError("--session-turns is measurement-only; drop --judge") - - -def _build_arms_output(args: argparse.Namespace, specs: list[str], questions: list[dict]) -> dict: - from dokimi.eval import Judge, JudgePanel - from dokimi.eval.panel import JUDGE_ROLES, JudgeConfig - from dokimi.eval.arm_runner import ArmRunner - from dokimi.eval.arms_run import probe_sampling, resolve_doc_library_id, validate_expected_skills - from dokimi.plugins import ( - create_plugins, - plugin_set_metadata, - validate_plugins_for_harness, - wrap_treatments, - ) - from dokimi.treatments import create_treatments - - _refuse_judged_session(args) - treatments = create_treatments(specs, top_k=args.top_k, rerank_threshold=args.rerank_threshold) - validate_expected_skills(questions, treatments) - try: - library_id = resolve_doc_library_id(treatments, args.product, args.context7_id) - except ValueError as exc: - print(f"Error: {exc}", file=sys.stderr) - raise SystemExit(1) from exc - plugins = create_plugins([s.strip() for s in args.plugins.split(",") if s.strip()]) - validate_plugins_for_harness(plugins, args.harness) - treatments = wrap_treatments(treatments, plugins) - plugin_set = plugin_set_metadata(plugins) - print(f"Arms: {', '.join(t.name for t in treatments)}") - if args.matrix_cell: - print(f"Matrix cell: {args.matrix_cell}") - if plugin_set["plugins"]: - print(f"Plugins: {plugin_set['plugin_set']} ({plugin_set['plugin_set_id']})") - - _check_sampling_support(args) - if args.judge: - from dokimi.eval.question_text import require_question_text - - # The judge refuses a blank question; here, that is before any answer is paid for. - # Every question is answered and then judged, so every one is checked. - for i, q in enumerate(questions): - require_question_text(q, i) - from dokimi.llm import unsupported_sampling_params - from dokimi.utils import bedrock_litellm_model - - judge_model = args.judge_model - if args.judge_provider == "amazon-bedrock" and not getattr(args, "judge_panel", None): - # A single Bedrock judge goes through get_llm, which maps a bare name first. - judge_model = bedrock_litellm_model(args.judge_model) - bad_judge = unsupported_sampling_params( - judge_model, args.judge_provider, - temperature=getattr(args, "judge_temperature", None), - ) - if bad_judge: - raise ValueError( - f"Judge provider '{args.judge_provider}' does not support " - f"{', '.join(bad_judge)} for model '{args.judge_model}'. Drop the " - "flag; every score would otherwise be null." - ) - probe_sampling(judge_model, args.judge_provider, - getattr(args, "judge_temperature", None), None) - - # Built before any answer is paid for: an unsupported judge provider raises here. - judge: Judge | JudgePanel | None = None - if args.judge: - panel_roles = [r.strip() for r in (getattr(args, "judge_panel", "") or "").split(",") if r.strip()] - if panel_roles: - unknown = [r for r in panel_roles if r not in JUDGE_ROLES] - if unknown: - raise ValueError( - f"Unknown --judge-panel role(s): {', '.join(unknown)}. " - f"Available: {', '.join(JUDGE_ROLES)}" - ) - # A panel reports grader spread and agreement per answer, which a - # single judge structurally cannot — judge choice alone moves scores. - judge = JudgePanel( - judges=[ - JudgeConfig(role=r, model=args.judge_model, - provider=args.judge_provider, - temperature=getattr(args, "judge_temperature", None)) - for r in panel_roles - ], - ) - else: - judge = Judge(model=args.judge_model, provider=args.judge_provider, - temperature=getattr(args, "judge_temperature", None)) - - runner = ArmRunner( - treatments, - model=args.model, - provider=args.provider, - max_iterations=args.max_iterations, - harness=args.harness, - plugin_set=plugin_set, - matrix_cell=args.matrix_cell, - max_tokens=getattr(args, "max_tokens", None), - grounded_from_topics=getattr(args, "grounded_from_topics", False), - stats_correction=getattr(args, "stats_correction", "holm"), - temperature=getattr(args, "temperature", None), - seed=getattr(args, "seed", None), - session_turns=getattr(args, "session_turns", None), - turn_reminder=getattr(args, "turn_reminder", None), - ) - records = runner.run( - library_name=args.product, - questions=questions, - library_id=library_id, - concurrency=args.concurrency, - ) - - evaluations = None - if judge is not None: - if isinstance(judge, JudgePanel): - roles = [j.role for j in judge.judges] - print(f"Judging arms with a {len(roles)}-judge panel: {', '.join(roles)}") - else: - print("Judging arms...") - evaluations = runner.judge( - judge, args.product, records, - baseline_arm=args.baseline_arm, concurrency=args.concurrency, - ) - return runner.build_output( - args.product, records, evaluations=evaluations, baseline_arm=args.baseline_arm, - judge=judge, - ) + if not (matrix_cells or matrix_cell): + return config + if not matrix_cells or not matrix_cell: + raise ValueError("--matrix-cells and --matrix-cell must be passed together") + cell = select_matrix_cell(load_matrix_cells(matrix_cells), matrix_cell) + validate_supported_matrix_harnesses([cell]) + return cell_config(config, cell) def _write_report(path: Path, text: str) -> None: diff --git a/dokimi/dokimi/eval/arms_run.py b/dokimi/dokimi/eval/arms_run.py index 4dfca1359..fa4babaae 100644 --- a/dokimi/dokimi/eval/arms_run.py +++ b/dokimi/dokimi/eval/arms_run.py @@ -1,21 +1,77 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""What an arms run checks and resolves before the first answer is paid for. +"""One arms run from a typed run configuration. -The arm-run commands (``arms run``, ``arms matrix-run``, and ``subjects run`` through -``matrix-run``) and ``eval`` judge-reliability share these: the doc arm's library id, the -``expected_skill`` labels, the harnesses a matrix cell may name, and the sampling probe. +``arms run``, ``arms matrix-run`` and ``subjects run`` each build an :class:`ArmsRunConfig` +once from their parsed flags; a matrix cell is that config with the cell's model, provider, +harness and plugins put in (:func:`cell_config`). :func:`run_arms` checks the config, builds +the arms, the judge and the :class:`~dokimi.eval.arm_runner.ArmRunner`, and returns the +``arms.v1`` output. Paths, saving, the rollup and exit codes stay with the commands. """ from __future__ import annotations import functools +from dataclasses import dataclass, replace -from dokimi.eval.cells import HARNESS_ARMS_RUNNER +from dokimi.defaults import ( + DEFAULT_ANSWER_MODEL, + DEFAULT_ANSWER_PROVIDER, + DEFAULT_JUDGE_MODEL, + DEFAULT_JUDGE_PROVIDER, +) +from dokimi.eval.cells import HARNESS_ARMS_RUNNER, MatrixCellDescriptor from dokimi.utils import question_id +@dataclass(frozen=True) +class ArmsRunConfig: + """Everything an arms run reads besides its arm specs and its questions. + + The field names are the ``dest`` names of the ``arms run`` flags, and a value is in the + flag's form (``plugins`` and ``judge_panel`` stay comma-separated). A default is what + ``arms run`` runs with when the flag is left out, so a command without the flag + (``subjects run`` has no ``--temperature``) runs with it too. + """ + + product: str + model: str = DEFAULT_ANSWER_MODEL + provider: str = DEFAULT_ANSWER_PROVIDER + harness: str = HARNESS_ARMS_RUNNER + plugins: str = "" + #: The matrix cell this run is, stamped into the output; ``None`` outside a matrix. + matrix_cell: str | None = None + context7_id: str | None = None + baseline_arm: str = "baseline" + stats_correction: str = "holm" + judge: bool = False + judge_model: str = DEFAULT_JUDGE_MODEL + judge_provider: str = DEFAULT_JUDGE_PROVIDER + judge_temperature: float | None = None + judge_panel: str = "" + top_k: int = 5 + rerank_threshold: float = 0.3 + max_iterations: int = 6 + max_tokens: int | None = None + grounded_from_topics: bool = False + temperature: float | None = None + seed: int | None = None + session_turns: int | None = None + turn_reminder: str | None = None + concurrency: int = 5 + + +class LibraryIdError(ValueError): + """No library id can be named for a doc arm, so the run stops before any answer.""" + + +def cell_config(config: ArmsRunConfig, cell: MatrixCellDescriptor) -> ArmsRunConfig: + """*config* as matrix cell *cell* runs it: the cell's name, model, provider, harness, plugins.""" + return replace(config, matrix_cell=cell.name, model=cell.model, provider=cell.provider, + harness=cell.harness, plugins=",".join(cell.plugin_specs)) + + def validate_supported_matrix_harnesses(cells) -> None: """Reject matrix harness labels that the in-process runner cannot execute.""" unsupported = [ @@ -105,6 +161,24 @@ def validate_expected_skills(questions: list[dict], treatments) -> None: f"{', '.join(unknown)}") +def check_sampling_support(config: ArmsRunConfig) -> None: + """Reject sampling flags the answer provider would refuse on every answer.""" + from dokimi.llm import unsupported_sampling_params + + bad = unsupported_sampling_params( + config.model, config.provider, + temperature=config.temperature, + seed=config.seed, + ) + if bad: + raise ValueError( + f"Provider '{config.provider}' does not support {', '.join(bad)} for " + f"model '{config.model}'. Drop the flag(s); every answer would " + "otherwise fail and the report would show null scores." + ) + probe_sampling(config.model, config.provider, config.temperature, config.seed) + + @functools.lru_cache(maxsize=None) def probe_sampling(model: str, provider: str, temperature, seed) -> None: """One short real call with the requested sampling, so a value the provider @@ -121,3 +195,152 @@ def probe_sampling(model: str, provider: str, temperature, seed) -> None: f"Probe call to '{provider}/{model}' with temperature={temperature}, " f"seed={seed} failed, so every call in the run would too: {exc}" ) from exc + + +def _refuse_judged_session(config: ArmsRunConfig) -> None: + """Refused before any answer is paid for: a session is measured, not judged.""" + if config.session_turns is not None and config.judge: + raise ValueError("--session-turns is measurement-only; drop --judge") + + +def _treatments(config: ArmsRunConfig, specs: list[str], questions: list[dict]): + """The arms of *specs*, wrapped in the config's plugins: (treatments, library id, plugin set).""" + from dokimi.plugins import ( + create_plugins, + plugin_set_metadata, + validate_plugins_for_harness, + wrap_treatments, + ) + from dokimi.treatments import create_treatments + + treatments = create_treatments(specs, top_k=config.top_k, rerank_threshold=config.rerank_threshold) + validate_expected_skills(questions, treatments) + try: + library_id = resolve_doc_library_id(treatments, config.product, config.context7_id) + except ValueError as exc: + raise LibraryIdError(str(exc)) from exc + plugins = create_plugins([s.strip() for s in config.plugins.split(",") if s.strip()]) + validate_plugins_for_harness(plugins, config.harness) + treatments = wrap_treatments(treatments, plugins) + plugin_set = plugin_set_metadata(plugins) + print(f"Arms: {', '.join(t.name for t in treatments)}") + if config.matrix_cell: + print(f"Matrix cell: {config.matrix_cell}") + if plugin_set["plugins"]: + print(f"Plugins: {plugin_set['plugin_set']} ({plugin_set['plugin_set_id']})") + return treatments, library_id, plugin_set + + +def _check_judge(config: ArmsRunConfig, questions: list[dict]) -> None: + """Refuse what the judge would refuse on every score, before any answer is paid for.""" + from dokimi.eval.question_text import require_question_text + from dokimi.llm import unsupported_sampling_params + from dokimi.utils import bedrock_litellm_model + + # The judge refuses a blank question; here, that is before any answer is paid for. + # Every question is answered and then judged, so every one is checked. + for i, q in enumerate(questions): + require_question_text(q, i) + judge_model = config.judge_model + if config.judge_provider == "amazon-bedrock" and not config.judge_panel: + # A single Bedrock judge goes through get_llm, which maps a bare name first. + judge_model = bedrock_litellm_model(config.judge_model) + bad_judge = unsupported_sampling_params( + judge_model, config.judge_provider, + temperature=config.judge_temperature, + ) + if bad_judge: + raise ValueError( + f"Judge provider '{config.judge_provider}' does not support " + f"{', '.join(bad_judge)} for model '{config.judge_model}'. Drop the " + "flag; every score would otherwise be null." + ) + probe_sampling(judge_model, config.judge_provider, config.judge_temperature, None) + + +def _build_judge(config: ArmsRunConfig): + """The single judge or the panel the config names; an unsupported judge provider raises here.""" + from dokimi.eval import Judge, JudgePanel + from dokimi.eval.panel import JUDGE_ROLES, JudgeConfig + + panel_roles = [r.strip() for r in config.judge_panel.split(",") if r.strip()] + if not panel_roles: + return Judge(model=config.judge_model, provider=config.judge_provider, + temperature=config.judge_temperature) + unknown = [r for r in panel_roles if r not in JUDGE_ROLES] + if unknown: + raise ValueError( + f"Unknown --judge-panel role(s): {', '.join(unknown)}. " + f"Available: {', '.join(JUDGE_ROLES)}" + ) + # A panel reports grader spread and agreement per answer, which a + # single judge structurally cannot — judge choice alone moves scores. + return JudgePanel( + judges=[ + JudgeConfig(role=r, model=config.judge_model, + provider=config.judge_provider, + temperature=config.judge_temperature) + for r in panel_roles + ], + ) + + +def _judge_records(runner, judge, config: ArmsRunConfig, records: list[dict]): + from dokimi.eval import JudgePanel + + if isinstance(judge, JudgePanel): + roles = [j.role for j in judge.judges] + print(f"Judging arms with a {len(roles)}-judge panel: {', '.join(roles)}") + else: + print("Judging arms...") + return runner.judge( + judge, config.product, records, + baseline_arm=config.baseline_arm, concurrency=config.concurrency, + ) + + +def run_arms(config: ArmsRunConfig, specs: list[str], questions: list[dict]) -> dict: + """Answer *questions* in every arm of *specs*, judge them when ``config.judge``, and + return the ``arms.v1`` output. + + Everything that can be refused (the config, the arms, the plugins, the sampling, the + judge) is refused before the first answer is paid for. A doc arm with no library id + raises :class:`LibraryIdError`; anything else refused raises ``ValueError``. + """ + from dokimi.eval.arm_runner import ArmRunner + + _refuse_judged_session(config) + treatments, library_id, plugin_set = _treatments(config, specs, questions) + check_sampling_support(config) + judge = None + if config.judge: + _check_judge(config, questions) + judge = _build_judge(config) + + runner = ArmRunner( + treatments, + model=config.model, + provider=config.provider, + max_iterations=config.max_iterations, + harness=config.harness, + plugin_set=plugin_set, + matrix_cell=config.matrix_cell, + max_tokens=config.max_tokens, + grounded_from_topics=config.grounded_from_topics, + stats_correction=config.stats_correction, + temperature=config.temperature, + seed=config.seed, + session_turns=config.session_turns, + turn_reminder=config.turn_reminder, + ) + records = runner.run( + library_name=config.product, + questions=questions, + library_id=library_id, + concurrency=config.concurrency, + ) + evaluations = None if judge is None else _judge_records(runner, judge, config, records) + return runner.build_output( + config.product, records, evaluations=evaluations, baseline_arm=config.baseline_arm, + judge=judge, + ) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index b8a876b70..644da66a2 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -12,11 +12,10 @@ "tests/test_judge_validity.py": {"lines": 2235, "rationale": "grew on main in #73 (judge-validity --context stored), 1423 -> 2235, before the ratchet landed; split it in its own PR"}, "tests/test_pipeline_question_set.py": {"lines": 1223}, "tests/test_repo_environment.py": {"lines": 1305, "rationale": "grew on main in #147 before the ratchet landed; +2 for the instructions assert of two crash tests (#103 review); split it in its own PR"}, - "tests/test_subjects.py": {"lines": 1064}, + "tests/test_subjects.py": {"lines": 1047}, "tests/test_task_formats.py": {"lines": 1003} }, "functions": { - "dokimi/commands/arms.py::_build_arms_output": {"complexity": 13}, "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 21}, "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 15}, diff --git a/dokimi/tests/test_arm_runner.py b/dokimi/tests/test_arm_runner.py index 5babbb31c..a46992e66 100644 --- a/dokimi/tests/test_arm_runner.py +++ b/dokimi/tests/test_arm_runner.py @@ -485,22 +485,21 @@ def test_build_output_records_judge_temperature_and_prompt_digest(monkeypatch): def test_an_unbuildable_judge_fails_before_any_answer_is_generated(monkeypatch): - from dokimi.commands.arms import _build_arms_output + from dokimi.eval.arms_run import ArmsRunConfig, run_arms calls = [] monkeypatch.setattr(ArmRunner, "run", lambda self, *a, **kw: calls.append(1) or []) - args = _parse_arms_run(["--judge", "--judge-model", "m"]) # The CLI no longer offers a provider Judge rejects, so set one the way a - # caller building its own Namespace could. - args.judge_provider = "openai-codex" + # caller building its own config could. + config = ArmsRunConfig(product="oneTBB", judge=True, judge_model="m", judge_provider="openai-codex") with pytest.raises(ValueError, match="openai-codex"): - _build_arms_output(args, ["baseline"], QUESTIONS[:1]) + run_arms(config, ["baseline"], QUESTIONS[:1]) assert calls == [] def test_matrix_run_checks_every_cell_before_running_the_first(monkeypatch, tmp_path): import json - import dokimi.commands.arms as arms_mod + import dokimi.eval.arms_run as arms_run from dokimi.cli import build_parser cells = tmp_path / "cells.json" @@ -513,9 +512,9 @@ def test_matrix_run_checks_every_cell_before_running_the_first(monkeypatch, tmp_ monkeypatch.setattr("dokimi.llm.unsupported_sampling_params", lambda model, provider, **kw: ["seed"] if model == "m-bad" else []) - monkeypatch.setattr("dokimi.eval.arms_run.probe_sampling", lambda *a: None) + monkeypatch.setattr(arms_run, "probe_sampling", lambda *a: None) ran = [] - monkeypatch.setattr(arms_mod, "_build_arms_output", lambda *a, **kw: ran.append(1)) + monkeypatch.setattr(arms_run, "run_arms", lambda *a, **kw: ran.append(1)) args = build_parser().parse_args( ["arms", "matrix-run", "--product", "oneTBB", "--questions", str(questions), "--arms", "baseline", "--matrix-cells", str(cells), "--seed", "1", @@ -948,7 +947,7 @@ def _litellm_resp(text): def test_a_bare_bedrock_judge_name_is_checked_as_the_judge_will_call_it(monkeypatch): import litellm import dokimi.eval.arms_run as arms_run - from dokimi.commands.arms import _build_arms_output + from dokimi.commands.arms import run_config_from_args from dokimi.utils import get_llm mapped = "us.anthropic.claude-haiku-4-5-20251001-v1:0" @@ -971,7 +970,7 @@ def completion(model, messages, **kw): arms_run.probe_sampling.cache_clear() args = _parse_arms_run(["--judge", "--judge-provider", "amazon-bedrock", "--judge-model", "claude-sonnet-4-5", "--judge-temperature", "0"]) - _build_arms_output(args, ["baseline"], QUESTIONS[:1]) + arms_run.run_arms(run_config_from_args(args), ["baseline"], QUESTIONS[:1]) get_llm("amazon-bedrock", "claude-sonnet-4-5", temperature=0.0).invoke("p") assert [m for m in checked if m.startswith("bedrock/")] == [f"bedrock/{mapped}"] diff --git a/dokimi/tests/test_arms_run_config.py b/dokimi/tests/test_arms_run_config.py new file mode 100644 index 000000000..f823df0ca --- /dev/null +++ b/dokimi/tests/test_arms_run_config.py @@ -0,0 +1,70 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``ArmsRunConfig``: its defaults are the ``arms run`` flags' defaults, and a run it cannot +name a library id for stops with the same message from every entry point. + +What each entry point hands the runner is pinned in ``test_arms_run_config_parity.py``. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import parse_args +from dokimi.commands.arms import run_config_from_args +from dokimi.eval.arm_runner import ArmRunner +from dokimi.eval.arms_run import ArmsRunConfig + + +def test_the_config_defaults_are_what_arms_run_runs_without_a_flag(): + # A command without some flag (subjects run has no --temperature) and a caller building + # the config itself both run with these defaults, so they must not drift from the flags'. + args = parse_args(["arms", "run", "--product", "demo", "--questions", "q.json", + "--arms", "baseline"]) + + assert run_config_from_args(args) == ArmsRunConfig(product="demo") + + +@pytest.fixture +def unresolvable(tmp_path: Path, monkeypatch) -> dict[str, Path]: + """A doc arm on a product no library id can be named for, and a guard on answering.""" + def refuse(product, client, registry=None): + raise ValueError(f"cannot name a library id for {product!r}") + + def answer(self, *a, **kw): + raise AssertionError("no answer may be paid for") + + monkeypatch.setattr("dokimi.mcp.factory.resolve_library_id_for_product", refuse) + monkeypatch.setattr(ArmRunner, "run", answer) + docs = tmp_path / "docs.md" + docs.write_text("# Demo\n", encoding="utf-8") + questions = tmp_path / "q.json" + questions.write_text(json.dumps([{"id": "q1", "question": "How?"}]), encoding="utf-8") + cells = tmp_path / "cells.json" + cells.write_text(json.dumps({"matrix": {"cells": [ + {"id": "only", "model": "m", "provider": "openai", "harness": "arms-runner"}, + ]}}), encoding="utf-8") + return {"docs": docs, "questions": questions, "cells": cells, "out": tmp_path / "out"} + + +@pytest.mark.parametrize("command", [ + ["arms", "run", "--out-json", "{out}/a.json", "--out-md", "{out}/a.md"], + ["arms", "matrix-run", "--matrix-cells", "{cells}", "--out-dir", "{out}"], +]) +def test_a_doc_arm_without_a_library_id_stops_the_run_before_any_answer(command, unresolvable, + capsys): + argv = [a.format(**unresolvable) for a in command] + args = parse_args([*argv, "--product", "demo", "--questions", str(unresolvable["questions"]), + "--arms", f"baseline,docs:local:{unresolvable['docs']}"]) + + with pytest.raises(SystemExit) as exc: + args.func(args) + + assert exc.value.code == 1 + # Not under "Error building arms/plugins" or "Error running matrix cell": the id is the + # product's, not an arm's or a cell's. + assert capsys.readouterr().err == "Error: cannot name a library id for 'demo'\n" diff --git a/dokimi/tests/test_session_runner.py b/dokimi/tests/test_session_runner.py index ad45078fd..b30b2332d 100644 --- a/dokimi/tests/test_session_runner.py +++ b/dokimi/tests/test_session_runner.py @@ -6,7 +6,6 @@ Every test runs against the ``fake_llm`` adapter, so nothing reaches a provider. """ -from argparse import Namespace from unittest.mock import MagicMock import pytest @@ -287,11 +286,11 @@ def test_plain_runs_are_unchanged_by_the_session_fields(fake_llm): def test_the_cli_refuses_judge_with_session_before_any_answer(fake_llm): - from dokimi.commands.arms import _build_arms_output + from dokimi.eval.arms_run import ArmsRunConfig, run_arms - args = Namespace(judge=True, session_turns=2, turn_reminder=None) + config = ArmsRunConfig(product="oneTBB", judge=True, session_turns=2) with pytest.raises(ValueError, match="measurement-only"): - _build_arms_output(args, ["baseline"], [SESSION_Q]) + run_arms(config, ["baseline"], [SESSION_Q]) assert fake_llm.requests == [] @@ -374,14 +373,14 @@ def _parse_arms_run(argv): def test_arms_run_wires_the_session_flags_into_the_run(fake_llm, monkeypatch): - import dokimi.commands.arms as arms_mod - from dokimi.commands.arms import _build_arms_output + import dokimi.eval.arms_run as arms_run + from dokimi.commands.arms import run_config_from_args - monkeypatch.setattr(arms_mod, "_check_sampling_support", lambda args: None) + monkeypatch.setattr(arms_run, "check_sampling_support", lambda config: None) fake_llm.push("a1", "a2", "r1", "r2") args = _parse_arms_run(["--session-turns", "2", "--turn-reminder", "BE TERSE"]) - out = _build_arms_output(args, ["baseline"], [SESSION_Q]) + out = arms_run.run_arms(run_config_from_args(args), ["baseline"], [SESSION_Q]) assert out["session"] == {"turns": 2, "turn_reminder": "BE TERSE"} assert out["arms"] == ["baseline", "baseline+reminder"] @@ -390,12 +389,12 @@ def test_arms_run_wires_the_session_flags_into_the_run(fake_llm, monkeypatch): def test_arms_run_without_session_flags_runs_single_turn(fake_llm, monkeypatch): - import dokimi.commands.arms as arms_mod - from dokimi.commands.arms import _build_arms_output + import dokimi.eval.arms_run as arms_run + from dokimi.commands.arms import run_config_from_args - monkeypatch.setattr(arms_mod, "_check_sampling_support", lambda args: None) + monkeypatch.setattr(arms_run, "check_sampling_support", lambda config: None) fake_llm.push("a1") - out = _build_arms_output(_parse_arms_run([]), ["baseline"], [SESSION_Q]) + out = arms_run.run_arms(run_config_from_args(_parse_arms_run([])), ["baseline"], [SESSION_Q]) assert "session" not in out and "session_turns" not in out["run_config"] diff --git a/dokimi/tests/test_subjects.py b/dokimi/tests/test_subjects.py index 8732cbb0a..fa618777f 100644 --- a/dokimi/tests/test_subjects.py +++ b/dokimi/tests/test_subjects.py @@ -999,23 +999,6 @@ def fake_call(prompt, model, provider="openai", api_key=None, system=None, **kw) assert cell["run_config"]["grounded_from_topics"] is True -def test_forwarded_run_config_covers_fields_read_with_a_default(): - """The forwarded set must stay a superset of what _build_arms_output consults. - - Anything read there with a getattr default is silently absent when a caller - building its own Namespace omits it, which is how a flag stops propagating. - """ - import inspect - import re - - from dokimi.commands import arms as arms_mod - - source = inspect.getsource(arms_mod._build_arms_output) - read_with_default = set(re.findall(r'getattr\(args,\s*"([a-z_0-9]+)"', source)) - missing = read_with_default - set(arms_mod.FORWARDED_RUN_CONFIG_FIELDS) - assert not missing, f"read with a getattr default but never forwarded: {sorted(missing)}" - - def test_subjects_list_without_a_root_names_the_flag(): """Descriptors live in a content checkout, so there is no in-repo default root.""" parser = build_parser() From 8a7971344be869f72cf2d46d6d69fd2524acbfbb Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 28 Oct 2026 09:05:21 -0700 Subject: [PATCH 03/12] refactor(subjects): build the arms run config directly in subjects run subjects run built a Namespace by hand from FORWARDED_RUN_CONFIG_FIELDS and called cmd_arms_matrix_run with it. Split the matrix run out of that handler into run_matrix(config, ...), which takes the config and the input and output paths, and have subjects run call it with run_config_from_args(args, product=..., baseline_arm=...). A field subjects run has no flag for takes the config's default, as before. FORWARDED_RUN_CONFIG_FIELDS and forwarded_run_config have no caller left and are removed. Refs #4 Refs #229 Signed-off-by: Vladislav Perevezentsev Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/arms.py | 89 +++++++++---------------- dokimi/dokimi/commands/subjects.py | 21 ++---- dokimi/tests/test_subjects.py | 20 +++--- dokimi/tests/test_subjects_run_input.py | 4 +- 4 files changed, 47 insertions(+), 87 deletions(-) diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 8c5b73cee..12ad18c06 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -25,35 +25,6 @@ if TYPE_CHECKING: from dokimi.eval.arms_run import ArmsRunConfig -# Run-configuration fields the arms commands read off parsed args, shared with -# any other command that drives an arms run by constructing its own Namespace. -# Kept in one place so a new flag reaches every caller: enumerating these at each -# call site is what lets a flag silently fall back to its default, since -# run_config_from_args gives a field the Namespace lacks the config's default and -# cannot tell a caller that omitted it from a user who chose the default. -FORWARDED_RUN_CONFIG_FIELDS = ( - "model", - "provider", - "harness", - "context7_id", - "judge", - "judge_model", - "judge_provider", - "top_k", - "rerank_threshold", - "max_iterations", - "max_tokens", - "grounded_from_topics", - "temperature", - "seed", - "judge_temperature", - "judge_panel", - "concurrency", - "stats_correction", - "session_turns", - "turn_reminder", -) - def run_config_from_args(args: argparse.Namespace, **overrides) -> ArmsRunConfig: """The run configuration in parsed *args*: the one place flags become an ``ArmsRunConfig``. @@ -70,22 +41,6 @@ def run_config_from_args(args: argparse.Namespace, **overrides) -> ArmsRunConfig return ArmsRunConfig(**{**values, **overrides}) -def forwarded_run_config(args: argparse.Namespace, **overrides) -> dict: - """Copy the shared run-configuration fields off *args*. - - Fields absent from *args* are omitted rather than defaulted, so - ``run_config_from_args`` applies the config's default instead of this helper - inventing one. Pass *overrides* for values the caller sets itself. - """ - forwarded = { - field: getattr(args, field) - for field in FORWARDED_RUN_CONFIG_FIELDS - if hasattr(args, field) - } - forwarded.update(overrides) - return forwarded - - def judge_panel_roles(value: str) -> str: """argparse type for --judge-panel: an unknown role fails before any answer is paid for.""" from dokimi.eval.panel import JUDGE_ROLES @@ -130,6 +85,24 @@ def cmd_arms_run(args: argparse.Namespace) -> None: def cmd_arms_matrix_run(args: argparse.Namespace) -> None: """Run every cell in a matrix.cells descriptor and write a rollup.""" + out_dir = Path(args.out_dir) if args.out_dir else Path(f"results/arms/{args.product}-matrix") + run_matrix( + run_config_from_args(args), arms=args.arms, questions=args.questions, + matrix_cells=args.matrix_cells, out_dir=out_dir, + out_json=Path(args.out_json) if args.out_json else None, + out_md=Path(args.out_md) if args.out_md else None, + plugin_deltas=args.plugin_deltas, + ) + + +def run_matrix(config: ArmsRunConfig, *, arms: str, questions: str, matrix_cells: str, + out_dir: Path, out_json: Path | None = None, out_md: Path | None = None, + plugin_deltas: bool = True) -> None: + """Run *config* in every cell of *matrix_cells* and write the cells, plugin deltas and rollup. + + Every cell is checked before the first one runs. *out_json* defaults to + ``matrix-rollup.json`` in *out_dir*, *out_md* to *out_json* with a ``.md`` suffix. + """ from dokimi.eval.arms_run import ( LibraryIdError, cell_config, @@ -142,11 +115,10 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: from dokimi.report.matrix_rollup_report import render_matrix_rollup_report from dokimi.report.plugin_delta_report import render_plugin_delta_report - config = run_config_from_args(args) try: - specs = _parse_arm_specs(args.arms) - questions = _load_questions(args.questions) - cells = load_matrix_cells(args.matrix_cells) + specs = _parse_arm_specs(arms) + loaded_questions = _load_questions(questions) + cells = load_matrix_cells(matrix_cells) validate_supported_matrix_harnesses(cells) _validate_unique_cell_filenames(cells) for cell in cells: @@ -155,7 +127,6 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: print(f"Error loading matrix run inputs: {exc}", file=sys.stderr) sys.exit(1) - out_dir = Path(args.out_dir) if args.out_dir else Path(f"results/arms/{args.product}-matrix") cells_dir = out_dir / "cells" try: cells_dir.mkdir(parents=True, exist_ok=True) @@ -165,10 +136,10 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: cell_artifacts = [] for cell in cells: safe_name = _safe_name(cell.name) - out_json = cells_dir / f"{safe_name}.json" - out_md = cells_dir / f"{safe_name}.md" + cell_json = cells_dir / f"{safe_name}.json" + cell_md = cells_dir / f"{safe_name}.md" try: - output = run_arms(cell_config(config, cell), specs, questions) + output = run_arms(cell_config(config, cell), specs, loaded_questions) except EmptyQuestionError: raise # a bad input record, not a cell error: cli.main reports it (exit 2) except LibraryIdError as exc: @@ -177,14 +148,14 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: except (ValueError, OSError, RecursionError) as exc: print(f"Error running matrix cell {cell.name!r}: {exc}", file=sys.stderr) sys.exit(1) - _save_arms_output(output, out_json, out_md) - cell_artifacts.append((cell.name, out_json, output)) + _save_arms_output(output, cell_json, cell_md) + cell_artifacts.append((cell.name, cell_json, output)) rollup = build_matrix_rollup( - library_name=args.product, + library_name=config.product, cell_artifacts=cell_artifacts, out_dir=out_dir, - compute_plugin_deltas=args.plugin_deltas, + compute_plugin_deltas=plugin_deltas, ) for row in rollup.get("plugin_deltas", []): payload = row.get("_artifact_payload") @@ -196,8 +167,8 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: print(f"OK Saved plugin delta: {delta_json}") persisted = strip_internal_payloads(rollup) - out_json = Path(args.out_json) if args.out_json else out_dir / "matrix-rollup.json" - out_md = Path(args.out_md) if args.out_md else out_json.with_suffix(".md") + out_json = out_json or out_dir / "matrix-rollup.json" + out_md = out_md or out_json.with_suffix(".md") _save_artifact_or_exit("matrix_rollup", persisted, out_json) _write_report(out_md, render_matrix_rollup_report(persisted)) print(f"OK Saved matrix rollup: {out_json}") diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py index e60b662ae..6f5420561 100644 --- a/dokimi/dokimi/commands/subjects.py +++ b/dokimi/dokimi/commands/subjects.py @@ -88,11 +88,7 @@ def cmd_subjects_show(args: argparse.Namespace) -> None: def cmd_subjects_run(args: argparse.Namespace) -> None: """Run awareness matrix cells for one subject and write a scorecard.""" from dokimi.artifacts import load_artifact, save_artifact - from dokimi.commands.arms import ( - _safe_name, - cmd_arms_matrix_run, - forwarded_run_config, - ) + from dokimi.commands.arms import _safe_name, run_config_from_args, run_matrix from dokimi.report.subject_report import render_subject_scorecard from dokimi.subjects import build_subject_scorecard, load_subject from dokimi.subjects.loader import awareness_arm_specs @@ -137,19 +133,12 @@ def cmd_subjects_run(args: argparse.Namespace) -> None: run_dir = out_dir / _safe_name(product) rollup_json = run_dir / "matrix-rollup.json" rollup_md = run_dir / "matrix-rollup.md" - run_args = argparse.Namespace( - product=product, - questions=questions_path, - arms=",".join(awareness_arm_specs(descriptor)), - matrix_cells=str(matrix_path), - matrix_cell=None, - out_dir=str(run_dir), - out_json=str(rollup_json), - out_md=str(rollup_md), + run_matrix( + run_config_from_args(args, product=product, baseline_arm=baseline_arm), + arms=",".join(awareness_arm_specs(descriptor)), questions=questions_path, + matrix_cells=str(matrix_path), out_dir=run_dir, out_json=rollup_json, out_md=rollup_md, plugin_deltas=not args.no_plugin_deltas, - **forwarded_run_config(args, plugins="", baseline_arm=baseline_arm), ) - cmd_arms_matrix_run(run_args) awareness_runs.append({ "product": product, "questions": questions_path, diff --git a/dokimi/tests/test_subjects.py b/dokimi/tests/test_subjects.py index fa618777f..8e31f6992 100644 --- a/dokimi/tests/test_subjects.py +++ b/dokimi/tests/test_subjects.py @@ -640,18 +640,18 @@ def test_subjects_reject_duplicate_runtime_arm_names(tmp_path: Path): def test_subjects_default_out_dir_sanitizes_subject_id(tmp_path: Path, monkeypatch): - def fake_matrix_run(args): - run_dir = Path(args.out_dir) + def fake_matrix_run(config, *, out_dir, **kw): + run_dir = Path(out_dir) run_dir.mkdir(parents=True) (run_dir / "matrix-rollup.json").write_text(json.dumps({ "schema_version": "matrix_rollup.v1", - "library_name": args.product, + "library_name": config.product, "cells": [], "plugin_deltas": [], "warnings": [], }), encoding="utf-8") - monkeypatch.setattr("dokimi.commands.arms.cmd_arms_matrix_run", fake_matrix_run) + monkeypatch.setattr("dokimi.commands.arms.run_matrix", fake_matrix_run) descriptor_path = tmp_path / "subject.toml" questions = tmp_path / "questions.json" questions.write_text(json.dumps([{"id": "q1", "question": "Q?"}]), encoding="utf-8") @@ -836,10 +836,10 @@ def test_subject_work_resolves_bare_task_names_against_tasks_root(tmp_path: Path def test_subject_work_preflight_loads_tasks_before_awareness(tmp_path: Path, monkeypatch, capsys): - def fail_if_called(args): + def fail_if_called(config, **kw): raise AssertionError("awareness matrix should not run before task preflight") - monkeypatch.setattr("dokimi.commands.arms.cmd_arms_matrix_run", fail_if_called) + monkeypatch.setattr("dokimi.commands.arms.run_matrix", fail_if_called) questions = tmp_path / "questions.json" questions.write_text(json.dumps([{"id": "q1", "question": "Q?"}]), encoding="utf-8") docs = tmp_path / "docs.md" @@ -876,18 +876,18 @@ def fail_if_called(args): def test_subjects_run_writes_work_task_runs_when_requested(tmp_path: Path, monkeypatch): - def fake_matrix_run(args): - run_dir = Path(args.out_dir) + def fake_matrix_run(config, *, out_dir, **kw): + run_dir = Path(out_dir) run_dir.mkdir(parents=True) (run_dir / "matrix-rollup.json").write_text(json.dumps({ "schema_version": "matrix_rollup.v1", - "library_name": args.product, + "library_name": config.product, "cells": [], "plugin_deltas": [], "warnings": [], }), encoding="utf-8") - monkeypatch.setattr("dokimi.commands.arms.cmd_arms_matrix_run", fake_matrix_run) + monkeypatch.setattr("dokimi.commands.arms.run_matrix", fake_matrix_run) task_dir = tmp_path / "task-one" (task_dir / "environment").mkdir(parents=True) (task_dir / "tests").mkdir() diff --git a/dokimi/tests/test_subjects_run_input.py b/dokimi/tests/test_subjects_run_input.py index 90b3c0b8f..f252eb901 100644 --- a/dokimi/tests/test_subjects_run_input.py +++ b/dokimi/tests/test_subjects_run_input.py @@ -98,7 +98,7 @@ def test_an_unwritable_scorecard_exits_1_naming_it(tmp_path, monkeypatch, capsys import dokimi.commands.arms import dokimi.subjects - monkeypatch.setattr(dokimi.commands.arms, "cmd_arms_matrix_run", lambda args: None) + monkeypatch.setattr(dokimi.commands.arms, "run_matrix", lambda config, **kw: None) monkeypatch.setattr(dokimi.artifacts, "load_artifact", lambda kind, path: {}) monkeypatch.setattr(dokimi.subjects, "build_subject_scorecard", lambda *a, **k: {}) monkeypatch.setattr("dokimi.commands.subjects._run_subject_work", lambda *a, **k: {}) @@ -118,7 +118,7 @@ def _stub_run(monkeypatch, saved): import dokimi.report.subject_report import dokimi.subjects - monkeypatch.setattr(dokimi.commands.arms, "cmd_arms_matrix_run", lambda args: None) + monkeypatch.setattr(dokimi.commands.arms, "run_matrix", lambda config, **kw: None) monkeypatch.setattr(dokimi.artifacts, "load_artifact", lambda kind, path: {}) monkeypatch.setattr(dokimi.artifacts, "save_artifact", lambda kind, data, path: saved.append(path)) monkeypatch.setattr(dokimi.subjects, "build_subject_scorecard", lambda *a, **k: {}) From dfb402367bbf656a74c980ec8186f807b0026966 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Wed, 28 Oct 2026 09:05:37 -0700 Subject: [PATCH 04/12] refactor(arms): run the matrix cells in dokimi/eval/arms_run.py The loop over the cells still lived in the command. Move it into the run module: check_matrix checks every cell's harness and sampling, and run_matrix calls it before the first cell is paid for, then runs the cells in turn and hands each finished cell and its arms.v1 output to a callback, so a later cell's failure keeps the earlier ones. A failing cell raises MatrixCellError naming it; a bad question record and LibraryIdError pass through. The command keeps the files: it checks the matrix before it makes the output dir, makes the dirs before the first paid call, saves each cell from the callback, and writes the plugin deltas and the rollup. The messages and exit codes are unchanged; the cell-name collision check now runs before the harness check, ahead of the paid sampling probe. Refs #4 Signed-off-by: Vladislav Perevezentsev Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/arms.py | 46 ++++++++++++---------------- dokimi/dokimi/eval/arms_run.py | 44 ++++++++++++++++++++++++-- dokimi/tests/test_arms_run_config.py | 42 +++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 28 deletions(-) diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 12ad18c06..b61ea4c99 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -98,18 +98,13 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: def run_matrix(config: ArmsRunConfig, *, arms: str, questions: str, matrix_cells: str, out_dir: Path, out_json: Path | None = None, out_md: Path | None = None, plugin_deltas: bool = True) -> None: - """Run *config* in every cell of *matrix_cells* and write the cells, plugin deltas and rollup. + """``arms matrix-run`` from a config: run every cell of *matrix_cells* and write the cells, + the plugin deltas and the rollup. - Every cell is checked before the first one runs. *out_json* defaults to - ``matrix-rollup.json`` in *out_dir*, *out_md* to *out_json* with a ``.md`` suffix. + *out_json* defaults to ``matrix-rollup.json`` in *out_dir*, *out_md* to *out_json* with a + ``.md`` suffix. """ - from dokimi.eval.arms_run import ( - LibraryIdError, - cell_config, - check_sampling_support, - run_arms, - validate_supported_matrix_harnesses, - ) + from dokimi.eval import arms_run from dokimi.eval.cells import load_matrix_cells from dokimi.eval.matrix_rollup import build_matrix_rollup, strip_internal_payloads from dokimi.report.matrix_rollup_report import render_matrix_rollup_report @@ -119,10 +114,8 @@ def run_matrix(config: ArmsRunConfig, *, arms: str, questions: str, matrix_cells specs = _parse_arm_specs(arms) loaded_questions = _load_questions(questions) cells = load_matrix_cells(matrix_cells) - validate_supported_matrix_harnesses(cells) _validate_unique_cell_filenames(cells) - for cell in cells: - check_sampling_support(cell_config(config, cell)) + arms_run.check_matrix(config, cells) except (ValueError, OSError, RecursionError) as exc: print(f"Error loading matrix run inputs: {exc}", file=sys.stderr) sys.exit(1) @@ -134,23 +127,24 @@ def run_matrix(config: ArmsRunConfig, *, arms: str, questions: str, matrix_cells print(f"Error: cannot write {cells_dir}: {exc}", file=sys.stderr) sys.exit(1) cell_artifacts = [] - for cell in cells: + + def save_cell(cell, output: dict) -> None: safe_name = _safe_name(cell.name) cell_json = cells_dir / f"{safe_name}.json" - cell_md = cells_dir / f"{safe_name}.md" - try: - output = run_arms(cell_config(config, cell), specs, loaded_questions) - except EmptyQuestionError: - raise # a bad input record, not a cell error: cli.main reports it (exit 2) - except LibraryIdError as exc: - print(f"Error: {exc}", file=sys.stderr) - sys.exit(1) - except (ValueError, OSError, RecursionError) as exc: - print(f"Error running matrix cell {cell.name!r}: {exc}", file=sys.stderr) - sys.exit(1) - _save_arms_output(output, cell_json, cell_md) + _save_arms_output(output, cell_json, cells_dir / f"{safe_name}.md") cell_artifacts.append((cell.name, cell_json, output)) + try: + arms_run.run_matrix(config, cells, specs, loaded_questions, on_cell=save_cell) + except EmptyQuestionError: + raise # a bad input record, not a cell error: cli.main reports it (exit 2) + except arms_run.LibraryIdError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + except arms_run.MatrixCellError as exc: + print(f"Error running matrix cell {exc.cell!r}: {exc}", file=sys.stderr) + sys.exit(1) + rollup = build_matrix_rollup( library_name=config.product, cell_artifacts=cell_artifacts, diff --git a/dokimi/dokimi/eval/arms_run.py b/dokimi/dokimi/eval/arms_run.py index fa4babaae..325ee317c 100644 --- a/dokimi/dokimi/eval/arms_run.py +++ b/dokimi/dokimi/eval/arms_run.py @@ -1,18 +1,20 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""One arms run from a typed run configuration. +"""Arms runs, one or a matrix of them, from a typed run configuration. ``arms run``, ``arms matrix-run`` and ``subjects run`` each build an :class:`ArmsRunConfig` once from their parsed flags; a matrix cell is that config with the cell's model, provider, harness and plugins put in (:func:`cell_config`). :func:`run_arms` checks the config, builds the arms, the judge and the :class:`~dokimi.eval.arm_runner.ArmRunner`, and returns the -``arms.v1`` output. Paths, saving, the rollup and exit codes stay with the commands. +``arms.v1`` output; :func:`run_matrix` does that for every cell, after checking them all. +Paths, saving, the rollup and exit codes stay with the commands. """ from __future__ import annotations import functools +from collections.abc import Callable from dataclasses import dataclass, replace from dokimi.defaults import ( @@ -22,6 +24,7 @@ DEFAULT_JUDGE_PROVIDER, ) from dokimi.eval.cells import HARNESS_ARMS_RUNNER, MatrixCellDescriptor +from dokimi.eval.question_text import EmptyQuestionError from dokimi.utils import question_id @@ -66,6 +69,14 @@ class LibraryIdError(ValueError): """No library id can be named for a doc arm, so the run stops before any answer.""" +class MatrixCellError(ValueError): + """A matrix cell refused or failed its run: the reason's message, and the cell's name.""" + + def __init__(self, cell: str, reason: Exception) -> None: + super().__init__(str(reason)) + self.cell = cell + + def cell_config(config: ArmsRunConfig, cell: MatrixCellDescriptor) -> ArmsRunConfig: """*config* as matrix cell *cell* runs it: the cell's name, model, provider, harness, plugins.""" return replace(config, matrix_cell=cell.name, model=cell.model, provider=cell.provider, @@ -344,3 +355,32 @@ def run_arms(config: ArmsRunConfig, specs: list[str], questions: list[dict]) -> config.product, records, evaluations=evaluations, baseline_arm=config.baseline_arm, judge=judge, ) + + +def check_matrix(config: ArmsRunConfig, cells: list[MatrixCellDescriptor]) -> None: + """Refuse a matrix with a cell the in-process runner cannot run, or whose provider would + refuse the sampling on every answer. Raises ``ValueError`` naming the first such cell.""" + validate_supported_matrix_harnesses(cells) + for cell in cells: + check_sampling_support(cell_config(config, cell)) + + +def run_matrix(config: ArmsRunConfig, cells: list[MatrixCellDescriptor], specs: list[str], + questions: list[dict], + on_cell: Callable[[MatrixCellDescriptor, dict], None]) -> None: + """Run *config* in every cell in turn, handing each cell and its ``arms.v1`` output to + *on_cell* as soon as the cell finishes, so a later cell's failure keeps the earlier ones. + + Every cell is checked (:func:`check_matrix`) before the first one is paid for. A cell + that fails raises :class:`MatrixCellError` naming it; a bad question record + (``EmptyQuestionError``) and :class:`LibraryIdError` are not a cell's and pass through. + """ + check_matrix(config, cells) + for cell in cells: + try: + output = run_arms(cell_config(config, cell), specs, questions) + except (EmptyQuestionError, LibraryIdError): + raise + except (ValueError, OSError, RecursionError) as exc: + raise MatrixCellError(cell.name, exc) from exc + on_cell(cell, output) diff --git a/dokimi/tests/test_arms_run_config.py b/dokimi/tests/test_arms_run_config.py index f823df0ca..41628b3a7 100644 --- a/dokimi/tests/test_arms_run_config.py +++ b/dokimi/tests/test_arms_run_config.py @@ -68,3 +68,45 @@ def test_a_doc_arm_without_a_library_id_stops_the_run_before_any_answer(command, # Not under "Error building arms/plugins" or "Error running matrix cell": the id is the # product's, not an arm's or a cell's. assert capsys.readouterr().err == "Error: cannot name a library id for 'demo'\n" + + +def test_a_failing_matrix_cell_is_named_and_the_cells_before_it_are_kept(tmp_path, provider_calls, + capsys): + provider_calls.fake.respond = lambda request: "Use parallel_for." + questions = tmp_path / "q.json" + questions.write_text(json.dumps([{"id": "q1", "question": "How?"}]), encoding="utf-8") + cells = tmp_path / "cells.json" + cells.write_text(json.dumps({"matrix": {"cells": [ + {"id": "ok", "model": "m", "provider": "openai", "harness": "arms-runner"}, + {"id": "bad", "model": "m", "provider": "openai", "harness": "arms-runner", + "plugins": ["plugin:no-such-plugin"]}, + ]}}), encoding="utf-8") + out = tmp_path / "out" + args = parse_args(["arms", "matrix-run", "--product", "demo", "--questions", str(questions), + "--arms", "baseline", "--matrix-cells", str(cells), "--out-dir", str(out)]) + + with pytest.raises(SystemExit) as exc: + args.func(args) + + assert exc.value.code == 1 + assert capsys.readouterr().err.startswith( + "Error running matrix cell 'bad': Unknown plugin spec: 'plugin:no-such-plugin'") + assert sorted(p.name for p in (out / "cells").iterdir()) == ["ok.json", "ok.md"] + + +def test_run_matrix_checks_every_cell_before_the_first_is_paid_for(monkeypatch, provider_calls): + from dokimi.eval.arms_run import run_matrix + from dokimi.eval.cells import MatrixCellDescriptor + + monkeypatch.setattr("dokimi.llm.unsupported_sampling_params", + lambda model, provider, **kw: ["seed"] if model == "m-bad" else []) + monkeypatch.setattr("dokimi.eval.arms_run.probe_sampling", lambda *a: None) + cells = [MatrixCellDescriptor("ok", "m-ok", "openai", "arms-runner"), + MatrixCellDescriptor("bad", "m-bad", "openai", "arms-runner")] + finished = [] + + with pytest.raises(ValueError, match="does not support seed for model 'm-bad'"): + run_matrix(ArmsRunConfig(product="demo", seed=1), cells, ["baseline"], + [{"id": "q1", "question": "How?"}], on_cell=lambda cell, out: finished.append(cell)) + + assert finished == [] and provider_calls.calls == [] From 84e6224d0449448b7226c16634b92d76bdb9d21a Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 28 Oct 2026 20:18:13 -0700 Subject: [PATCH 05/12] test(arms): pin that matrix-run reports a blank question with exit 2 run_matrix re-raises EmptyQuestionError past the MatrixCellError wrapper, so a blank question is bad input (exit 2), not a failed cell (exit 1). Nothing pinned it: dropping it from the tuple left the suite green. Signed-off-by: Alexander Rybkin Signed-off-by: Nikolay Petrov --- dokimi/tests/test_empty_question_rule.py | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/dokimi/tests/test_empty_question_rule.py b/dokimi/tests/test_empty_question_rule.py index f2b5f95a9..5ce51ea49 100644 --- a/dokimi/tests/test_empty_question_rule.py +++ b/dokimi/tests/test_empty_question_rule.py @@ -196,6 +196,32 @@ def test_arms_run_refuses_a_blank_question_before_any_answer_is_paid_for(tmp_pat assert answered == [] and not out.exists() +def test_matrix_run_reports_a_blank_question_as_bad_input_not_a_failed_cell(tmp_path, monkeypatch, capsys): + """A blank question is the question set's fault, so it exits 2 like ``arms run``, not 1 as a cell failure.""" + import dokimi.eval.arms_run as arms_run + from dokimi import cli + + questions = tmp_path / "q.json" + questions.write_text(json.dumps({"questions": [{"id": "q1", "question": "How?"}, + {"id": "q2", "question": " "}]}), encoding="utf-8") + cells = tmp_path / "cells.json" + cells.write_text(json.dumps({"matrix": {"cells": [ + {"id": "c", "model": "m", "provider": "openai", "harness": "arms-runner"}]}}), encoding="utf-8") + answered = [] + monkeypatch.setattr(ArmRunner, "run", lambda self, library_name, questions, **kw: answered.append(1) or [ + _arms_record(q["id"], q["question"]) for q in questions]) + monkeypatch.setattr(arms_run, "probe_sampling", lambda *a: None) + out = tmp_path / "mx" + monkeypatch.setattr(sys, "argv", ["dokimi", "arms", "matrix-run", "--product", "lib", "--questions", str(questions), + "--arms", "baseline", "--matrix-cells", str(cells), "--judge", + "--out-dir", str(out)]) + with pytest.raises(SystemExit) as exc: + cli.main() + err = capsys.readouterr().err + assert exc.value.code == 2 and err.startswith("Error: question 'q2'"), err # not "Error running matrix cell" + assert answered == [] and not list(out.glob("cells/*.json")) + + # ── a record that would never be judged is skipped, not refused ────────── def _blank(qid="q2", **extra): From dd0ff8239019856a75c586bdb64ff0937a0126fc Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Thu, 29 Oct 2026 06:26:44 -0700 Subject: [PATCH 06/12] test(answers): import resolve_doc_library_id from dokimi.eval.arms_run #253 came in from main with the old dokimi.commands.arms._resolve_doc_library_id, which this branch moved and renamed; the merge failed test_both_stacks_match_pinned_values with an ImportError. Signed-off-by: Vladislav Perevezentsev Signed-off-by: Nikolay Petrov --- dokimi/tests/test_answering_stack_parity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py index 0a2daca21..7ad72a58e 100644 --- a/dokimi/tests/test_answering_stack_parity.py +++ b/dokimi/tests/test_answering_stack_parity.py @@ -262,7 +262,7 @@ def _observe(tmp_path: Path, provider_calls) -> Dict[str, Tuple[Any, Any]]: mp, tmp_path / "p-noid", provider_calls, context7_id=None, questions=QUESTIONS[:1] ) - from dokimi.commands.arms import _resolve_doc_library_id + from dokimi.eval.arms_run import resolve_doc_library_id from dokimi.registry import ProductRegistry from dokimi.treatments.arms import DocTreatment @@ -272,7 +272,7 @@ def _observe(tmp_path: Path, provider_calls) -> Dict[str, Tuple[Any, Any]]: f" description: fixture\n context7_id: {REGISTRY_ID}\n", encoding="utf-8", ) - a_lib = _resolve_doc_library_id( + a_lib = resolve_doc_library_id( [DocTreatment(a.client, name="docs", top_k=3, reranker=a.reranker)], PRODUCT, registry=ProductRegistry(registry_path), From 82a991e53ae451337517c7aa10c6b409eac94027 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Thu, 29 Oct 2026 06:16:05 -0700 Subject: [PATCH 07/12] fix(arms): ArmsRunConfig picks the default judge for its judge_provider judge_model defaulted to the Claude judge whatever judge_provider said, so ArmsRunConfig(judge=True, judge_provider="openai") judged with claude-sonnet-5-5 on openai, while arms run --judge-provider openai runs gpt-6.1-sol. The CLI paths were right only because parse_args resolves the --judge-model sentinel first. The field now defaults to that sentinel and resolves it the way parse_args does; a provider with no default judge (openrouter) needs judge_model, as on the command line. Signed-off-by: Vladislav Perevezentsev Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arms_run.py | 14 ++++++++++++-- dokimi/tests/test_arms_run_config.py | 25 +++++++++++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/eval/arms_run.py b/dokimi/dokimi/eval/arms_run.py index 325ee317c..84a75669c 100644 --- a/dokimi/dokimi/eval/arms_run.py +++ b/dokimi/dokimi/eval/arms_run.py @@ -20,8 +20,9 @@ from dokimi.defaults import ( DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, - DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, + JUDGE_MODEL_FOR_PROVIDER, + default_judge_model, ) from dokimi.eval.cells import HARNESS_ARMS_RUNNER, MatrixCellDescriptor from dokimi.eval.question_text import EmptyQuestionError @@ -49,7 +50,8 @@ class ArmsRunConfig: baseline_arm: str = "baseline" stats_correction: str = "holm" judge: bool = False - judge_model: str = DEFAULT_JUDGE_MODEL + #: Left out, the default judge that runs on ``judge_provider``, as for ``--judge-model``. + judge_model: str = JUDGE_MODEL_FOR_PROVIDER judge_provider: str = DEFAULT_JUDGE_PROVIDER judge_temperature: float | None = None judge_panel: str = "" @@ -64,6 +66,14 @@ class ArmsRunConfig: turn_reminder: str | None = None concurrency: int = 5 + def __post_init__(self) -> None: + if self.judge_model is JUDGE_MODEL_FOR_PROVIDER: + model = default_judge_model(self.judge_provider) + if model is None: + raise ValueError(f"judge provider {self.judge_provider!r} has no default judge " + f"model; pass judge_model") + object.__setattr__(self, "judge_model", model) + class LibraryIdError(ValueError): """No library id can be named for a doc arm, so the run stops before any answer.""" diff --git a/dokimi/tests/test_arms_run_config.py b/dokimi/tests/test_arms_run_config.py index 41628b3a7..35be2c477 100644 --- a/dokimi/tests/test_arms_run_config.py +++ b/dokimi/tests/test_arms_run_config.py @@ -20,13 +20,30 @@ from dokimi.eval.arms_run import ArmsRunConfig -def test_the_config_defaults_are_what_arms_run_runs_without_a_flag(): +ARMS_RUN = ["arms", "run", "--product", "demo", "--questions", "q.json", "--arms", "baseline"] + + +@pytest.mark.parametrize("judge_provider", [[], ["--judge-provider", "openai"]]) +def test_the_config_defaults_are_what_arms_run_runs_without_a_flag(judge_provider): # A command without some flag (subjects run has no --temperature) and a caller building # the config itself both run with these defaults, so they must not drift from the flags'. - args = parse_args(["arms", "run", "--product", "demo", "--questions", "q.json", - "--arms", "baseline"]) + # The judge model's default hangs on the judge provider, as --judge-model's does. + args = parse_args([*ARMS_RUN, *judge_provider]) + config = ArmsRunConfig(product="demo", judge_provider=args.judge_provider) + + assert run_config_from_args(args) == config - assert run_config_from_args(args) == ArmsRunConfig(product="demo") + +def test_a_judge_provider_with_no_default_judge_needs_a_judge_model(capsys): + with pytest.raises(SystemExit) as exc: + parse_args([*ARMS_RUN, "--judge-provider", "openrouter"]) + assert exc.value.code == 2 + assert "--judge-provider openrouter has no default judge model" in capsys.readouterr().err + + with pytest.raises(ValueError, match="'openrouter' has no default judge model"): + ArmsRunConfig(product="demo", judge_provider="openrouter") + assert ArmsRunConfig(product="demo", judge_provider="openrouter", + judge_model="m").judge_model == "m" @pytest.fixture From adf0aff65f27c1692026db970426c90fdd65eeda Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Thu, 29 Oct 2026 06:16:05 -0700 Subject: [PATCH 08/12] test(arms): describe what the parity tests pin, not the hand-off they were written against Signed-off-by: Vladislav Perevezentsev Signed-off-by: Nikolay Petrov --- dokimi/tests/test_arms_run_config_parity.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dokimi/tests/test_arms_run_config_parity.py b/dokimi/tests/test_arms_run_config_parity.py index 5bcf85636..7f2f8017d 100644 --- a/dokimi/tests/test_arms_run_config_parity.py +++ b/dokimi/tests/test_arms_run_config_parity.py @@ -3,12 +3,11 @@ """What each arms entry point hands the runner, for every run-configuration field. -``arms run``, ``arms matrix-run`` and ``subjects run`` all end in an arms run, each by its -own path: ``matrix-run`` copies its Namespace per cell, ``subjects run`` builds one by hand -from ``FORWARDED_RUN_CONFIG_FIELDS``, and the shared code reads most fields with a -``getattr`` default. A field one path drops is indistinguishable from a user who chose the -default, so this pins, per entry point, what reaches the runner when every field it takes -is set away from its default and when every field is left out. +``arms run``, ``arms matrix-run`` and ``subjects run`` all end in an arms run, each from its +own flags: ``matrix-run`` puts each cell's model, provider, harness and plugins in, and +``subjects run`` has only some of the flags. A field one path drops is indistinguishable from +a user who chose the default, so this pins, per entry point, what reaches the runner when +every field it takes is set away from its default and when every field is left out. The argv goes through ``dokimi.cli.parse_args`` (what ``main`` calls) to the command's own handler. The boundary is the ``ArmRunner`` constructor and its ``run``/``judge``/ From 6c47f44519715201dd928f1082c8bc673980d89f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:52:23 -0700 Subject: [PATCH 09/12] chore(growth): lower the test_arm_runner baseline after the rebase onto main check_growth.py --update, once the arms split is on top of main. Refs #4 Signed-off-by: Nikolay Petrov --- dokimi/scripts/growth_baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 644da66a2..75a2197ec 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -5,7 +5,7 @@ "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/report/model_compare.py": {"lines": 1207, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, - "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"}, + "tests/test_arm_runner.py": {"lines": 1175, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, "tests/test_container_agent.py": {"lines": 1251, "rationale": "pre-existing when the gate was introduced"}, "tests/test_judge_reliability.py": {"lines": 1695, "rationale": "grew on main in #73 (judge-validity --context stored), 1505 -> 1695, before the ratchet landed; split it in its own PR"}, From 29bdc60326642260f8f34a3f49af163d3d4cfd98 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:05 -0700 Subject: [PATCH 10/12] test(arms): point the output-error tests at the moved run_arms _build_arms_output moved to dokimi/eval/arms_run.py as run_arms, so the matrix-run tests from #238 patch the new name. Found while rebasing #270 onto main. Refs #4 Signed-off-by: Nikolay Petrov --- dokimi/tests/test_arms_output_errors.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dokimi/tests/test_arms_output_errors.py b/dokimi/tests/test_arms_output_errors.py index 989be7a56..048849bb8 100644 --- a/dokimi/tests/test_arms_output_errors.py +++ b/dokimi/tests/test_arms_output_errors.py @@ -119,7 +119,7 @@ def _matrix_run(tmp_path: Path, out_dir: Path, *extra: str): def _stub_matrix(monkeypatch, tmp_path): from dokimi.commands import arms - monkeypatch.setattr(arms, "_build_arms_output", lambda *a, **k: {}) + monkeypatch.setattr("dokimi.eval.arms_run.run_arms", lambda *a, **k: {}) monkeypatch.setattr(arms, "_save_arms_output", lambda *a, **k: None) delta = tmp_path / "out" / "delta.json" rollup = {"plugin_deltas": [{"artifact": str(delta), "_artifact_payload": {"d": 1}}]} @@ -190,12 +190,10 @@ def test_matrix_run_exits_1_on_an_unreadable_questions_file(tmp_path, capsys, ki @pytest.mark.parametrize("error", [PermissionError("denied"), RecursionError("deep")], ids=lambda e: type(e).__name__) def test_matrix_run_exits_1_when_a_cell_cannot_read_its_inputs(tmp_path, monkeypatch, capsys, error): - from dokimi.commands import arms - def build(*a, **k): raise error - monkeypatch.setattr(arms, "_build_arms_output", build) + monkeypatch.setattr("dokimi.eval.arms_run.run_arms", build) args = _matrix_run(tmp_path, tmp_path / "out") with pytest.raises(SystemExit) as exc: args.func(args) From 4877615d886194f0d8fc480447e865ef5bc28062 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:31:41 -0700 Subject: [PATCH 11/12] test(arms): keep the config-defaults test's comment block to two lines The Comment blocks check (#298, on main since the rebase) fails a block of three lines that a PR adds. The rest of the reason is in the PR. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_arms_run_config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/test_arms_run_config.py b/dokimi/tests/test_arms_run_config.py index 35be2c477..7ca5f94a8 100644 --- a/dokimi/tests/test_arms_run_config.py +++ b/dokimi/tests/test_arms_run_config.py @@ -25,9 +25,8 @@ @pytest.mark.parametrize("judge_provider", [[], ["--judge-provider", "openai"]]) def test_the_config_defaults_are_what_arms_run_runs_without_a_flag(judge_provider): - # A command without some flag (subjects run has no --temperature) and a caller building - # the config itself both run with these defaults, so they must not drift from the flags'. - # The judge model's default hangs on the judge provider, as --judge-model's does. + # A command without a flag (subjects run has no --temperature) and a caller building the + # config both run these defaults, so they must not drift from the flags'; see #270. args = parse_args([*ARMS_RUN, *judge_provider]) config = ArmsRunConfig(product="demo", judge_provider=args.judge_provider) From dc016592cd94a84037898633b47dc364cfa57904 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:34:16 -0700 Subject: [PATCH 12/12] fix(arms): check_matrix names the cell a refusal is about The docstring promises a ValueError naming the first bad cell, but a harness or sampling refusal did not say which cell it came from. Each cell's refusal now starts with its name. Harness checks still run for every cell before the first sampling probe is paid for. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/arms_run.py | 11 +++++++++-- dokimi/tests/test_arms_run_config.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/eval/arms_run.py b/dokimi/dokimi/eval/arms_run.py index 84a75669c..34e05e405 100644 --- a/dokimi/dokimi/eval/arms_run.py +++ b/dokimi/dokimi/eval/arms_run.py @@ -370,9 +370,16 @@ def run_arms(config: ArmsRunConfig, specs: list[str], questions: list[dict]) -> def check_matrix(config: ArmsRunConfig, cells: list[MatrixCellDescriptor]) -> None: """Refuse a matrix with a cell the in-process runner cannot run, or whose provider would refuse the sampling on every answer. Raises ``ValueError`` naming the first such cell.""" - validate_supported_matrix_harnesses(cells) for cell in cells: - check_sampling_support(cell_config(config, cell)) + try: + validate_supported_matrix_harnesses([cell]) + except ValueError as exc: + raise ValueError(f"matrix cell '{cell.name}': {exc}") from exc + for cell in cells: + try: + check_sampling_support(cell_config(config, cell)) + except ValueError as exc: + raise ValueError(f"matrix cell '{cell.name}': {exc}") from exc def run_matrix(config: ArmsRunConfig, cells: list[MatrixCellDescriptor], specs: list[str], diff --git a/dokimi/tests/test_arms_run_config.py b/dokimi/tests/test_arms_run_config.py index 7ca5f94a8..cdaefb559 100644 --- a/dokimi/tests/test_arms_run_config.py +++ b/dokimi/tests/test_arms_run_config.py @@ -126,3 +126,28 @@ def test_run_matrix_checks_every_cell_before_the_first_is_paid_for(monkeypatch, [{"id": "q1", "question": "How?"}], on_cell=lambda cell, out: finished.append(cell)) assert finished == [] and provider_calls.calls == [] + + +def test_check_matrix_names_the_first_cell_whose_sampling_the_provider_refuses(monkeypatch): + from dokimi.eval.arms_run import check_matrix + from dokimi.eval.cells import MatrixCellDescriptor + + monkeypatch.setattr("dokimi.llm.unsupported_sampling_params", + lambda model, provider, **kw: ["seed"] if model == "m-bad" else []) + monkeypatch.setattr("dokimi.eval.arms_run.probe_sampling", lambda *a: None) + cells = [MatrixCellDescriptor("ok", "m-ok", "openai", "arms-runner"), + MatrixCellDescriptor("bad", "m-bad", "openai", "arms-runner")] + + with pytest.raises(ValueError, match="^matrix cell 'bad': Provider 'openai' does not support"): + check_matrix(ArmsRunConfig(product="demo", seed=1), cells) + + +def test_check_matrix_names_the_first_cell_with_an_unsupported_harness(): + from dokimi.eval.arms_run import check_matrix + from dokimi.eval.cells import MatrixCellDescriptor + + cells = [MatrixCellDescriptor("ok", "m", "openai", "arms-runner"), + MatrixCellDescriptor("odd", "m", "openai", "other-runner")] + + with pytest.raises(ValueError, match="^matrix cell 'odd': matrix cells only support"): + check_matrix(ArmsRunConfig(product="demo"), cells)