diff --git a/.github/workflows/agent-quality.yml b/.github/workflows/agent-quality.yml index 6f42eb7c4..09c62c151 100644 --- a/.github/workflows/agent-quality.yml +++ b/.github/workflows/agent-quality.yml @@ -25,27 +25,8 @@ defaults: # Runners: see the header of ci.yml. jobs: - # THE executable-track proof for this repo. It is now the only job that - # builds and verifies a Docker task, because the task definitions no longer - # live here: they are supplied from an external content checkout via - # --tasks-root (see dokimi/docs/contributing-terminal-bench-task.md - # for the format). That makes this the right shape of test anyway -- it - # exercises the framework against content it did not author. - # - # Real end-to-end proof that this repo's own harnesses can pick up and run a - # skill/task from an external catalog. Clones a real, public - # uxlfoundation/skills checkout, scans it with `catalog scan`, then runs one - # of its real tasks through our own `docker-oracle` harness (build + solve + - # verify: the build and verifier steps `docker-claude`/`docker-claude-skill` - # share) and `docker-probe` (the in-container agent launch those arms use, - # without the LLM). It asserts the actual - # pass rate in "Verify pass rate" below -- `tasks run`'s own exit code does - # not reflect pass/fail (it is a reporting command, used by callers that want - # partial results), so without that explicit check this job would stay green - # even if the external task failed verification. Pinned to a fixed commit - # (UXL_SKILLS_COMMIT below) and retried 3x on clone so this job's result - # reflects our own code, not an upstream repo edit or a transient network - # blip. + # Runs a real task from a pinned external catalog through docker-oracle and docker-probe. + # `tasks run` exits 0 for a failed verification, so "Verify pass rate" below asserts the pass rate. external-catalog-smoke: name: External catalog smoke test (uxlfoundation/skills) runs-on: ${{ github.repository_owner == 'intel' && github.event_name == 'pull_request' && 'dokimi-ec2' || 'ubuntu-latest' }} diff --git a/dokimi/changelog.d/20261021_180000_napetrov_outcome-completeness-2.md b/dokimi/changelog.d/20261021_180000_napetrov_outcome-completeness-2.md index 40af7a4b4..d2b5884e9 100644 --- a/dokimi/changelog.d/20261021_180000_napetrov_outcome-completeness-2.md +++ b/dokimi/changelog.d/20261021_180000_napetrov_outcome-completeness-2.md @@ -1,3 +1,3 @@ ### Fixed -- **Nothing measured is not a pass (second batch)** — `tasks run` exits 1 as `incomplete` when every cell errored (the artifact and report are still written; partly errored runs keep exit 0 and their counts). The novelty gate's status is `NOT MEASURED`, not `PASS`, for a cell with no check measured. `run` refuses with one line and exit 1 when a selected document does not load, instead of a KeyError or OSError traceback. `baseline compare` says "Nothing to compare (0 scored)" when no question is scored on both sides, and its "No notable changes" line counts the questions compared; the snapshot `compare` report says "Nothing to compare (0 scored)" instead of "No regressions" when either snapshot has no document. +- **Nothing measured is not a pass (second batch)** — `tasks run` exits 1 as `incomplete` when every cell errored (the artifact and report are still written). The novelty gate's status is `NOT MEASURED`, not `PASS`, for a cell with no check measured. `run` refuses with one line and exit 1 when a selected document does not load, instead of a KeyError or OSError traceback. `baseline compare` says "Nothing to compare (0 scored)" when no question is scored on both sides, and its "No notable changes" line counts the questions compared; the snapshot `compare` report says "Nothing to compare (0 scored)" instead of "No regressions" when either snapshot has no document. diff --git a/dokimi/changelog.d/20261026_evaluation_completion.md b/dokimi/changelog.d/20261026_evaluation_completion.md new file mode 100644 index 000000000..bcadf21e1 --- /dev/null +++ b/dokimi/changelog.d/20261026_evaluation_completion.md @@ -0,0 +1,19 @@ + + +### Changed + +- Task runs preserve artifacts and exit nonzero for any errored, unscored or missing planned cell, including partly errored runs that previously exited 0; supported dry runs remain plans and verifier failures remain scored outcomes. + +### Fixed + +- `baseline compare` now treats a blank, non-string or case-variant "Unknown" provider or model, and a panel member without a `provider/` prefix, as not recorded, for every run: the "not compared" warning appears and `--strict` exits 1 where it used to pass. +- A command harness with no reward file records its exit code as reward 1.0 or 0.0, so `tasks run` counts it as scored; exit 126, 127, signal exits, Windows crash statuses (NTSTATUS error codes) and a reward file with no reward in it are no verdict, even with a `reward: x` on the output: the cell is errored and stays out of `pass_rate` and the paired test. `avg_reward` of command harnesses now includes these exit-code cells, so it does not compare with older `task_runs` artifacts. +- `tasks run` exits 2 before running when a task or harness is listed twice, including one task under two spellings and two tasks with the same directory name. +- Undecided judge-validity checks visibly report INCONCLUSIVE, with the existing default and strict exit policies. + +### Added + +- Opt-in baseline context-score and paired-lift regression thresholds, requiring complete matched scoring and known matching model identities. A regression and an identity mismatch exit 1; partial or insufficient evidence exits 2. `--min-matched` without a threshold exits 2. Existing `--strict` remains an identity check. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index db3f19ccd..a46e330d8 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -486,8 +486,12 @@ The report gives, per degradation, the pairs scored, the gate accuracy with its 95% CI and PASS or FAIL; `--sequential` adds a `mode` line and marks a degradation still undecided at the stop `continue`, with the reason under INCONCLUSIVE (cap reached, or the -answer pool exhausted). Exit 1 on FAIL, and on an inconclusive pass with -`--strict`. A malformed input, such as a record that would be judged with no +answer pool exhausted). An undecided result prints `RESULT: INCONCLUSIVE` +when it is above the gate and `RESULT: FAIL (INCONCLUSIVE, see above)` when it +is below, never PASS. Exit policy is unchanged: a below-threshold point estimate exits 1; +an above-threshold inconclusive result warns and exits 0 by default, or 1 with +`--strict`. A settled PASS exits 0. JSON retains `passed` (the point-estimate +check) and `conclusive`; promotion requires both to be true. A malformed input, such as a record that would be judged with no question text, exits 2, so it never reads as a failed gate. `--mode pairwise` shows the judge both answers, in both orders, and asks which @@ -811,6 +815,28 @@ on a bad eval file or `manifest.json`, and `baseline list` exits 2 on a bad lack a string `name` and `path` or have a `summary` that is not an object, is rejected the same way. +Regression promotion is opt-in and separate from identity-only `--strict`: + +```bash +python cli.py baseline compare --baseline onetbb-gpt55-docs \ + --eval results/onetbb_candidate/eval/onetbb.json \ + --max-context-drop 1 --max-lift-drop 0.5 --min-matched 20 +``` + +`--max-context-drop` limits the mean saved-minus-current context score. +`--max-lift-drop` limits the mean saved-minus-current paired lift, where lift +is context score minus the no-context score for the same question. Both use +unrounded per-question aggregates, not stored rounded deltas or manifest means. +Thresholds are finite, non-negative score points; equality passes. Either flag +enables the gate; when both are supplied both must pass. `--min-matched` defaults +to 1 and needs a threshold flag (alone it exits 2). The gate exits 1 on a +regression or an unknown or mismatched answer/judge identity. Insufficient +evidence exits 2, since the input can be fixed: every question in both artifacts +must match uniquely and have finite scores in both conditions, and at least +`--min-matched` must match. Unmatched questions, duplicates, errors and partial +scoring cannot promote a candidate. Invalid thresholds and malformed records +exit 2. Issue #94 moves these thresholds to `gate`. + Use baselines for release-to-release tracking. Use `--questions-from` for model-to-model fairness. diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index f673acf55..9ed9f11d7 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -579,10 +579,12 @@ a verdict. The rules mirror intel/skills' `tools/check_harbor_job.py`: | no trial file (with or without the job's), more than one trial, or not a JSON object | errored | An errored cell is out of the pass-rate denominator and the paired test, as -for `docker-*` and BenchFlow. When every cell of a `tasks run` errored, it -still writes the artifact and the report, then exits 1 as `incomplete` -(`0 scored, N errored of N cells`); a run with some cells scored exits 0 and -prints the errored count per harness. `metrics.n_trials` counts the trial files and +for `docker-*` and BenchFlow. When any planned cell of a `tasks run` errors, lacks a finite score, or is missing, +the command writes the artifact and report, then exits 1 as `incomplete`. +It reports scored, errored, unscored and missing counts and names missing cells. +A completed verifier failure (reward 0) is scored evidence and does not itself +change the exit code. A supported `--dry-run` plan exits 0 without claiming +execution; unsupported cells still fail. `metrics.n_trials` counts the trial files and `metrics.result_json` names the one read. A `harbor` that `dokimi` kills at the task budget is a failed cell (reward 0, returncode 124), as above. @@ -604,8 +606,11 @@ refuses the task instead of running it with the internet the task forbade. Both now land as errored cells. A `command:` harness (including a `--command-template` that wraps `harbor`) -keeps its own rule: a `reward.txt` / `reward.json` under the run dir, else a -`reward: x` on its output, else exit code 0 is a pass. +keeps its own rule: a `reward.txt` / `reward.json` under the run dir, else (no +reward file) a `reward: x` on its output, else exit code 0 is a pass. Exit 126 or 127, a +signal exit, a Windows crash status (an NTSTATUS error code such as `0xC0000409`), or a reward file with no reward in it is no verdict: the cell is +errored (`metrics.solver_error`). Only a reward file with a reward in it outranks such an +exit; a `reward: x` on the output does not. ### Network policy diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..c1fdd5dbc 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1578,6 +1578,54 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-context-drop", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.baseline_regression.threshold#83cba01472d6" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--max-lift-drop", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.commands.baseline_regression.threshold#83cba01472d6" + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--min-matched", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" + }, { "action": "_StoreAction", "choices": null, diff --git a/dokimi/docs/user-journeys/journeys.md b/dokimi/docs/user-journeys/journeys.md index 1f26b9c5e..a6b678d39 100644 --- a/dokimi/docs/user-journeys/journeys.md +++ b/dokimi/docs/user-journeys/journeys.md @@ -717,19 +717,21 @@ cost. `python cli.py baseline compare --strict`), as do static gates (`python cli.py repo scan --fail-on-fail`). `baseline compare --strict` adds a non-zero exit only when a model cannot be verified or differs from the -baseline; a regression in the scores is printed and exits 0. Failing on a regression beyond -a threshold is **proposed** (AC-J12-001). Blocking policy is **partial** -(AC-J12-002): `repo tasks check --allow-unsupported` decides host-unsupported -tasks; budget-stopped and inconclusive judge results have no policy. +baseline. Explicit `--max-context-drop` and `--max-lift-drop` thresholds block +score regressions, requiring complete matched scores and known matching models +(AC-J12-001). `eval judge-validity --strict` blocks inconclusive checks; without +it an above-threshold inconclusive check warns and exits 0. Blocking policy remains +partial for budget-stopped checks (AC-J12-002). Native `tasks run` rejects any +missing, unscored or errored planned cell and preserves the artifacts (AC-J12-004). ### Use cases and acceptance #### UC-J12-001 Gate a PR on a budgeted check -- **AC-J12-001** A strict baseline comparison exits non-zero on a regression beyond the threshold. -- **AC-J12-002** An inconclusive or budget-stopped check is reported as such, and whether it blocks is the configured policy's decision. *Partial: one flag, for host-unsupported tasks; budget-stopped and inconclusive judge results have none.* +- **AC-J12-001** An explicitly gated baseline comparison exits non-zero on a regression beyond its context-score or paired-lift threshold; `--strict` alone checks model identity. +- **AC-J12-002** An inconclusive or budget-stopped check is reported as such, and whether it blocks is the configured policy's decision. *Partial: explicit flags cover host-unsupported tasks and inconclusive judge checks; general budget-stop policy remains proposed.* - **AC-J12-003** The reusable CI check runs only the tasks a change touches, adds a smoke set when shared harness files change, and fails on an unknown task name instead of checking nothing. -- **AC-J12-004** A gate on a run whose result lacks planned cells reports it incomplete and names the missing cells; it never passes on the cells that finished. *Proposed.* +- **AC-J12-004** A gate on a run whose result lacks planned cells reports it incomplete and names the missing cells; it never passes on the cells that finished. *Partial: enforced for native task runs.* --- diff --git a/dokimi/dokimi/commands/baseline.py b/dokimi/dokimi/commands/baseline.py index 47506594d..b6f264339 100644 --- a/dokimi/dokimi/commands/baseline.py +++ b/dokimi/dokimi/commands/baseline.py @@ -13,6 +13,7 @@ from typing import Any, Dict, List, Optional, Tuple from dokimi.utils import model_identity, same_model +from dokimi.commands import baseline_regression from dokimi.exit_codes import EXIT_USAGE BASELINES_DIR = Path("baselines/eval") @@ -99,9 +100,10 @@ def _recorded_models(eval_data: Dict[str, Any]) -> Dict[str, Optional[Tuple[str, meta = {} def pair(provider: Any, model: Any) -> Optional[Tuple[str, str]]: - if not provider or not model or "unknown" in (provider, model): + if any(not isinstance(v, str) or not v.strip() or v.strip().lower() == "unknown" + for v in (provider, model)): return None - if model_identity(str(model))[0] is None: # a route with no id after it ("anthropic/") names no model + if model_identity(str(model))[0] in (None, "unknown"): # no id after the route ("anthropic/"), or only "unknown" return None return str(provider), str(model) @@ -109,7 +111,8 @@ def pair(provider: Any, model: Any) -> Optional[Tuple[str, str]]: if str(judge_model or "").startswith("panel("): panel = eval_data.get("panel_config") models = panel.get("models") if isinstance(panel, dict) else None - if not (isinstance(models, list) and all(isinstance(m, str) for m in models)): + if not (isinstance(models, list) and all(isinstance(m, str) and "/" in m + and pair(*m.split("/", 1)) is not None for m in models)): models = [] judge = ("panel", ",".join(sorted(models))) if models else None else: @@ -276,7 +279,8 @@ def _pick_entry(args: argparse.Namespace, baselines: list, eval_data: dict) -> d def _report_model_checks(args: argparse.Namespace, entry: dict, baseline_data: dict, eval_data: dict) -> None: mismatches, unverified = _model_mismatches(baseline_data, eval_data) - strict = getattr(args, "strict", False) + strict = getattr(args, "strict", False) or baseline_regression.enabled(args) + policy = "--strict" if getattr(args, "strict", False) else "regression gate" base_rec, cur_rec = _recorded_models(baseline_data), _recorded_models(eval_data) for role in unverified: b, c = base_rec[role], cur_rec[role] @@ -287,13 +291,13 @@ def _report_model_checks(args: argparse.Namespace, entry: dict, baseline_data: d sides = [name for name, rec in (("baseline", base_rec), ("current eval", cur_rec)) if rec[role] is None] print(f"⚠ {role} model not recorded in {' and '.join(sides)}; not compared", file=sys.stderr) if unverified and strict: - print(f"❌ --strict cannot verify the {' and '.join(unverified)} model against " + print(f"❌ {policy} cannot verify the {' and '.join(unverified)} model against " f"baseline '{entry['name']}'", file=sys.stderr) raise SystemExit(1) if mismatches: detail = "".join(f"\n {line}" for line in mismatches) if strict: - print(f"❌ Baseline '{entry['name']}' was scored with different models (--strict):{detail}", + print(f"❌ Baseline '{entry['name']}' was scored with different models ({policy}):{detail}", file=sys.stderr) raise SystemExit(1) print(f"⚠ Baseline model warning: '{entry['name']}' was scored with different models;" @@ -316,6 +320,7 @@ def _print_metric_table(cur_summary: dict, base_summary: dict) -> None: def cmd_baseline_compare(args: argparse.Namespace) -> None: """Compare an eval result against a saved baseline.""" + baseline_regression.require_gate_for_min_matched(args) eval_path = Path(args.eval) eval_data = _read_json_object(eval_path, 2) # 1 is the --strict verdict manifest = _baseline_manifest(2) @@ -333,8 +338,9 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None: # Read both record lists before the model checks: a malformed file must exit 2, not reach the --strict verdict. try: - baseline_evals = {e.key: e for e in records(baseline_data.get("evaluations", []), - source=entry["path"], kind="evaluations")} + baseline_records = records(baseline_data.get("evaluations", []), + source=entry["path"], kind="evaluations") + baseline_evals = {e.key: e for e in baseline_records} current_evals = records(eval_data.get("evaluations", []), source=str(eval_path), kind="evaluations") except RecordError as exc: print(f"❌ {exc}", file=sys.stderr) @@ -377,6 +383,8 @@ def cmd_baseline_compare(args: argparse.Namespace) -> None: else: print(f"\n✅ No notable changes vs baseline ({n_compared} of {len(current_evals)} questions compared).") + baseline_regression.check(args, baseline_records, current_evals) + def register(sub, positive_int) -> None: """Add the `baseline` subcommand group.""" @@ -403,4 +411,12 @@ def register(sub, positive_int) -> None: bl_cmp.add_argument("--strict", action="store_true", help="Exit 1 instead of warning when the baseline's answer or judge model " "differs from the current eval's") + bl_cmp.add_argument("--max-context-drop", type=baseline_regression.threshold, default=None, + help="Gate on the mean matched context-score drop in score points (equality passes)") + bl_cmp.add_argument("--max-lift-drop", type=baseline_regression.threshold, default=None, + help="Gate on the mean matched (context minus baseline) lift drop in score points") + bl_cmp.add_argument("--min-matched", type=positive_int, default=None, + help="Minimum matched scored questions under a regression gate (default: 1); " + "needs --max-context-drop or --max-lift-drop; complete coverage and " + "matching known models are always required") bl_cmp.set_defaults(func=cmd_baseline_compare) diff --git a/dokimi/dokimi/commands/baseline_regression.py b/dokimi/dokimi/commands/baseline_regression.py new file mode 100644 index 000000000..125ceef3b --- /dev/null +++ b/dokimi/dokimi/commands/baseline_regression.py @@ -0,0 +1,70 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Opt-in, matched-question promotion checks for saved evaluation baselines.""" + +from __future__ import annotations + +import argparse +import math +from decimal import Decimal, InvalidOperation + +from dokimi.exit_codes import usage_exit +from dokimi.records import BASELINE, CONTEXT, Record + + +def threshold(value: str) -> Decimal: + """argparse type for a permitted drop in score points, kept exact for the Decimal comparison.""" + try: + number = Decimal(value) + if number.is_finite() and number >= 0: + return number + except InvalidOperation: + pass + raise argparse.ArgumentTypeError("expected a finite non-negative number") + + +def enabled(args: argparse.Namespace) -> bool: + return any(getattr(args, name, None) is not None for name in ("max_context_drop", "max_lift_drop")) + + +def require_gate_for_min_matched(args: argparse.Namespace) -> None: + if getattr(args, "min_matched", None) is not None and not enabled(args): + usage_exit("--min-matched needs --max-context-drop or --max-lift-drop") + + +def _scored(rows: list[Record]) -> dict[str, tuple[Decimal, Decimal]]: + values = {} + for row in rows: + context, baseline = row.score(CONTEXT), row.score(BASELINE) + if row.error or any(s.error or s.incomplete for s in row.scores.values()): + continue + if context is not None and baseline is not None and math.isfinite(context) and math.isfinite(baseline): + context_value = Decimal(str(context)) + values[row.key] = (context_value, context_value - Decimal(str(baseline))) + return values + + +def check(args: argparse.Namespace, baseline: list[Record], current: list[Record]) -> None: + """Require complete unique paired evidence, then compare unrounded score drops.""" + if not enabled(args): + return + base, cur = _scored(baseline), _scored(current) + matched = base.keys() & cur.keys() + minimum = getattr(args, "min_matched", None) or 1 + if len(matched) < minimum or len(matched) != len(baseline) or len(matched) != len(current): + usage_exit(f"Regression gate: insufficient or partial evidence ({len(matched)} matched scored, " + f"{len(baseline)} baseline, {len(current)} current; minimum {minimum}; " + "complete unique coverage required)") + failed = False + for index, name in enumerate(("max_context_drop", "max_lift_drop")): + limit = getattr(args, name, None) + if limit is None: + continue + drop = sum((base[key][index] - cur[key][index] for key in sorted(matched)), Decimal(0)) / len(matched) + failed |= drop > limit + metric = "context score" if index == 0 else "paired lift" + print(f"{metric}: drop={drop:.12g}, allowed={limit:.12g} ({len(matched)} matched)") + print("Regression gate: " + ("REGRESSION" if failed else "PASS")) + if failed: + raise SystemExit(1) diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 18f1da851..161a81203 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -437,13 +437,13 @@ def progress(done: int, total: int) -> None: # --sequential ignores --bases, so pointing at it would send the user nowhere. more = "Raise --max-bases or add answers (see above)." if args.sequential else "Re-run with more --bases." print( - "\n⚠️ Pass is inconclusive at this sample size: " + "\n⚠️ INCONCLUSIVE at this sample size: " f"{', '.join(summary['inconclusive_variants'])}. {more}", file=sys.stderr, ) if summary["unmeasured_variants"]: print( - f"\n⚠️ Pass is inconclusive: no pair was built for {', '.join(summary['unmeasured_variants'])} " + f"\n⚠️ INCONCLUSIVE: no pair was built for {', '.join(summary['unmeasured_variants'])} " "(the degradation left every answer unchanged).", file=sys.stderr, ) @@ -517,7 +517,7 @@ def register(sub, positive_int) -> None: help=f"Pairwise accuracy gate, per degradation (default: {DEFAULT_MIN_ACCURACY}); " "with --sequential it must lie in 0.03-0.97") validity_p.add_argument("--strict", action="store_true", - help="Also exit 1 when a PASS is inconclusive — its 95%% CI " + help="Also exit 1 when an above-threshold result is INCONCLUSIVE — its 95%% CI " "straddles the gate, so the run cannot tell it from a FAIL") validity_p.add_argument("--sequential", action="store_true", help="Add bases in rounds and stop each degradation once an SPRT can call " diff --git a/dokimi/dokimi/commands/tasks.py b/dokimi/dokimi/commands/tasks.py index cf04ea8c3..2af69cbc6 100644 --- a/dokimi/dokimi/commands/tasks.py +++ b/dokimi/dokimi/commands/tasks.py @@ -131,6 +131,7 @@ def _select_swebench(args: argparse.Namespace) -> list: ids = None if args.all else [t.strip() for t in (args.tasks or "").split(",") if t.strip()] if ids == []: usage_exit("No tasks selected. Pass --tasks or --all.") + _refuse_repeats("task", ids or []) try: if ids is None: skipped = sorted(KNOWN_BAD.keys() & {str(r.get("instance_id")) for r in read_instances(Path(args.swe_bench))}) @@ -146,6 +147,12 @@ def _select_swebench(args: argparse.Namespace) -> list: return tasks +def _refuse_repeats(kind: str, names: list[str]) -> None: + repeated = sorted({n for n in names if names.count(n) > 1}) + if repeated: + usage_exit(f"Repeated {kind}: {', '.join(repeated)}. List each once; use --repeats to run it again.") + + def _select_tasks(args: argparse.Namespace) -> list: from dokimi.harnesses import discover_tasks, load_task @@ -157,7 +164,10 @@ def _select_tasks(args: argparse.Namespace) -> list: tasks = discover_tasks(root) else: task_names = [t.strip() for t in args.tasks.split(",") if t.strip()] + _refuse_repeats("task", task_names) tasks = [load_task(t, root=root) for t in task_names] + # cells and output dirs are keyed by the loaded name, not the spelling + _refuse_repeats("task", [t.name for t in tasks]) except (ValueError, OSError, RecursionError) as exc: # a missing or unreadable task, deeply nested YAML usage_exit(f"Error: {exc}") if not tasks: @@ -183,11 +193,41 @@ def _print_summary(output: dict) -> None: cost_delta = cmp_row.get("cost_delta_usd") cost_s = "" if cost_delta is None else f", cost delta ${cost_delta:+.4f}" print(f"{harness} vs {cmp_row['baseline_harness']}: pass-rate delta {delta_s}{cost_s}") - per_harness = output["summary"]["per_harness"].values() - cells = sum(s["n"] for s in per_harness) - errored = sum(s.get("errored", 0) for s in per_harness) - if cells and errored == cells: # no cell has a verdict: "0/0 pass (n/a)" is not a result - print(f"✗ incomplete: every cell errored (0 scored, {errored} errored of {cells} cells)", file=sys.stderr) + + +def _cell_state(row: dict, dry_run: bool) -> str: + from dokimi.harnesses.runner import _num, is_errored + + if is_errored(row): + return "errored" + if dry_run and row.get("metrics", {}).get("dry_run"): + return "planned" + return "unscored" if _num(row.get("reward")) is None else "scored" + + +def _check_completion(output: dict, *, dry_run: bool = False) -> None: + """A completed failing grade is evidence; an errored or absent cell is not.""" + from collections import Counter + + planned = {(task["name"], harness, rep) + for task in output["tasks"] for harness in output["harnesses"] + for rep in range(1, output.get("repeats", 1) + 1)} + seen = {(r["task"], r["harness"], r.get("repeat", 1)) for r in output["results"]} + counts = Counter(_cell_state(row, dry_run) for row in output["results"]) + missing = planned - seen + duplicate = len(output["results"]) - len(seen) + unplanned = seen - planned + if dry_run: + print(f"Plan: {counts['planned']} supported cells; not executed") + incomplete = counts["errored"] + counts["unscored"] + len(missing) + duplicate + len(unplanned) + if incomplete or not planned: + print(f"✗ incomplete: {counts['scored']} scored, {counts['errored']} errored, " + f"{counts['unscored']} unscored, {len(missing)} missing of {len(planned)} planned cells", + file=sys.stderr) + if duplicate or unplanned: + print(f" invalid identities: {duplicate} duplicate, {len(unplanned)} unplanned", file=sys.stderr) + for task, harness, rep in sorted(missing): + print(f" missing: {task} / {harness} / repeat {rep}", file=sys.stderr) sys.exit(1) @@ -199,6 +239,7 @@ def cmd_tasks_run(args: argparse.Namespace) -> None: harnesses = [h.strip() for h in args.harnesses.split(",") if h.strip()] if not harnesses: usage_exit("No harnesses selected. Pass --harnesses codex,claude-code.") + _refuse_repeats("harness", harnesses) if args.baseline_harness and args.baseline_harness not in harnesses: print(f"Also running baseline harness: {args.baseline_harness}") harnesses.insert(0, args.baseline_harness) @@ -247,6 +288,7 @@ def cmd_tasks_run(args: argparse.Namespace) -> None: print(f"Saved task run report: {out_md}") _print_summary(output) + _check_completion(output, dry_run=args.dry_run) def register(sub, positive_int) -> None: diff --git a/dokimi/dokimi/eval/judge_validity.py b/dokimi/dokimi/eval/judge_validity.py index df5b90c08..981113541 100644 --- a/dokimi/dokimi/eval/judge_validity.py +++ b/dokimi/dokimi/eval/judge_validity.py @@ -413,7 +413,7 @@ def format_summary(self) -> str: if not s["n_pairs"]: result += " (no graded pair was scored)" elif not s["conclusive"]: - result += " (inconclusive — see above)" + result = "INCONCLUSIVE (see above)" if s["passed"] else "FAIL (INCONCLUSIVE, see above)" lines.append(" RESULT: " + result) return "\n".join(lines) diff --git a/dokimi/dokimi/harnesses/command.py b/dokimi/dokimi/harnesses/command.py index 3600b9744..7ab2ffa01 100644 --- a/dokimi/dokimi/harnesses/command.py +++ b/dokimi/dokimi/harnesses/command.py @@ -155,10 +155,13 @@ def run( given_dir / Path(harbor_metrics["result_json"]).relative_to(output_dir)) passed = reward is not None and reward >= 1.0 else: - reward = _find_reward(output_dir, stdout, stderr) - passed = reward is not None and reward >= 1.0 + # A printed reward is no verdict from a command that never ran or was killed. + printed = (stdout, stderr) if not _never_finished(returncode) else ("", "") + reward = _find_reward(output_dir, *printed) if reward is None: - passed = returncode == 0 + reward = _exit_code_reward(returncode, output_dir) + harbor_metrics = _no_verdict(reward, returncode, output_dir) + passed = reward is not None and reward >= 1.0 metrics: dict[str, Any] = { "dry_run": False, @@ -390,6 +393,37 @@ def _task_prompt(task: TaskSpec) -> str: ) +# Windows reports a crash or Ctrl-C as an NTSTATUS error code (0xC0000005, 0xC000013A, 0xC0000409). +_NTSTATUS_ERROR = 0xC0000000 + + +def _exit_code_reward(returncode: int, output_dir: Path | None) -> float | None: + """1.0 or 0.0 from an exit code; None when the command never ran (126, 127), was killed by a + signal or a Windows crash status, or left a reward file with no reward in it.""" + if _never_finished(returncode) or _has_reward_file(output_dir): + return None + return 1.0 if returncode == 0 else 0.0 + + +def _never_finished(returncode: int) -> bool: + return (returncode < 0 or returncode in (126, 127) or 128 < returncode <= 192 + or returncode >= _NTSTATUS_ERROR) + + +def _no_verdict(reward: float | None, returncode: int, output_dir: Path | None) -> dict[str, Any]: + """An errored cell when the exit code gave no verdict, so the summary leaves it out of pass_rate.""" + if reward is not None: + return {} + no_reward = ", reward file has no reward" if _has_reward_file(output_dir) else "" + return {"solver_error": f"no verdict: exit {returncode}{no_reward}"} + + +def _has_reward_file(output_dir: Path | None) -> bool: + if not output_dir: + return False + return any(next(output_dir.rglob(name), None) for name in ("reward.txt", "reward.json")) + + def _find_reward(output_dir: Path | None, stdout: str, stderr: str) -> float | None: if output_dir: candidates = sorted(output_dir.rglob("reward.txt")) + sorted( @@ -401,6 +435,8 @@ def _find_reward(output_dir: Path | None, stdout: str, stderr: str) -> float | N ) if parsed is not None: return parsed + if candidates: + return None # an unreadable reward file is not an absent one: no output fallback parsed = _parse_reward_text(stdout) return parsed if parsed is not None else _parse_reward_text(stderr) diff --git a/dokimi/dokimi/harnesses/runner.py b/dokimi/dokimi/harnesses/runner.py index 8f805aa13..92501893b 100644 --- a/dokimi/dokimi/harnesses/runner.py +++ b/dokimi/dokimi/harnesses/runner.py @@ -5,7 +5,9 @@ from __future__ import annotations +import hashlib import math +import re from dataclasses import dataclass, field, replace from datetime import datetime, timezone from pathlib import Path @@ -565,4 +567,8 @@ def _error_result(harness_id: str, task: TaskSpec, output_dir: Path, exc: Except def _safe_id(value: str) -> str: - return value.replace("/", "_").replace(":", "_").replace(" ", "_") + """A directory name for a harness id. Characters Windows refuses add a digest of the id, so ids that differ only in them stay apart.""" + name = value.replace("/", "_").replace(":", "_").replace(" ", "_") + if not re.search(r'[\\<>"|?*]', name): + return name + return re.sub(r'[\\<>"|?*]', "_", name) + "-" + hashlib.sha256(value.encode()).hexdigest()[:8] diff --git a/dokimi/dokimi/records.py b/dokimi/dokimi/records.py index 92678d6a2..a27c2eaf7 100644 --- a/dokimi/dokimi/records.py +++ b/dokimi/dokimi/records.py @@ -121,6 +121,17 @@ def agreement(self) -> Optional[float]: """Panel inter-rater agreement (0-1), when a panel scored it.""" return _number(self.raw.get("agreement_score")) + @property + def incomplete(self) -> bool: + """Whether a panel judge's vote failed or carries no score (``votes`` or ``panel.votes``).""" + panel = self.raw.get("panel") + lists = (self.raw.get("votes"), panel.get("votes") if isinstance(panel, Mapping) else None) + return any( + isinstance(votes, list) and any( + not isinstance(v, Mapping) or v.get("error") or _number(v.get("aggregate")) is None + for v in votes) + for votes in lists) + @dataclass(frozen=True) class Record: diff --git a/dokimi/scripts/check_smoke_runs.py b/dokimi/scripts/check_smoke_runs.py index 48dfe09ce..e444a3d08 100755 --- a/dokimi/scripts/check_smoke_runs.py +++ b/dokimi/scripts/check_smoke_runs.py @@ -4,8 +4,9 @@ """Pass/fail gate for the external-catalog smoke test's ``task_runs.v1`` artifact. -``tasks run`` is a reporting command: its exit code does not say whether the -task passed. This gate does, for ``docker-oracle`` and ``docker-probe``: +``tasks run`` exits 1 for an incomplete run (an errored, unscored or missing +cell), not for failed verification. This gate adds that check, for +``docker-oracle`` and ``docker-probe``: - every cell ran and passed: ``n > 0``, ``passed == n`` and no errored cell. ``pass_rate`` alone is not enough, because an errored cell is left out of its diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index f6e6eadef..312cb39e7 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -8,8 +8,8 @@ "tests/test_arm_runner.py": {"lines": 1175, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, "tests/test_container_agent.py": {"lines": 1235, "rationale": "pre-existing when the gate was introduced"}, - "tests/test_judge_reliability.py": {"lines": 1695, "rationale": "grew on main in #73 (judge-validity --context stored), 1505 -> 1695, before the ratchet landed; split it in its own PR"}, - "tests/test_judge_validity.py": {"lines": 2235, "rationale": "grew on main in #73 (judge-validity --context stored), 1423 -> 2235, before the ratchet landed; split it in its own PR"}, + "tests/test_judge_reliability.py": {"lines": 1694, "rationale": "grew on main in #73 (judge-validity --context stored), 1505 -> 1695, before the ratchet landed; split it in its own PR"}, + "tests/test_judge_validity.py": {"lines": 2234, "rationale": "grew on main in #73 (judge-validity --context stored), 1423 -> 2235, before the ratchet landed; split it in its own PR"}, "tests/test_pipeline_question_set.py": {"lines": 1223}, "tests/test_repo_environment.py": {"lines": 1305, "rationale": "grew on main in #147 before the ratchet landed; +2 for the instructions assert of two crash tests (#103 review); split it in its own PR"}, "tests/test_subjects.py": {"lines": 1047}, diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index f8e037fda..662816a2e 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -105,14 +105,17 @@ --max-atoms --max-bases --max-batch-usd +--max-context-drop # baseline compare score tolerance in points, not a spend cap; moves to the gate policy (#94) --max-cost-usd --max-file-words --max-iterations +--max-lift-drop # baseline compare score tolerance in points, not a spend cap; moves to the gate policy (#94) --max-resource-words --max-tokens --max-turns --metric --min-accuracy +--min-matched # baseline compare gate: no registry concept yet; moves to the gate policy (#94) --mode --multi-run --no-classify diff --git a/dokimi/tests/outcomes/journeys.yaml b/dokimi/tests/outcomes/journeys.yaml index 1a435da4a..a8290287d 100644 --- a/dokimi/tests/outcomes/journeys.yaml +++ b/dokimi/tests/outcomes/journeys.yaml @@ -302,15 +302,14 @@ criteria: invokes: ["repo tasks check --task", "repo validate --json"] stubs: [Docker (FakeRunner)] AC-J07-005: - delivery: partial - enforce: false + delivery: available + enforce: true capability: judge-validity - surfaces: ["eval judge-validity --sequential"] - # The test pins what happens today, which is not what the criterion asks for. + surfaces: ["eval judge-validity --sequential", "eval judge-validity --strict"] evidence: - {id: cjm-judge-cap-inconclusive, kind: unit, stubs: [judge scores (fixed function)]} - gaps: - - "a capped, undecided run above the gate prints RESULT: PASS (inconclusive ...) and exits 0 with a warning; the criterion asks for no pass" + scope: The visible verdict is INCONCLUSIVE; --strict blocks an above-threshold undecided check. + AC-J07-006: delivery: proposed AC-J07-007: @@ -327,13 +326,14 @@ criteria: invokes: ["repo run --reps"] stubs: [Docker (FakeRunner), agent (fake harness executable)] AC-J11-002: - delivery: partial - enforce: false + delivery: available + enforce: true capability: judge-validity + surfaces: ["eval judge-validity --sequential", "eval judge-validity --strict"] evidence: - {id: cjm-judge-cap-inconclusive, kind: unit, stubs: [judge scores (fixed function)]} - gaps: - - "a reached cap is flagged inconclusive (conclusive: false), but the verdict line still reads PASS (inconclusive ...) and the exit code is 0, so a script reading the result sees a pass" + scope: The visible verdict is INCONCLUSIVE; --strict blocks an above-threshold undecided check. + AC-J11-003: delivery: available enforce: true @@ -505,9 +505,16 @@ criteria: # ---------------------------------------------------------------- J12 AC-J12-001: - delivery: proposed - # baseline compare --strict exits 1 only on an unverified or mismatched model; compare --spec finds CRITICAL and exits 0. - gaps: ["baseline compare --strict adds a non-zero exit only on an unverified or mismatched model; a regression beyond a threshold exits 0 and no threshold is defined"] + delivery: available + enforce: true + capability: baseline-regression + surfaces: ["baseline compare --max-context-drop --max-lift-drop"] + evidence: + - id: cjm-baseline-regression + kind: cli_scenario + invokes: ["baseline compare --max-context-drop --max-lift-drop"] + stubs: [] + AC-J12-002: delivery: partial enforce: true @@ -519,7 +526,7 @@ criteria: stubs: [Docker (host probe fake)] - {id: cjm-repo-eval-workflow, kind: unit, stubs: [dokimi (stub executable)]} gaps: - - the policy is one flag for host-unsupported tasks; budget-stopped and inconclusive judge results have none + - Host-unsupported tasks and inconclusive judge checks have explicit flags; a general budget-stop policy remains proposed. AC-J12-003: delivery: available enforce: true @@ -527,8 +534,14 @@ criteria: evidence: - {id: cjm-repo-eval-selection, kind: unit, stubs: [git (temporary repository)]} AC-J12-004: - delivery: proposed - decision_ref: https://github.com/intel/dokimi/pull/161 + delivery: partial + enforce: true + capability: tasks-completion + surfaces: ["tasks run"] + evidence: + - {id: cjm-tasks-completion, kind: unit, stubs: [task result rows]} + scope: Native tasks run rejects missing, unscored and errored cells after saving artifacts. + gaps: ["Other evaluation tracks do not share this completion check."] # ---------------------------------------------------------------- J5, J6, J8 AC-J05-003: diff --git a/dokimi/tests/test_baseline_compare.py b/dokimi/tests/test_baseline_compare.py index 160f10bcd..2555fea52 100644 --- a/dokimi/tests/test_baseline_compare.py +++ b/dokimi/tests/test_baseline_compare.py @@ -271,6 +271,16 @@ def test_an_unknown_answer_model_is_not_a_model_change(compare, capsys): assert "unknown/unknown" not in err and "Baseline model warning" not in err +@pytest.mark.parametrize("role", ["answer", "judge"]) +def test_a_non_string_identity_is_not_recorded_so_the_gate_cannot_pass(compare, capsys, role): + # The eval schema allows a string or null; str(123) == str(123) used to verify the model. + data = _eval(**{role: (123, 456)}) + with pytest.raises(SystemExit) as exc: + compare(data, data, "--max-context-drop", "0") + assert exc.value.code == 1 + assert f"regression gate cannot verify the {role} model" in capsys.readouterr().err + + def test_a_judge_recorded_only_in_run_metadata_is_compared(compare, capsys): old = _eval(judge=("openai", "gpt-4o-mini")) old["run_metadata"].update(judge_provider=old.pop("judge_provider"), @@ -357,3 +367,202 @@ def test_compare_counts_only_questions_scored_on_both_sides(compare, capsys): cur["evaluations"] += extra compare(base, cur) assert "(1 of 3 questions compared)" in capsys.readouterr().out + + +@pytest.mark.outcome("cjm-baseline-regression") +@pytest.mark.parametrize("flag", ["--max-context-drop", "--max-lift-drop"]) +def test_regression_gate_uses_unrounded_scores_not_stored_delta(compare, capsys, flag): + base, cur = _eval(), _eval() + cur["evaluations"][0]["with_docs"]["aggregate"] = 6.999 + with pytest.raises(SystemExit) as exc: + compare(base, cur, flag, "0") + assert exc.value.code == 1 + out = capsys.readouterr().out + assert "REGRESSION" in out and "Comparing against baseline" in out + + +@pytest.mark.parametrize("flag", ["--max-context-drop", "--max-lift-drop"]) +def test_regression_gate_allows_equality(compare, capsys, flag): + base, cur = _eval(), _eval() + cur["evaluations"][0]["with_docs"]["aggregate"] = 6.5 + compare(base, cur, flag, "0.5") + assert "Regression gate: PASS" in capsys.readouterr().out + + +@pytest.mark.parametrize("kind, code", [ + ("empty", 2), ("unmatched", 2), ("unscored", 2), ("nan", 2), ("duplicate", 2), ("unknown", 1), ("mismatch", 1)]) +def test_regression_gate_refuses_incomplete_evidence(compare, capsys, kind, code): + base, cur = _eval(), _eval() + row = cur["evaluations"][0] + if kind == "empty": + cur["evaluations"] = [] + elif kind == "unmatched": + row["question_id"] = "another" + elif kind == "unscored": + row["without_docs"] = None + elif kind == "nan": + row["with_docs"]["aggregate"] = float("nan") + elif kind == "duplicate": + cur["evaluations"].append(dict(row)) + elif kind == "unknown": + cur.pop("run_metadata") + else: + cur["judge_model"] = "other" + with pytest.raises(SystemExit) as exc: + compare(base, cur, "--max-context-drop", "0") + assert exc.value.code == code + assert "Regression gate: PASS" not in capsys.readouterr().out + + +@pytest.mark.parametrize("where", ["votes", "panel"]) +def test_regression_gate_refuses_a_panel_score_with_a_failed_vote(compare, capsys, where): + base, cur = _eval(), _eval() + votes = [{"aggregate": 7.0, "error": None}, {"aggregate": None, "error": "judge failed"}] + block = cur["evaluations"][0]["with_docs"] + if where == "votes": + block["votes"] = votes + else: + block["panel"] = {"votes": votes} + with pytest.raises(SystemExit) as exc: + compare(base, cur, "--max-context-drop", "0") + assert exc.value.code == 2 + assert "Regression gate: PASS" not in capsys.readouterr().out + + +@pytest.mark.parametrize("value", ["nan", "inf", "-1", "oops"]) +def test_regression_threshold_requires_finite_nonnegative_number(value, capsys): + with pytest.raises(SystemExit) as exc: + build_parser().parse_args(["baseline", "compare", "--eval", "x", "--max-lift-drop", value]) + assert exc.value.code == 2 + assert "finite non-negative" in capsys.readouterr().err + + +def test_regression_minimum_matched_count(compare, capsys): + with pytest.raises(SystemExit) as exc: + compare(_eval(), _eval(), "--max-context-drop", "0", "--min-matched", "2") + assert exc.value.code == 2 + assert "insufficient" in capsys.readouterr().err + + +def test_min_matched_without_a_drop_gate_is_a_usage_error(compare, capsys): + with pytest.raises(SystemExit) as exc: + compare(_eval(), _eval(), "--min-matched", "20") + assert exc.value.code == 2 + assert "--min-matched needs --max-context-drop or --max-lift-drop" in capsys.readouterr().err + + +@pytest.mark.parametrize("fails, flags", [ + ("context", ("--max-context-drop", "0", "--max-lift-drop", "10")), + ("lift", ("--max-context-drop", "10", "--max-lift-drop", "0")), +]) +def test_either_gate_failing_is_a_regression(compare, capsys, fails, flags): + base, cur = _eval(), _eval() + block = cur["evaluations"][0] + if fails == "context": + block["with_docs"]["aggregate"] = 5.0 + block["without_docs"]["aggregate"] = 0.0 + else: + block["without_docs"]["aggregate"] = 6.0 + with pytest.raises(SystemExit) as exc: + compare(base, cur, *flags) + assert exc.value.code == 1 + assert "Regression gate: REGRESSION" in capsys.readouterr().out + + +def test_strict_remains_identity_only_even_for_large_regression(compare, capsys): + cur = _eval(delta=-5) + cur["evaluations"][0]["with_docs"]["aggregate"] = 0 + compare(_eval(), cur, "--strict") + assert "Regression gate" not in capsys.readouterr().out + + +@pytest.mark.parametrize("score, limit", [(6.9, "0.1"), (6.8, "0.2"), (6.7, "0.3")]) +def test_decimal_threshold_equality_passes(compare, capsys, score, limit): + base, cur = _eval(), _eval() + cur["evaluations"][0]["with_docs"]["aggregate"] = score + compare(base, cur, "--max-context-drop", limit, "--max-lift-drop", limit) + assert "Regression gate: PASS" in capsys.readouterr().out + + +@pytest.mark.parametrize("flag, fails", [("--max-context-drop", False), ("--max-lift-drop", True)]) +def test_context_score_and_lift_are_distinct_metrics(compare, capsys, flag, fails): + base, cur = _eval(), _eval() + # Context quality is unchanged, but the unaided model improved: lift fell. + cur["evaluations"][0]["without_docs"]["aggregate"] = 6.0 + if fails: + with pytest.raises(SystemExit) as exc: + compare(base, cur, flag, "0") + assert exc.value.code == 1 + assert "paired lift: drop=1" in capsys.readouterr().out + else: + compare(base, cur, flag, "0") + assert "Regression gate: PASS" in capsys.readouterr().out + + +@pytest.mark.parametrize("side", ["base", "current"]) +def test_gate_refuses_partial_question_coverage(compare, capsys, side): + base, cur = _eval(), _eval() + data = base if side == "base" else cur + data["evaluations"].append({**data["evaluations"][0], "question_id": "extra"}) + with pytest.raises(SystemExit) as exc: + compare(base, cur, "--max-lift-drop", "100") + assert exc.value.code == 2 + assert "complete unique coverage required" in capsys.readouterr().err + + +@pytest.mark.outcome("cjm-baseline-regression") +def test_gate_matches_questions_by_id_not_list_position(compare, capsys): + base, cur = _eval(), _eval() + base["evaluations"].append({**_eval(delta=3)["evaluations"][0], "question_id": "q2"}) + cur["evaluations"] = list(reversed(base["evaluations"])) + compare(base, cur, "--max-context-drop", "0", "--max-lift-drop", "0", "--min-matched", "2") + out = capsys.readouterr().out + assert "(2 matched)" in out and "Regression gate: PASS" in out + + +@pytest.mark.parametrize("model", ["unknown/unknown", "openai/ Unknown ", "bare-model", "openai/openai/unknown"]) +def test_regression_gate_refuses_unknown_panel_members(compare, capsys, model): + with pytest.raises(SystemExit) as exc: + compare(_panel(model), _panel(model), "--max-context-drop", "0") + assert exc.value.code == 1 + assert "cannot verify the judge model" in capsys.readouterr().err + + +def _scores(*context: float) -> dict: + data = _eval() + data["evaluations"] = [{"question_id": f"q{i}", "with_docs": {"aggregate": score}, + "without_docs": {"aggregate": 0.0}, "delta": score} + for i, score in enumerate(context)] + return data + + +@pytest.mark.parametrize("base_scores, cur_scores", [((5.0, 5.0), (4.0, 4.0)), ((5.0,), (9.0,))]) +def test_regression_gate_compares_the_mean_drop_and_ignores_improvement( + compare, capsys, base_scores, cur_scores): + compare(_scores(*base_scores), _scores(*cur_scores), "--max-context-drop", "1") + out = capsys.readouterr().out + assert "Regression gate: PASS" in out + + +def test_regression_gate_needs_at_least_one_matched_question_by_default(compare, capsys): + with pytest.raises(SystemExit) as exc: + compare(_scores(), _scores(), "--max-context-drop", "0") + assert exc.value.code == 2 + assert "insufficient" in capsys.readouterr().err + + +@pytest.mark.parametrize("flag", ["--max-context-drop", "--max-lift-drop"]) +def test_regression_gate_threshold_keeps_every_digit_of_an_equal_drop(compare, capsys, flag): + compare(_scores(0.2), _scores(0.09999999999999999), flag, "0.10000000000000001") + assert "Regression gate: PASS" in capsys.readouterr().out + + +@pytest.mark.parametrize("judge, answer", [ + (("openai", "openai/unknown"), ("openai", "gpt-6.1-sol")), + (("anthropic", "claude-sonnet-5-5"), ("openai", "openai/unknown")), +]) +def test_regression_gate_refuses_an_unknown_model_behind_a_route(compare, capsys, judge, answer): + with pytest.raises(SystemExit) as exc: + compare(_eval(answer=answer, judge=judge), _eval(answer=answer, judge=judge), "--max-context-drop", "0") + assert exc.value.code == 1 + assert "cannot verify" in capsys.readouterr().err diff --git a/dokimi/tests/test_harbor_results.py b/dokimi/tests/test_harbor_results.py index e58aea93f..07710b233 100644 --- a/dokimi/tests/test_harbor_results.py +++ b/dokimi/tests/test_harbor_results.py @@ -311,10 +311,70 @@ def test_a_command_template_keeps_its_exit_code_fallback(tmp_path): result = create_harness("command:my-wrapper {task_path}", runner=FakeRunner()).run( task, output_dir=tmp_path / "cell") - assert (result.passed, result.reward) == (True, None) + assert (result.passed, result.reward) == (True, 1.0) assert "error" not in result.metrics +@pytest.mark.parametrize("returncode, expected", [(0, (True, 1.0)), (3, (False, 0.0))]) +def test_a_command_template_scores_its_exit_code_when_it_prints_no_reward(tmp_path, returncode, expected): + task = load_task(_fixture_task(tmp_path / "tasks")) + result = create_harness("command:my-wrapper {task_path}", runner=FakeRunner(lambda call: reply("", "", returncode))).run( + task, output_dir=tmp_path / "cell") + + assert (result.passed, result.reward) == expected + + +@pytest.mark.parametrize("stdout, stderr", [("", ""), ("reward: 1", ""), ("", "reward: 1"), ("1", "reward: 1")]) +@pytest.mark.parametrize("returncode", [126, 127, 137, -9, 3221225477, 3221225786, 3221226505]) +def test_a_command_that_never_ran_or_was_killed_has_no_verdict(tmp_path, returncode, stdout, stderr): + """Nor from a reward marker on its output: only a reward file outranks such an exit.""" + task = load_task(_fixture_task(tmp_path / "tasks")) + result = create_harness("command:my-wrapper {task_path}", runner=FakeRunner(lambda call: reply(stdout, stderr, returncode))).run( + task, output_dir=tmp_path / "cell") + + assert (result.passed, result.reward) == (False, None) + assert result.metrics["solver_error"] == f"no verdict: exit {returncode}" + _assert_errored_not_scored(result) + + +def test_a_reward_file_is_the_verdict_of_a_killed_command(tmp_path): + task = load_task(_fixture_task(tmp_path / "tasks")) + + def handler(call): + (Path(call.env["DOKIMI_OUTPUT_DIR"]) / "reward.txt").write_text("1", encoding="utf-8") + return reply("reward: 0", "", 137) + + result = create_harness("command:my-wrapper {task_path}", runner=FakeRunner(handler)).run( + task, output_dir=tmp_path / "cell") + + assert (result.passed, result.reward) == (True, 1.0) + assert "solver_error" not in result.metrics + + +def _assert_errored_not_scored(result): + from dokimi.harnesses.runner import summarize_task_results + + stats = summarize_task_results([result.as_dict()])["per_harness"][result.harness] + assert (stats["scored"], stats["errored"], stats["pass_rate"]) == (0, 1, None) + + +@pytest.mark.parametrize("content, output", [("2.0", ""), ("2.0", "reward: 1"), ("", "reward: 1")]) +def test_a_command_with_an_unreadable_reward_file_is_not_scored_from_its_exit_code(tmp_path, content, output): + """Nor from a reward marker on its output: the file, not the agent's log, is the verdict.""" + task = load_task(_fixture_task(tmp_path / "tasks")) + + def handler(call): + (Path(call.env["DOKIMI_OUTPUT_DIR"]) / "reward.txt").write_text(content, encoding="utf-8") + return reply(output) + + result = create_harness("command:my-wrapper {task_path}", runner=FakeRunner(handler)).run( + task, output_dir=tmp_path / "cell") + + assert (result.passed, result.reward) == (False, None) + assert "reward file has no reward" in result.metrics["solver_error"] + _assert_errored_not_scored(result) + + @pytest.mark.parametrize("reward_val, expect_pass, expect_reward", [ (float("inf"), False, None), # Non-finite: error (float("nan"), False, None), # Non-finite: error diff --git a/dokimi/tests/test_judge_reliability.py b/dokimi/tests/test_judge_reliability.py index f4ef68905..c3c15a1b6 100644 --- a/dokimi/tests/test_judge_reliability.py +++ b/dokimi/tests/test_judge_reliability.py @@ -97,11 +97,10 @@ def at_the_gate(pair): # right on 4 pairs of 5: SPRT never decides before the c seq = s["per_variant"]["truncate_20"]["sequential"] assert seq["decision"] == "continue" and seq["stopped_by"] == "cap" assert seq["n"] == 20 # the cap was reached, not an early decision - # Above the gate but undecided is an inconclusive pass (exit 0 with a warning), never a - # conclusive one: the verdict is flagged inconclusive and the report says so. + # An undecided sample is visibly INCONCLUSIVE, even above the point-estimate gate. assert s["conclusive"] is False summary = report.format_summary() - assert "RESULT: PASS (inconclusive" in summary and "answer pool exhausted" not in summary + assert "RESULT: INCONCLUSIVE" in summary and "answer pool exhausted" not in summary def test_sequential_says_when_the_pool_ran_out_before_the_cap(): diff --git a/dokimi/tests/test_judge_validity.py b/dokimi/tests/test_judge_validity.py index 996219644..0155899aa 100644 --- a/dokimi/tests/test_judge_validity.py +++ b/dokimi/tests/test_judge_validity.py @@ -685,7 +685,7 @@ def test_a_clear_pass_is_conclusive(self): def test_inconclusive_is_stated_in_the_summary_text(self): text = self._report(20, 3).format_summary() assert "INCONCLUSIVE" in text - assert "RESULT: PASS (inconclusive" in text + assert "RESULT: INCONCLUSIVE" in text def test_interval_narrows_as_pairs_are_added(self): def half_width(n): @@ -793,7 +793,7 @@ def test_a_fail_one_variant_settles_is_conclusive(self): def test_a_fail_that_only_straddles_stays_inconclusive(self): summary = self._report(20, 5).summary() assert (summary["passed"], summary["conclusive"]) == (False, False) - assert self._report(20, 5).format_summary().endswith("RESULT: FAIL (inconclusive — see above)") + assert self._report(20, 5).format_summary().endswith("RESULT: FAIL (INCONCLUSIVE, see above)") def test_an_accuracy_on_the_gate_does_not_say_more_pairs_cannot_help(self): summary = self._report(20, 4).summary() @@ -971,18 +971,17 @@ def _run(tmp_path, monkeypatch, capsys, bases, *extra, sign=1.0): code = exc.code return code, capsys.readouterr().err - def test_an_inconclusive_pass_warns_and_exits_zero(self, tmp_path, monkeypatch, capsys): + @pytest.mark.outcome("cjm-judge-cap-inconclusive") + def test_an_inconclusive_pass_warns_and_exits_zero_unless_strict(self, tmp_path, monkeypatch, capsys): code, err = self._run(tmp_path, monkeypatch, capsys, 5) assert code == 0 - assert "Pass is inconclusive at this sample size: truncate_20" in err + assert "INCONCLUSIVE at this sample size: truncate_20" in err + assert self._run(tmp_path, monkeypatch, capsys, 5, "--strict")[0] == 1 def test_the_hint_names_the_knob_the_mode_reads(self, tmp_path, monkeypatch, capsys): assert "more --bases" in self._run(tmp_path, monkeypatch, capsys, 5)[1] err = self._run(tmp_path, monkeypatch, capsys, 5, "--sequential")[1] - assert "inconclusive" in err and "--max-bases" in err and "more --bases" not in err - - def test_strict_turns_an_inconclusive_pass_into_a_failure(self, tmp_path, monkeypatch, capsys): - assert self._run(tmp_path, monkeypatch, capsys, 5, "--strict")[0] == 1 + assert "INCONCLUSIVE" in err and "--max-bases" in err and "more --bases" not in err def test_strict_keeps_a_settled_pass(self, tmp_path, monkeypatch, capsys): assert self._run(tmp_path, monkeypatch, capsys, 20, "--strict") == (0, "") @@ -1120,7 +1119,7 @@ def test_a_requested_variant_with_no_pair_is_reported_and_blocks_a_conclusive_pa assert s["conclusive"] is False text = report.format_summary() assert "strip_code" in text and "not measured: no pair, 20 no-op" in text - assert "RESULT: PASS (inconclusive" in text + assert "RESULT: INCONCLUSIVE" in text def test_table_cells_sit_under_their_headers(): diff --git a/dokimi/tests/test_tasks_command.py b/dokimi/tests/test_tasks_command.py index 9145951b4..6465fa9d5 100644 --- a/dokimi/tests/test_tasks_command.py +++ b/dokimi/tests/test_tasks_command.py @@ -187,7 +187,7 @@ def _run_tasks(tmp_path, monkeypatch, harness_for): args = build_parser().parse_args([ "tasks", "run", "--tasks", "toy-task", "--tasks-root", str(tmp_path), "--harnesses", "codex", "--baseline-harness", "claude-code", "--model", "m", - "--out-json", str(tmp_path / "runs.json"), "--dry-run", + "--out-json", str(tmp_path / "runs.json"), "--dry-run", "--output-dir", str(tmp_path / "task-runs"), ]) try: args.func(args) @@ -202,15 +202,164 @@ def test_tasks_run_with_every_cell_errored_is_incomplete(tmp_path, monkeypatch, out, err = capsys.readouterr() assert code == 1 assert "codex: 0/0 pass (n/a), 1 errored" in out - assert "incomplete: every cell errored (0 scored, 2 errored of 2 cells)" in err + assert "incomplete: 0 scored, 2 errored, 0 unscored, 0 missing of 2 planned cells" in err assert (tmp_path / "runs.json").is_file() # the artifact is still written -def test_tasks_run_with_some_cells_errored_keeps_exit_0_and_counts(tmp_path, monkeypatch, capsys): +def test_tasks_run_with_some_cells_errored_is_incomplete(tmp_path, monkeypatch, capsys): code = _run_tasks(tmp_path, monkeypatch, lambda hid: _RaisingHarness() if hid == "codex" else None) out, err = capsys.readouterr() - assert code == 0 - assert "codex: 0/0 pass (n/a), 1 errored" in out and "incomplete" not in err + assert code == 1 + assert "codex: 0/0 pass (n/a), 1 errored" in out and "incomplete" in err + + +@pytest.mark.parametrize("reward, expected", [(1.0, 0), (0.0, 0), (None, 1)]) +def test_native_completion_preserves_grade_failures(tmp_path, monkeypatch, capsys, reward, expected): + from dokimi.harnesses.models import HarnessResult + + class Harness: + def __init__(self, hid): + self.hid = hid + + def run(self, task, **kwargs): + return HarnessResult(self.hid, task.name, [], 0, 0, reward, reward == 1.0) + + code = _run_tasks(tmp_path, monkeypatch, lambda hid: Harness(hid)) + assert code == expected + assert (tmp_path / "runs.json").exists() + if reward is None: + assert "2 unscored" in capsys.readouterr().err + + +@pytest.mark.outcome("cjm-tasks-completion") +@pytest.mark.parametrize("rows", [[], [{"task": "t", "harness": "h", "reward": 1.0}]]) +def test_task_completion_refuses_missing_planned_cells(rows, capsys): + from dokimi.commands.tasks import _check_completion + + output = {"tasks": [{"name": "t"}], "harnesses": ["h"], "repeats": 2, "results": rows} + with pytest.raises(SystemExit) as exc: + _check_completion(output) + assert exc.value.code == 1 + assert "missing" in capsys.readouterr().err + + +@pytest.mark.parametrize("script, expected", [("pass", 0), ("raise SystemExit(1)", 0)]) +def test_command_harness_exit_code_verdict_is_a_complete_run(tmp_path, capsys, script, expected): + import sys + + _fixture_task(tmp_path) + harness = f'command:"{Path(sys.executable).as_posix()}" -c "{script}"' + args = build_parser().parse_args([ + "tasks", "run", "--tasks", "toy-task", "--tasks-root", str(tmp_path), + "--harnesses", harness, "--out-json", str(tmp_path / "runs.json"), + "--output-dir", str(tmp_path / "task-runs"), + ]) + try: + args.func(args) + code = 0 + except SystemExit as exc: + code = exc.code + assert code == expected, capsys.readouterr().err + + +@pytest.mark.parametrize("reward", [float("nan"), float("inf"), True]) +def test_task_completion_refuses_a_reward_that_is_not_a_number(reward, capsys): + from dokimi.commands.tasks import _check_completion + + row = {"task": "t", "harness": "h", "reward": reward} + with pytest.raises(SystemExit) as exc: + _check_completion({"tasks": [{"name": "t"}], "harnesses": ["h"], "results": [row]}) + assert exc.value.code == 1 + assert "1 unscored" in capsys.readouterr().err + + +def test_task_completion_accepts_every_repeat_of_a_complete_grid(): + from dokimi.commands.tasks import _check_completion + + rows = [{"task": "t", "harness": "h", "repeat": n, "reward": 1.0} for n in (1, 2)] + _check_completion({"tasks": [{"name": "t"}], "harnesses": ["h"], "repeats": 2, "results": rows}) + + +@pytest.mark.parametrize("flag, value, repeated", [ + ("--harnesses", "command:true,command:true", "command:true"), + ("--tasks", "toy-task,toy-task", "toy-task"), + ("--tasks", "toy-task,{root}/toy-task", "toy-task"), + ("--tasks", "toy-task,toy-task/", "toy-task"), + ("--tasks", "{root}/toy-task,{root}/other/toy-task", "toy-task"), +]) +def test_tasks_run_refuses_a_repeated_task_or_harness_before_running(tmp_path, capsys, flag, value, repeated): + _fixture_task(tmp_path) + (tmp_path / "other").mkdir() + _fixture_task(tmp_path / "other") + value = value.format(root=tmp_path) + argv = ["tasks", "run", "--tasks", "toy-task", "--tasks-root", str(tmp_path), + "--harnesses", "command:true", "--out-json", str(tmp_path / "runs.json")] + argv[argv.index(flag) + 1] = value + args = build_parser().parse_args(argv) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert repeated in capsys.readouterr().err + assert not (tmp_path / "runs.json").exists() + + +@pytest.mark.outcome("cjm-tasks-completion") +def test_task_completion_refuses_an_empty_plan(capsys): + from dokimi.commands.tasks import _check_completion + + with pytest.raises(SystemExit) as exc: + _check_completion({"tasks": [], "harnesses": [], "results": []}) + assert exc.value.code == 1 + assert "0 planned cells" in capsys.readouterr().err + + +@pytest.mark.parametrize("extra", ["duplicate", "unplanned"]) +def test_task_completion_refuses_ambiguous_cell_identity(extra, capsys): + from dokimi.commands.tasks import _check_completion + + row = {"task": "t", "harness": "h", "reward": 1.0} + other = dict(row, task="other") if extra == "unplanned" else dict(row) + output = {"tasks": [{"name": "t"}], "harnesses": ["h"], "results": [row, other]} + with pytest.raises(SystemExit) as exc: + _check_completion(output) + assert exc.value.code == 1 + assert extra in capsys.readouterr().err + + +def test_unsupported_dry_run_preserves_artifacts(tmp_path, capsys): + task = _fixture_task(tmp_path) + (task / "tests" / "test.sh").unlink() + out = tmp_path / "plan.json" + args = build_parser().parse_args([ + "tasks", "run", "--tasks", "toy-task", "--tasks-root", str(tmp_path), + "--harnesses", "codex", "--dry-run", "--out-json", str(out), "--output-dir", str(tmp_path / "task-runs"), + ]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + assert "1 errored" in capsys.readouterr().err + assert out.is_file() and out.with_suffix(".md").is_file() + + +@pytest.mark.outcome("cjm-tasks-completion") +def test_partial_suite_writes_remaining_results_before_refusing(tmp_path, monkeypatch, capsys): + import json + from dokimi.harnesses import TaskSuiteRunner + + real_run = TaskSuiteRunner.run + + def partial_run(self): + output = real_run(self) + output["results"].pop() + return output + + monkeypatch.setattr(TaskSuiteRunner, "run", partial_run) + code = _run_tasks(tmp_path, monkeypatch, lambda hid: None) + assert code == 1 + assert "missing: toy-task / codex / repeat 1" in capsys.readouterr().err + data = json.loads((tmp_path / "runs.json").read_text()) + assert len(data["results"]) == 1 and data["results"][0]["harness"] == "claude-code" + assert (tmp_path / "runs.md").is_file() def test_tasks_run_with_a_missing_task_exits_with_a_message(tmp_path): @@ -261,3 +410,12 @@ def test_tasks_inspect_of_a_missing_path_exits_with_an_error_prefix(tmp_path): with pytest.raises(SystemExit) as excinfo: args.func(args) assert str(excinfo.value).startswith("Error:") + + +def test_a_harness_id_with_characters_windows_refuses_makes_a_valid_cell_directory_name(): + from dokimi.harnesses.runner import _safe_id + + name = _safe_id(r'command:"C:\py\python.exe" -c "a|b?*"') + assert not any(c in name for c in '"\\<>|?*:/ ') + assert _safe_id('command:echo "a?"') != _safe_id('command:echo "a*"') + assert _safe_id("command:my-wrapper {task_path}") == "command_my-wrapper_{task_path}"