diff --git a/dokimi/changelog.d/20261028_120000_antonwolfy_trigger-probe-empty.md b/dokimi/changelog.d/20261028_120000_antonwolfy_trigger-probe-empty.md new file mode 100644 index 000000000..a8e7b665a --- /dev/null +++ b/dokimi/changelog.d/20261028_120000_antonwolfy_trigger-probe-empty.md @@ -0,0 +1,11 @@ +### Fixed + +- **`catalog trigger-probe` no longer reports an empty probe as a run** — a + `--trigger-set` with no labelled query (a questions set without `expected_skill`, + or an empty set) ran nothing, wrote `"answers": []`, printed `"summary": {}` and + exited 0. It now exits 2 with `Error: --trigger-set has no labelled queries` + before any run, and writes no `--out` file. When runs happened but Claude Code + listed no skills in any of them (an older Claude Code, or every run failing before + its init event), `not_listed` stayed `[]` without a word; a warning on stderr now + says `not_listed` was not checked. The output JSON and the exit code of that case + are unchanged. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 083cbf9aa..691ab695a 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -593,7 +593,9 @@ left on as a user has them (`--sealed` turns them off). The skill chosen is read transcript's `Skill` calls, and the output's `skill_selection_summary` is scored by the same code as the `skill-catalog:` arm, one column per trial. `max_turns_hit` counts runs that ended on `--max-turns` (scored as no load), and the command exits 1 when Claude Code -did not list a catalog skill (`not_listed`). Records are kept in `.partial` while the +did not list a catalog skill (`not_listed`). When no run listed any skill, `not_listed` +stays empty and a warning on stderr says it was not checked. A trigger set with no +labelled query exits 2 before any run. Records are kept in `.partial` while the probe runs: ```bash diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index b149613b5..abecfcdda 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -65,6 +65,9 @@ def one(q): records = list(pool.map(one, queries)) runs = [v for r in records for v in r["arms"].values()] listed = {s.rsplit(":", 1)[-1] for v in runs for s in v["tool_usage"]["skill"]["listed"]} + if runs and not listed: # an older Claude Code, or every run failed before its init event + print("warning: Claude Code listed no skills in any run; not_listed was not checked", + file=sys.stderr) # A skill the harness never offered (a frontmatter name that is not its directory's, # a nested layout) reads as a 0% recall; say so instead of scoring it. not_listed = sorted(names - listed) if listed else [] @@ -108,6 +111,9 @@ def _labelled_queries(args: argparse.Namespace, names: set) -> list: except (OSError, ValueError) as exc: raise SystemExit(f"--trigger-set: {exc}") from exc queries = [q for q in queries if q.labelled][: args.limit or None] + if not queries: # else nothing runs, and an empty summary reads as a probe that ran + raise InputError(f"--trigger-set has no labelled queries: {args.trigger_set}" + + (f" (read with --target {args.target!r})" if args.target else "")) # As the skill-catalog: arm refuses it: a label no run can load scores as a miss. unknown = sorted({s for q in queries for s in q.acceptable} - names) if unknown: diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 041789890..81bad7980 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -377,6 +377,41 @@ def test_cli_refuses_a_label_that_names_no_catalog_skill(tmp_path, monkeypatch): _cli(tmp_path, monkeypatch, "--out", "probe.json", expected=["sum-fast", "io-read"]) +@pytest.mark.parametrize("trigger_set, extra", [ + ({"questions": [{"id": "a", "question": "q a"}]}, ()), + ({"questions": [{"id": "a", "question": "q a"}]}, ("--target", "sum-fast")), + ({"queries": []}, ()), +], ids=["unlabelled", "unlabelled-with-target", "empty-reach-set"]) +def test_cli_exits_2_on_a_set_with_no_labelled_query_before_any_run(tmp_path, monkeypatch, capsys, + trigger_set, extra): + from dokimi import cli + + runs: list = [] + monkeypatch.setattr(tp, "probe_trigger_set", lambda queries, *a, **kw: runs.append(queries) or []) + _catalog(tmp_path) + (tmp_path / "ts.json").write_text(json.dumps(trigger_set), encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("sys.argv", ["dokimi", "catalog", "trigger-probe", "--catalog", "cat", "--trigger-set", + "ts.json", "--claude", "sh", "--out", "probe.json", *extra]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 2 + err = capsys.readouterr().err + assert err.startswith("Error: --trigger-set has no labelled queries: ts.json") + assert ("--target 'sum-fast'" in err) == bool(extra) + assert runs == [] and not (tmp_path / "probe.json").exists() + assert not (tmp_path / "probe.json.partial").exists() + + +def test_cli_warns_when_no_run_listed_any_skill(tmp_path, monkeypatch, capsys): + warning = "warning: Claude Code listed no skills in any run; not_listed was not checked" + _cli(tmp_path, monkeypatch, "--out", "probe.json", init_skills=()) + assert warning in capsys.readouterr().err + assert json.loads((tmp_path / "probe.json").read_text(encoding="utf-8"))["not_listed"] == [] + _cli(tmp_path, monkeypatch, "--out", "probe.json") + assert "warning" not in capsys.readouterr().err + + def test_cli_fails_when_claude_code_did_not_list_a_catalog_skill(tmp_path, monkeypatch): with pytest.raises(SystemExit, match="did not list: io-read"): _cli(tmp_path, monkeypatch, "--out", "probe.json", init_skills=("sum-fast", "dataviz"))