fix(catalog): trigger-probe exits 2 on a set with no labelled query, warns when no skill was listed

dokimi/changelog.d/20261028_120000_antonwolfy_trigger-probe-empty.md

@@ -0,0 +1,11 @@
+### Fixed
+
+- **`catalog trigger-probe` no longer reports an empty probe as a run** — a
+  `--trigger-set` with no labelled query (a questions set without `expected_skill`,
+  or an empty set) ran nothing, wrote `"answers": []`, printed `"summary": {}` and
+  exited 0. It now exits 2 with `Error: --trigger-set has no labelled queries`
+  before any run, and writes no `--out` file. When runs happened but Claude Code
+  listed no skills in any of them (an older Claude Code, or every run failing before
+  its init event), `not_listed` stayed `[]` without a word; a warning on stderr now
+  says `not_listed` was not checked. The output JSON and the exit code of that case
+  are unchanged.

dokimi/docs/evaluating-treatments.md

@@ -593,7 +593,9 @@ left on as a user has them (`--sealed` turns them off). The skill chosen is read
 transcript's `Skill` calls, and the output's `skill_selection_summary` is scored by the
 same code as the `skill-catalog:` arm, one column per trial. `max_turns_hit` counts runs
 that ended on `--max-turns` (scored as no load), and the command exits 1 when Claude Code
-did not list a catalog skill (`not_listed`). Records are kept in `<out>.partial` while the
+did not list a catalog skill (`not_listed`). When no run listed any skill, `not_listed`
+stays empty and a warning on stderr says it was not checked. A trigger set with no
+labelled query exits 2 before any run. Records are kept in `<out>.partial` while the
 probe runs:
 
 ```bash

dokimi/dokimi/commands/catalog_probe.py

@@ -65,6 +65,9 @@ def one(q):
         records = list(pool.map(one, queries))
     runs = [v for r in records for v in r["arms"].values()]
     listed = {s.rsplit(":", 1)[-1] for v in runs for s in v["tool_usage"]["skill"]["listed"]}
+    if runs and not listed:  # an older Claude Code, or every run failed before its init event
+        print("warning: Claude Code listed no skills in any run; not_listed was not checked",
+              file=sys.stderr)
     # A skill the harness never offered (a frontmatter name that is not its directory's,
     # a nested layout) reads as a 0% recall; say so instead of scoring it.
     not_listed = sorted(names - listed) if listed else []
@@ -108,6 +111,9 @@ def _labelled_queries(args: argparse.Namespace, names: set) -> list:
     except (OSError, ValueError) as exc:
         raise SystemExit(f"--trigger-set: {exc}") from exc
     queries = [q for q in queries if q.labelled][: args.limit or None]
+    if not queries:  # else nothing runs, and an empty summary reads as a probe that ran
+        raise InputError(f"--trigger-set has no labelled queries: {args.trigger_set}"
+                         + (f" (read with --target {args.target!r})" if args.target else ""))
     # As the skill-catalog: arm refuses it: a label no run can load scores as a miss.
     unknown = sorted({s for q in queries for s in q.acceptable} - names)
     if unknown:

dokimi/tests/test_trigger_probe.py

@@ -377,6 +377,41 @@ def test_cli_refuses_a_label_that_names_no_catalog_skill(tmp_path, monkeypatch):
     _cli(tmp_path, monkeypatch, "--out", "probe.json", expected=["sum-fast", "io-read"])
 
 
+@pytest.mark.parametrize("trigger_set, extra", [
+    ({"questions": [{"id": "a", "question": "q a"}]}, ()),
+    ({"questions": [{"id": "a", "question": "q a"}]}, ("--target", "sum-fast")),
+    ({"queries": []}, ()),
+], ids=["unlabelled", "unlabelled-with-target", "empty-reach-set"])
+def test_cli_exits_2_on_a_set_with_no_labelled_query_before_any_run(tmp_path, monkeypatch, capsys,
+                                                                    trigger_set, extra):
+    from dokimi import cli
+
+    runs: list = []
+    monkeypatch.setattr(tp, "probe_trigger_set", lambda queries, *a, **kw: runs.append(queries) or [])
+    _catalog(tmp_path)
+    (tmp_path / "ts.json").write_text(json.dumps(trigger_set), encoding="utf-8")
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr("sys.argv", ["dokimi", "catalog", "trigger-probe", "--catalog", "cat", "--trigger-set",
+                                     "ts.json", "--claude", "sh", "--out", "probe.json", *extra])
+    with pytest.raises(SystemExit) as exc:
+        cli.main()
+    assert exc.value.code == 2
+    err = capsys.readouterr().err
+    assert err.startswith("Error: --trigger-set has no labelled queries: ts.json")
+    assert ("--target 'sum-fast'" in err) == bool(extra)
+    assert runs == [] and not (tmp_path / "probe.json").exists()
+    assert not (tmp_path / "probe.json.partial").exists()
+
+
+def test_cli_warns_when_no_run_listed_any_skill(tmp_path, monkeypatch, capsys):
+    warning = "warning: Claude Code listed no skills in any run; not_listed was not checked"
+    _cli(tmp_path, monkeypatch, "--out", "probe.json", init_skills=())
+    assert warning in capsys.readouterr().err
+    assert json.loads((tmp_path / "probe.json").read_text(encoding="utf-8"))["not_listed"] == []
+    _cli(tmp_path, monkeypatch, "--out", "probe.json")
+    assert "warning" not in capsys.readouterr().err
+
+
 def test_cli_fails_when_claude_code_did_not_list_a_catalog_skill(tmp_path, monkeypatch):
     with pytest.raises(SystemExit, match="did not list: io-read"):
         _cli(tmp_path, monkeypatch, "--out", "probe.json", init_skills=("sum-fast", "dataviz"))