From 8bc08b7119097d04bb138575500e7b92ef3b3e09 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 11:33:30 +0000 Subject: [PATCH 1/2] test(novelty): pin the all-vs-all seed, console cap and name tie-break The review of #172 left three outputs of `novelty routing --all-vs-all` unpinned: the full suite passed with `seed=0` hard-coded in `_cmd_routing_all_vs_all`, with the console's top-five cap cut to three, and with the name tie-break of attractors (and of an attractor's `from` list) reversed. - A CLI run with a mocked selector and `--seed 5` asserts that the saved model arm records seed 5. - A field where six skills take another skill's queries, six down to one, asserts the exact five attractor lines the console prints for each arm. - Two attractors tied on count, listed in the field against name order, and an attractor whose two golds tie, assert the exact attractor rows: name order in `attractors` and in `from`. Each test fails under the change it pins. Signed-off-by: Anton Volkov --- .../tests/test_novelty_routing_all_vs_all.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/dokimi/tests/test_novelty_routing_all_vs_all.py b/dokimi/tests/test_novelty_routing_all_vs_all.py index 2c3f5b2e6..f5735b6f2 100644 --- a/dokimi/tests/test_novelty_routing_all_vs_all.py +++ b/dokimi/tests/test_novelty_routing_all_vs_all.py @@ -740,3 +740,67 @@ def transport(prompt: str, system: str | None): with pytest.raises(SystemExit, match="^1 of 1 model call"): args.func(args) assert "makes 1 model calls" in capsys.readouterr().err + + +def _word_client(model, provider, role, args): + """A selector that picks the card holding the request's word, wherever it is shuffled.""" + def transport(prompt: str, system: str | None): + request, cards = prompt.split("REQUEST:", 1)[1].split("TOOLS:", 1) + lines = cards.strip().splitlines() + return json.dumps({"pick": next(i + 1 for i, line in enumerate(lines) + if request.strip() in line)}), None + return LLMClient(model=model, provider=provider, role=role, transport=transport) + + +def test_the_seed_on_the_command_line_is_the_seed_the_saved_model_arm_records( + tmp_path, monkeypatch): + import dokimi.commands.novelty as novelty_cmd + + monkeypatch.setattr(novelty_cmd, "_client", _word_client) + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary", "db": "tune a slow database query"}) + queries = _write_queries(tmp_path, {"perf": ["profile"], "db": ["database"]}) + out = tmp_path / "o.json" + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(out), "--model", "m", + "--seed", "5") + args.func(args) + model = json.loads(out.read_text(encoding="utf-8"))["result"]["model"] + assert (model["model"], model["seed"]) == ("m", 5) + + +def test_console_prints_the_top_five_attractors_of_each_arm_by_count( + tmp_path, capsys, monkeypatch): + # Each of six skills takes the gold queries that name its word: alpha one, ... foxtrot six. + import dokimi.commands.novelty as novelty_cmd + + monkeypatch.setattr(novelty_cmd, "_client", _word_client) + words = {"alpha": "anvil", "bravo": "barley", "charlie": "cobalt", "delta": "dune", + "echo": "ember", "foxtrot": "fjord"} + root = tmp_path / "catalog" + _cat(root, {**words, "gold": "something unrelated entirely"}) + queries = _write_queries(tmp_path, {"gold": [word for count, word in + enumerate(words.values(), 1) + for _ in range(count)]}) + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(tmp_path / "o.json"), + "--model", "m") + args.func(args) + lines = [line for line in capsys.readouterr().out.splitlines() if "sibling" in line] + assert lines == [f"{arm}: {name} picked for {count} of 21 sibling queries (from gold)" + for arm in ("lexical", "model") + for name, count in [("foxtrot", 6), ("echo", 5), ("delta", 4), + ("charlie", 3), ("bravo", 2)]] + + +def test_attractors_tied_on_count_and_golds_tied_in_from_are_in_name_order(): + # zeta comes first in the field and mu first in the queries: name order has to win. + field = [Candidate("zeta", "zebra stripes"), Candidate("alpha", "apple orchard"), + Candidate("mu", "something unrelated entirely"), + Candidate("beta", "another unrelated thing")] + queries = {"mu": ["apple", "orchard", "zebra"], "beta": ["zebra"], "zeta": ["stripes"]} + assert all_vs_all(field, queries)["lexical"]["attractors"] == [ + {"skill": "alpha", "wrongly_picked": 2, "sibling_queries": 5, "rate": 0.4, + "from": ["mu"]}, + {"skill": "zeta", "wrongly_picked": 2, "sibling_queries": 4, "rate": 0.5, + "from": ["beta", "mu"]}] From 7737770954fa2fbf51bd188d7f0dd2df03cd8d4c Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 16:04:23 +0100 Subject: [PATCH 2/2] test(novelty): make the arms differ in the top-five attractor test The lexical and model arms printed the same five lines, so _print_attractors printing one arm's rows under both labels still passed. The selector now sends each word to the mirror skill's card, so the model arm ranks alpha..echo and leaves out foxtrot, while the lexical arm ranks foxtrot..bravo and leaves out alpha. Signed-off-by: Anton Volkov --- .../tests/test_novelty_routing_all_vs_all.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/dokimi/tests/test_novelty_routing_all_vs_all.py b/dokimi/tests/test_novelty_routing_all_vs_all.py index f5735b6f2..f3076861b 100644 --- a/dokimi/tests/test_novelty_routing_all_vs_all.py +++ b/dokimi/tests/test_novelty_routing_all_vs_all.py @@ -5,6 +5,7 @@ from __future__ import annotations +import functools import json import pytest @@ -742,13 +743,17 @@ def transport(prompt: str, system: str | None): assert "makes 1 model calls" in capsys.readouterr().err -def _word_client(model, provider, role, args): - """A selector that picks the card holding the request's word, wherever it is shuffled.""" +def _word_client(model, provider, role, args, swap=None): + """A selector that picks the card holding the request's word, wherever it is shuffled. + + With ``swap``, it picks the card holding ``swap[word]`` instead. + """ def transport(prompt: str, system: str | None): request, cards = prompt.split("REQUEST:", 1)[1].split("TOOLS:", 1) + word = (swap or {}).get(request.strip(), request.strip()) lines = cards.strip().splitlines() return json.dumps({"pick": next(i + 1 for i, line in enumerate(lines) - if request.strip() in line)}), None + if word in line)}), None return LLMClient(model=model, provider=provider, role=role, transport=transport) @@ -771,12 +776,15 @@ def test_the_seed_on_the_command_line_is_the_seed_the_saved_model_arm_records( def test_console_prints_the_top_five_attractors_of_each_arm_by_count( tmp_path, capsys, monkeypatch): - # Each of six skills takes the gold queries that name its word: alpha one, ... foxtrot six. + # Lexically, each of six skills takes the gold queries that name its word: alpha one, ... + # foxtrot six. The selector sends each word to the mirror skill's card instead (alpha's + # to foxtrot), so the model arm counts the other way round and leaves out foxtrot. import dokimi.commands.novelty as novelty_cmd - monkeypatch.setattr(novelty_cmd, "_client", _word_client) words = {"alpha": "anvil", "bravo": "barley", "charlie": "cobalt", "delta": "dune", "echo": "ember", "foxtrot": "fjord"} + mirror = dict(zip(words.values(), reversed(words.values()), strict=True)) + monkeypatch.setattr(novelty_cmd, "_client", functools.partial(_word_client, swap=mirror)) root = tmp_path / "catalog" _cat(root, {**words, "gold": "something unrelated entirely"}) queries = _write_queries(tmp_path, {"gold": [word for count, word in @@ -787,10 +795,12 @@ def test_console_prints_the_top_five_attractors_of_each_arm_by_count( "--model", "m") args.func(args) lines = [line for line in capsys.readouterr().out.splitlines() if "sibling" in line] + top_five = {"lexical": [("foxtrot", 6), ("echo", 5), ("delta", 4), ("charlie", 3), + ("bravo", 2)], + "model": [("alpha", 6), ("bravo", 5), ("charlie", 4), ("delta", 3), + ("echo", 2)]} assert lines == [f"{arm}: {name} picked for {count} of 21 sibling queries (from gold)" - for arm in ("lexical", "model") - for name, count in [("foxtrot", 6), ("echo", 5), ("delta", 4), - ("charlie", 3), ("bravo", 2)]] + for arm, rows in top_five.items() for name, count in rows] def test_attractors_tied_on_count_and_golds_tied_in_from_are_in_name_order():