From cea00661d5ea43c083357b68039ea4903726f91c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 21:23:51 -0700 Subject: [PATCH 01/19] feat(cli): route table as data, sectioned help, cost labels, data schema The top-level route list now comes from dokimi/cli_routes.py: each root sits in one of four sections (The loop, Subjects, Evals, Analysis and data), and every route's help line ends in [free] or [paid]. A route is paid when any model call is possible in some flag combination; a root is paid if any leaf is. `dokimi data schema list|show KIND` prints a schema as JSON; cli_routes is the first kind. The table also carries tombstones (an old argv prefix that moved prints "moved: dokimi ..." and exits 2, never runs). The table is empty for now, and the test covers the mechanism with a test-only entry. Handler-neutral: no command moves or renames. Route help text stays with each parser, except run and compare, which get a one-line help from the table. Refs #94 (A1), #276 Signed-off-by: Nikolay Petrov --- ...261028_120000_napetrov_cli-routes-table.md | 17 ++ dokimi/dokimi/cli.py | 76 +++++- dokimi/dokimi/cli_routes.py | 230 ++++++++++++++++++ dokimi/dokimi/commands/data.py | 42 ++++ dokimi/tests/test_cli_routes.py | 150 ++++++++++++ dokimi/tests/test_import_architecture.py | 2 +- 6 files changed, 513 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md create mode 100644 dokimi/dokimi/cli_routes.py create mode 100644 dokimi/dokimi/commands/data.py create mode 100644 dokimi/tests/test_cli_routes.py diff --git a/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md b/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md new file mode 100644 index 000000000..62b1f3573 --- /dev/null +++ b/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md @@ -0,0 +1,17 @@ +### Added + +- **CLI route table** — the top-level `dokimi --help` lists the commands by + section (The loop, Subjects, Evals, Analysis and data), and every command's + help line ends in `[free]` or `[paid]`: `paid` when the command can make a + model call in some flag combination. The table lives in `dokimi/cli_routes.py`; + `dokimi data schema show cli_routes` prints it as JSON. +- **`dokimi data schema list|show KIND`** — the new `data` command group; `show` + prints a schema kind as JSON. `cli_routes` is the only kind for now. +- **Tombstones** — the route table can name an old argv prefix that moved; running + it prints `moved: dokimi ` and exits 2 without running anything. + No command has moved yet, so the table is empty. + +### Changed + +- `dokimi run` and `dokimi compare` have a one-line help in the top-level list. + Their parsers have no help text, so the route table supplies it. diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index 32ee6e64c..ea018a1e0 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -19,6 +19,7 @@ catalog, context, dashboard, + data, dataset, evaluate, explore, @@ -35,11 +36,73 @@ tasks, tools, ) -from dokimi import __version__, process +from dokimi import __version__, cli_routes, process from dokimi.console import tolerate_console_encoding from dokimi.defaults import JUDGE_MODEL_FOR_PROVIDER, default_judge_model +def _subparsers(parser: argparse.ArgumentParser) -> argparse._SubParsersAction | None: + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + return action + return None + + +def _route_help(text: str | None, path: tuple[str, ...]) -> str: + """The one-line help of a route with its cost label: the parser's text, else the table's.""" + base = text or cli_routes.summary_of(path) or "" + cost = cli_routes.cost_of(path) + return f"{base} [{cost}]" if cost else base + + +def _label_routes(parser: argparse.ArgumentParser, prefix: tuple[str, ...] = ()) -> None: + """Append the cost label to every subcommand's help line, at every depth.""" + sub = _subparsers(parser) + if sub is None: + return + listed = {action.dest: action for action in sub._choices_actions} + visited: set[int] = set() + for name, child in sub.choices.items(): + if id(child) in visited: # an alias: the same parser under a second name + continue + visited.add(id(child)) + path = (*prefix, name) + if name in listed: + listed[name].help = _route_help(listed[name].help, path) + _label_routes(child, path) + + +class _RoutedParser(argparse.ArgumentParser): + """The top-level parser. Its ``--help`` lists the roots by section (``dokimi.cli_routes``).""" + + routed = False # set on the top-level parser only; its subparsers keep argparse's layout + + def format_help(self) -> str: + if not self.routed: + return super().format_help() + sub = _subparsers(self) + listed = {action.dest: action for action in sub._choices_actions} if sub else {} + fmt = self._get_formatter() + fmt.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) + for section in cli_routes.SECTIONS: + roots = [root for root in cli_routes.ROOTS if root.section == section] + if not roots: + continue + fmt.start_section(section) # the formatter adds the colon + for root in roots: + action = listed.get(root.name) + if action is None: # a route with no help= in its parser: listed from the table + action = argparse.Action([], root.name, help=_route_help(None, (root.name,))) + if root.aliases: + action.metavar = f"{root.name} ({', '.join(root.aliases)})" + fmt.add_argument(action) + fmt.end_section() + fmt.start_section("options") + fmt.add_arguments([action for action in self._actions if action.option_strings]) + fmt.end_section() + return fmt.format_help() + + def build_parser() -> argparse.ArgumentParser: """Build CLI argument parser.""" @@ -53,7 +116,8 @@ def positive_int(value: str) -> int: raise argparse.ArgumentTypeError(f"must be >= 1, got {value}") return ivalue - p = argparse.ArgumentParser(prog="dokimi") + p = _RoutedParser(prog="dokimi") + p.routed = True p.add_argument("--version", action="version", version=f"%(prog)s {__version__}") sub = p.add_subparsers(dest="cmd", required=True) @@ -81,7 +145,9 @@ def positive_int(value: str) -> int: repo.register(sub, positive_int) # repo (repositories as subjects) tools.register(sub, positive_int) # tools (tool-choice probes) context.register(sub, positive_int) # context (context window of a run) + data.register(sub, positive_int) # data (schema; M2 adds export, import and models) + _label_routes(p) return p @@ -92,7 +158,11 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: read once every flag is parsed, so the two flags can come in either order. """ parser = build_parser() - args = parser.parse_args(argv) + raw = list(sys.argv[1:] if argv is None else argv) + moved = cli_routes.moved_argv(raw) + if moved is not None: # a tombstone: say where the command went, do not run anything + parser.exit(2, cli_routes.moved_text(moved) + "\n") + args = parser.parse_args(raw) if getattr(args, "judge_model", None) is JUDGE_MODEL_FOR_PROVIDER: model = default_judge_model(args.judge_provider) if model is None: diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py new file mode 100644 index 000000000..e853582b5 --- /dev/null +++ b/dokimi/dokimi/cli_routes.py @@ -0,0 +1,230 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The CLI route table: where each root and leaf is listed in ``dokimi --help``, whether it +can spend money, and which old paths are tombstones. + +``dokimi.cli`` reads the table to build the sectioned help and the ``[free]``/``[paid]`` +labels, and ``dokimi data schema show cli_routes`` prints it. Help text stays with each +command's parser; the table gives a one-line help only for a route whose parser has none. + +Cost: ``paid`` means the route can make a model call (an LLM, or an agent harness) in some +flag combination, and ``free`` means it never does. A root is ``paid`` when any leaf under +it is. The sections follow issue #94 §2.1. The table describes today's routes: a root sits in +the section of the operation it will hold once the renames of #94 §4 land. +""" + +from __future__ import annotations + +import shlex +from dataclasses import dataclass + +SECTIONS: tuple[str, ...] = ("The loop", "Subjects", "Evals", "Analysis and data") + +FREE = "free" +PAID = "paid" + + +@dataclass(frozen=True) +class Root: + """A top-level name. ``aliases`` are alternative names for it.""" + + name: str + section: str + aliases: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Leaf: + """An operation: a parser with no subcommands. ``path`` is its argv, e.g. ``("eval", "score")``.""" + + path: tuple[str, ...] + cost: str + summary: str | None = None # one-line help, used only when the parser has none + + +@dataclass(frozen=True) +class Tombstone: + """An old argv prefix that now moved: it answers with ``new`` and exits 2, never forwards.""" + + old: tuple[str, ...] + new: tuple[str, ...] + + +ROOTS: tuple[Root, ...] = ( + Root("evaluate", "The loop"), + Root("baseline", "The loop"), + Root("report", "The loop"), + Root("answers", "The loop"), + Root("benchmark", "The loop"), + Root("dashboard", "The loop"), + Root("arms", "The loop"), + Root("subjects", "The loop"), + Root("tasks", "The loop"), + Root("explore", "The loop"), + Root("run", "Subjects"), + Root("compare", "Subjects"), + Root("library", "Subjects", aliases=("product",)), + Root("catalog", "Subjects"), + Root("novelty", "Subjects"), + Root("repo", "Subjects"), + Root("personas", "Evals"), + Root("questions", "Evals"), + Root("eval", "Evals"), + Root("dataset", "Analysis and data"), + Root("ingest", "Analysis and data"), + Root("tools", "Analysis and data"), + Root("context", "Analysis and data"), + Root("data", "Analysis and data"), +) + +LEAVES: tuple[Leaf, ...] = ( + Leaf(("evaluate",), PAID), + Leaf(("baseline", "save"), FREE), + Leaf(("baseline", "list"), FREE), + Leaf(("baseline", "compare"), FREE), + Leaf(("report", "eval"), FREE), + Leaf(("report", "generate"), FREE), + Leaf(("report", "model-compare"), FREE), + Leaf(("answers", "generate"), PAID), + Leaf(("benchmark", "run"), PAID), + Leaf(("benchmark", "preflight"), FREE), + Leaf(("benchmark", "batch"), PAID), + Leaf(("dashboard", "generate"), FREE), + Leaf(("arms", "run"), PAID), + Leaf(("arms", "matrix-run"), PAID), + Leaf(("arms", "plugin-delta"), FREE), + Leaf(("subjects", "list"), FREE), + Leaf(("subjects", "show"), FREE), + Leaf(("subjects", "run"), PAID), + Leaf(("tasks", "list"), FREE), + Leaf(("tasks", "inspect"), FREE), + Leaf(("tasks", "harnesses"), FREE), + Leaf(("tasks", "run"), PAID), + Leaf(("explore", "list"), FREE), + Leaf(("explore", "run"), PAID), + Leaf(("run",), FREE, "Run the static benchmark spec and save a snapshot"), + Leaf(("compare",), FREE, "Compare two saved benchmark snapshots"), + Leaf(("library", "list"), FREE), + Leaf(("library", "show"), FREE), + Leaf(("library", "intents"), FREE), + Leaf(("catalog", "scan"), FREE), + Leaf(("catalog", "formats"), FREE), + Leaf(("catalog", "trigger-set"), PAID), + Leaf(("catalog", "trigger-probe"), PAID), + Leaf(("novelty", "analyze"), PAID), + Leaf(("novelty", "atoms"), PAID), + Leaf(("novelty", "probe"), PAID), + Leaf(("novelty", "shorten"), PAID), + Leaf(("novelty", "routing"), PAID), + Leaf(("novelty", "items"), FREE), + Leaf(("novelty", "view"), FREE), + Leaf(("repo", "validate"), FREE), + Leaf(("repo", "scan"), FREE), + Leaf(("repo", "inspect"), FREE), + Leaf(("repo", "fetch"), FREE), + Leaf(("repo", "reach"), PAID), + Leaf(("repo", "budget"), FREE), + Leaf(("repo", "tasks", "check"), FREE), + Leaf(("repo", "run"), PAID), + Leaf(("repo", "report"), FREE), + Leaf(("repo", "compare"), FREE), + Leaf(("personas", "discover"), PAID), + Leaf(("personas", "approve"), FREE), + Leaf(("questions", "generate"), PAID), + Leaf(("questions", "analyze"), PAID), + Leaf(("questions", "refine"), PAID), + Leaf(("questions", "panel-review"), PAID), + Leaf(("questions", "import"), FREE), + Leaf(("eval", "score"), PAID), + Leaf(("eval", "ragas"), PAID), + Leaf(("eval", "grounding"), FREE), + Leaf(("eval", "judge-validity"), PAID), + Leaf(("eval", "checklist"), PAID), + Leaf(("eval", "judge-wording"), PAID), + Leaf(("eval", "cross-judge"), PAID), + Leaf(("eval", "pairwise"), PAID), + Leaf(("eval", "reference"), PAID), + Leaf(("eval", "native-import"), FREE), + Leaf(("eval", "panel-score"), PAID), + Leaf(("dataset", "export"), FREE), + Leaf(("ingest", "docling"), FREE), + Leaf(("tools", "preference"), PAID), + Leaf(("context", "trace"), FREE), + Leaf(("data", "schema", "list"), FREE), + Leaf(("data", "schema", "show"), FREE), +) + +# No command has moved yet: the renames of #94 §5 M1-M3 add entries here. +TOMBSTONES: tuple[Tombstone, ...] = () + +SCHEMA_VERSION = "cli_routes.v1" + + +def _leaf(path: tuple[str, ...]) -> Leaf | None: + return next((leaf for leaf in LEAVES if leaf.path == path), None) + + +def cost_of(path: tuple[str, ...]) -> str | None: + """``free`` or ``paid`` for a leaf or a root; None for a route the table does not know.""" + leaf = _leaf(path) + if leaf is not None: + return leaf.cost + under = [leaf for leaf in LEAVES if leaf.path[: len(path)] == path] + if not under: + return None + return PAID if any(leaf.cost == PAID for leaf in under) else FREE + + +def summary_of(path: tuple[str, ...]) -> str | None: + """The table's fallback one-line help for a route whose parser has none.""" + leaf = _leaf(path) + return leaf.summary if leaf is not None else None + + +def moved_argv( + argv: list[str], tombstones: tuple[Tombstone, ...] | None = None +) -> list[str] | None: + """The argv a tombstone rewrites ``argv`` to, or None when no tombstone matches its start.""" + for tomb in TOMBSTONES if tombstones is None else tombstones: + if tuple(argv[: len(tomb.old)]) == tomb.old: + return [*tomb.new, *argv[len(tomb.old) :]] + return None + + +def moved_text(argv: list[str]) -> str: + """The line a tombstone prints: the new command with the argv rewritten.""" + return "moved: " + shlex.join(["dokimi", *argv]) + + +def tombstone_problems(tombstones: tuple[Tombstone, ...] | None = None) -> list[str]: + """Why a tombstone is wrong: its old path is still a live route, or it points nowhere.""" + live = {leaf.path for leaf in LEAVES} + problems = [] + for tomb in TOMBSTONES if tombstones is None else tombstones: + if tomb.old in live: + problems.append(f"{' '.join(tomb.old)} is still a live route") + if tomb.new not in live: + problems.append( + f"{' '.join(tomb.old)} points at {' '.join(tomb.new)}, which is not a route" + ) + return problems + + +def document() -> dict: + """The table as JSON, for ``dokimi data schema show cli_routes``.""" + return { + "schema_version": SCHEMA_VERSION, + "sections": list(SECTIONS), + "roots": [ + { + "name": root.name, + "section": root.section, + "aliases": list(root.aliases), + "cost": cost_of((root.name,)), + } + for root in ROOTS + ], + "leaves": [{"path": list(leaf.path), "cost": leaf.cost} for leaf in LEAVES], + "tombstones": [{"old": list(tomb.old), "new": list(tomb.new)} for tomb in TOMBSTONES], + } diff --git a/dokimi/dokimi/commands/data.py b/dokimi/dokimi/commands/data.py new file mode 100644 index 000000000..077999bb5 --- /dev/null +++ b/dokimi/dokimi/commands/data.py @@ -0,0 +1,42 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""data: schemas of the artifacts and of the CLI routes. ``data schema show KIND`` prints one as JSON.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Callable + +from dokimi import cli_routes + +SCHEMA_KINDS: dict[str, Callable[[], dict]] = { + "cli_routes": cli_routes.document, +} + + +def cmd_schema_list(args: argparse.Namespace) -> None: + """Print the schema kinds, one per line.""" + for kind in SCHEMA_KINDS: + print(kind) + + +def cmd_schema_show(args: argparse.Namespace) -> None: + """Print one schema kind as JSON.""" + print(json.dumps(SCHEMA_KINDS[args.kind](), indent=2)) + + +def register(sub, positive_int): # noqa: ARG001 - the registry passes it to every command + data_p = sub.add_parser("data", help="Schemas of the artifacts and the CLI routes") + data_sub = data_p.add_subparsers(dest="data_cmd", required=True) + + schema_p = data_sub.add_parser("schema", help="Print an artifact or route schema") + schema_sub = schema_p.add_subparsers(dest="schema_cmd", required=True) + + list_p = schema_sub.add_parser("list", help="List the schema kinds") + list_p.set_defaults(func=cmd_schema_list) + + show_p = schema_sub.add_parser("show", help="Print one schema kind as JSON") + show_p.add_argument("kind", choices=sorted(SCHEMA_KINDS)) + show_p.set_defaults(func=cmd_schema_show) diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py new file mode 100644 index 000000000..2558c7123 --- /dev/null +++ b/dokimi/tests/test_cli_routes.py @@ -0,0 +1,150 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The CLI route table: sections, cost labels, tombstones, and ``data schema``.""" + +from __future__ import annotations + +import argparse +import json +import re + +import pytest + +from dokimi import cli_routes +from dokimi.cli import build_parser, parse_args + +SECTION_HEADING = re.compile(r"^(%s):$" % "|".join(re.escape(s) for s in cli_routes.SECTIONS)) + + +def _subparsers(parser: argparse.ArgumentParser) -> argparse._SubParsersAction | None: + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + return action + return None + + +def _leaf_paths( + parser: argparse.ArgumentParser, prefix: tuple[str, ...] = () +) -> set[tuple[str, ...]]: + """Paths of every parser that has no subcommands, aliases resolved to their parser once.""" + found: set[tuple[str, ...]] = set() + sub = _subparsers(parser) + if sub is None: + return {prefix} if prefix else set() + seen: set[int] = set() + for name, child in sub.choices.items(): + if id(child) in seen: + continue + seen.add(id(child)) + found |= _leaf_paths(child, (*prefix, name)) + return found + + +def _help_actions(parser: argparse.ArgumentParser, prefix: tuple[str, ...] = ()): + """(path, pseudo action) for every subcommand that has a help line, at every depth.""" + sub = _subparsers(parser) + if sub is None: + return + for action in sub._choices_actions: + yield (*prefix, action.dest), action + seen: set[int] = set() + for name, child in sub.choices.items(): + if id(child) in seen: # an alias + continue + seen.add(id(child)) + yield from _help_actions(child, (*prefix, name)) + + +def test_help_lists_each_root_under_its_section(): + text = build_parser().format_help() + listed: dict[str, str] = {} + section = None + order: list[str] = [] + for line in text.splitlines(): + if SECTION_HEADING.match(line): + section = line[:-1] + order.append(section) + elif line == "options:": + section = None + elif section and re.match(r"^ \S", line): + listed[line.split()[0]] = section + assert order == list(cli_routes.SECTIONS) + assert listed == {root.name: root.section for root in cli_routes.ROOTS} + + +def test_every_route_help_line_ends_in_its_cost_label(): + top = build_parser() + checked = 0 + for path, action in _help_actions(top): + cost = cli_routes.cost_of(path) + assert cost in ("free", "paid"), path + assert action.help and action.help.endswith(f" [{cost}]"), (path, action.help) + checked += 1 + assert checked >= len(cli_routes.LEAVES) + + +def test_run_and_compare_have_a_one_line_help_in_the_top_level_list(): + text = build_parser().format_help() + for name in ("run", "compare"): + line = next(line for line in text.splitlines() if re.match(rf"^ {name}\s", line)) + assert re.search(r"\S.* \[(free|paid)\]$", line), line + + +def test_route_table_matches_the_parser_leaves_and_roots(): + top = build_parser() + assert _leaf_paths(top) == {leaf.path for leaf in cli_routes.LEAVES} + sub = _subparsers(top) + assert sub is not None + names = set(sub.choices) + aliases = {alias for root in cli_routes.ROOTS for alias in root.aliases} + assert names - aliases == {root.name for root in cli_routes.ROOTS} + assert aliases == {"product"} + + +def test_a_root_costs_paid_when_any_of_its_leaves_is_paid(): + assert cli_routes.cost_of(("repo",)) == "paid" + assert cli_routes.cost_of(("library",)) == "free" + assert cli_routes.cost_of(("data",)) == "free" + assert cli_routes.cost_of(("eval", "judge-validity")) == "paid" + assert cli_routes.cost_of(("no", "such", "route")) is None + + +def test_the_tombstone_table_is_empty_and_its_check_finds_a_live_route(): + assert cli_routes.TOMBSTONES == () + assert cli_routes.tombstone_problems() == [] + bad = (cli_routes.Tombstone(("eval", "score"), ("eval", "inspect")),) + assert cli_routes.tombstone_problems(bad) # the old path is still a live route + + +def test_a_tombstone_rewrites_the_argv_and_exits_2(monkeypatch, capsys): + # Test-only entry: no command moves in this release. + monkeypatch.setattr( + cli_routes, + "TOMBSTONES", + (cli_routes.Tombstone(("old-tool", "inspect"), ("eval", "inspect")),), + ) + with pytest.raises(SystemExit) as exc: + parse_args(["old-tool", "inspect", "data.json"]) + assert exc.value.code == 2 + assert "moved: dokimi eval inspect data.json" in capsys.readouterr().err + + +def test_an_argv_that_no_tombstone_matches_is_parsed_as_before(): + args = parse_args(["data", "schema", "list"]) + assert args.cmd == "data" + + +def test_data_schema_show_prints_the_route_table_as_json(capsys): + args = parse_args(["data", "schema", "show", "cli_routes"]) + args.func(args) + doc = json.loads(capsys.readouterr().out) + assert doc == cli_routes.document() + assert doc["schema_version"] == "cli_routes.v1" + assert {"path": ["eval", "score"], "cost": "paid"} in doc["leaves"] + + +def test_data_schema_list_names_the_kinds(capsys): + args = parse_args(["data", "schema", "list"]) + args.func(args) + assert capsys.readouterr().out.split() == ["cli_routes"] diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index 92c46ae7d..ab1b5425c 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -44,7 +44,7 @@ #: its own layer and any layer below it. LAYERS: tuple[tuple[str, tuple[str, ...]], ...] = ( # The argparse front end: parses flags, calls into the tracks, prints. - ("frontend", ("dokimi.__main__", "dokimi.cli", "dokimi.commands")), + ("frontend", ("dokimi.__main__", "dokimi.cli", "dokimi.cli_routes", "dokimi.commands")), # Whole tracks and the tools beside them: static run, novelty, executable tasks, # repositories, subjects, reports, dashboards, the one-command pipeline. ("tracks", ( From cf5166bc49bcd20e2717d58f8e5f1c6cdf0fa035 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 21:39:19 -0700 Subject: [PATCH 02/19] fix(cli): tombstone check follows groups, aliases and roots moved_argv matches by argv prefix, but tombstone_problems compared paths to the leaf list only. A tombstone on a group (repo tasks) therefore passed the check and shadowed live routes under it, and a root or alias destination (library) was wrongly reported as not a route. A path now counts as a route when it names a root or alias, or a group or leaf under one. Self-review of the A1 commit (blocking). Signed-off-by: Nikolay Petrov --- dokimi/dokimi/cli.py | 2 ++ dokimi/dokimi/cli_routes.py | 20 +++++++++++++++++--- dokimi/tests/test_cli_routes.py | 14 ++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index ea018a1e0..b4cd4b9e0 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -84,6 +84,8 @@ def format_help(self) -> str: listed = {action.dest: action for action in sub._choices_actions} if sub else {} fmt = self._get_formatter() fmt.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) + # A parser root missing from cli_routes.ROOTS is left out of this list; + # test_route_table_matches_the_parser_leaves_and_roots fails on it. for section in cli_routes.SECTIONS: roots = [root for root in cli_routes.ROOTS if root.section == section] if not roots: diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index e853582b5..89938eb1b 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -197,14 +197,28 @@ def moved_text(argv: list[str]) -> str: return "moved: " + shlex.join(["dokimi", *argv]) +def _is_route(path: tuple[str, ...]) -> bool: + """True when argv ``path`` names a root or an alias, or a group or leaf under one. + + ``moved_argv`` matches by prefix, so a tombstone on a group shadows every route under it. + """ + if not path: + return False + heads = {name: root.name for root in ROOTS for name in (root.name, *root.aliases)} + head = heads.get(path[0]) + if head is None: + return False + full = (head, *path[1:]) + return any(leaf.path[: len(full)] == full for leaf in LEAVES) + + def tombstone_problems(tombstones: tuple[Tombstone, ...] | None = None) -> list[str]: """Why a tombstone is wrong: its old path is still a live route, or it points nowhere.""" - live = {leaf.path for leaf in LEAVES} problems = [] for tomb in TOMBSTONES if tombstones is None else tombstones: - if tomb.old in live: + if _is_route(tomb.old): problems.append(f"{' '.join(tomb.old)} is still a live route") - if tomb.new not in live: + if not _is_route(tomb.new): problems.append( f"{' '.join(tomb.old)} points at {' '.join(tomb.new)}, which is not a route" ) diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 2558c7123..4301ea782 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -117,6 +117,20 @@ def test_the_tombstone_table_is_empty_and_its_check_finds_a_live_route(): assert cli_routes.tombstone_problems(bad) # the old path is still a live route +def test_the_tombstone_check_follows_groups_aliases_and_roots(): + # moved_argv matches by prefix: a group tombstone shadows the live routes under it. + shadows = (cli_routes.Tombstone(("repo", "tasks"), ("tasks", "run")),) + assert cli_routes.tombstone_problems(shadows) + # an alias is a live route too + alias = (cli_routes.Tombstone(("product",), ("library", "list")),) + assert cli_routes.tombstone_problems(alias) + # a root is a valid destination, and an unknown name is a valid old path + root = (cli_routes.Tombstone(("old-tool",), ("library",)),) + assert cli_routes.tombstone_problems(root) == [] + nowhere = (cli_routes.Tombstone(("old-tool",), ("no-such-root",)),) + assert cli_routes.tombstone_problems(nowhere) + + def test_a_tombstone_rewrites_the_argv_and_exits_2(monkeypatch, capsys): # Test-only entry: no command moves in this release. monkeypatch.setattr( From a654a8b9000d9614a746ce035786d674771aebf3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 21:48:10 -0700 Subject: [PATCH 03/19] fix(cli): tombstone on a path under a live leaf; no stray space in labels A tombstone whose old path extends a live leaf (compare x, where compare takes files) shadows that leaf's invocations, so tombstone_problems now reports it. The tombstone check resolves aliases through one helper. A route with neither parser help nor a table summary gets its label without a leading space. Self-review round 2 (no blocking; these two nits and the test count fixed). Signed-off-by: Nikolay Petrov --- dokimi/dokimi/cli.py | 2 +- dokimi/dokimi/cli_routes.py | 29 +++++++++++++++++++++-------- dokimi/tests/test_cli_routes.py | 3 +++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index b4cd4b9e0..affec419f 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -52,7 +52,7 @@ def _route_help(text: str | None, path: tuple[str, ...]) -> str: """The one-line help of a route with its cost label: the parser's text, else the table's.""" base = text or cli_routes.summary_of(path) or "" cost = cli_routes.cost_of(path) - return f"{base} [{cost}]" if cost else base + return " ".join(part for part in (base, f"[{cost}]" if cost else "") if part) def _label_routes(parser: argparse.ArgumentParser, prefix: tuple[str, ...] = ()) -> None: diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 89938eb1b..0659004d5 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -197,19 +197,30 @@ def moved_text(argv: list[str]) -> str: return "moved: " + shlex.join(["dokimi", *argv]) +def _resolve(path: tuple[str, ...]) -> tuple[str, ...] | None: + """``path`` with its first word resolved from an alias to its root; None if no root has it.""" + if not path: + return None + heads = {name: root.name for root in ROOTS for name in (root.name, *root.aliases)} + head = heads.get(path[0]) + return None if head is None else (head, *path[1:]) + + def _is_route(path: tuple[str, ...]) -> bool: """True when argv ``path`` names a root or an alias, or a group or leaf under one. ``moved_argv`` matches by prefix, so a tombstone on a group shadows every route under it. """ - if not path: - return False - heads = {name: root.name for root in ROOTS for name in (root.name, *root.aliases)} - head = heads.get(path[0]) - if head is None: - return False - full = (head, *path[1:]) - return any(leaf.path[: len(full)] == full for leaf in LEAVES) + full = _resolve(path) + return full is not None and any(leaf.path[: len(full)] == full for leaf in LEAVES) + + +def _extends_a_leaf(path: tuple[str, ...]) -> bool: + """True when ``path`` is longer than a live leaf it starts with: the leaf takes it as arguments.""" + full = _resolve(path) + return full is not None and any( + len(leaf.path) < len(full) and full[: len(leaf.path)] == leaf.path for leaf in LEAVES + ) def tombstone_problems(tombstones: tuple[Tombstone, ...] | None = None) -> list[str]: @@ -218,6 +229,8 @@ def tombstone_problems(tombstones: tuple[Tombstone, ...] | None = None) -> list[ for tomb in TOMBSTONES if tombstones is None else tombstones: if _is_route(tomb.old): problems.append(f"{' '.join(tomb.old)} is still a live route") + if _extends_a_leaf(tomb.old): + problems.append(f"{' '.join(tomb.old)} extends a live route and would shadow it") if not _is_route(tomb.new): problems.append( f"{' '.join(tomb.old)} points at {' '.join(tomb.new)}, which is not a route" diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 4301ea782..2b8a9ce5a 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -129,6 +129,9 @@ def test_the_tombstone_check_follows_groups_aliases_and_roots(): assert cli_routes.tombstone_problems(root) == [] nowhere = (cli_routes.Tombstone(("old-tool",), ("no-such-root",)),) assert cli_routes.tombstone_problems(nowhere) + # compare takes positional files, so "compare x" is a live invocation, not a free old path + extends = (cli_routes.Tombstone(("compare", "x"), ("compare",)),) + assert cli_routes.tombstone_problems(extends) def test_a_tombstone_rewrites_the_argv_and_exits_2(monkeypatch, capsys): From 08ac435f18e61db7ca7ec51658b7de7cfb52389d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 21:57:13 -0700 Subject: [PATCH 04/19] fix(cli): tombstones match alias argv, reject an empty or option old path moved_argv compared raw argv, so `product old` skipped a tombstone on `library old`. It now compares canonical paths (an alias is replaced by its root's name on both sides). An empty old path matched every argv, and an option (--version) is not a command; tombstone_problems reports both. Self-review round 3 (2 blocking, both fixed; the alias-ownership check is new). Signed-off-by: Nikolay Petrov --- dokimi/dokimi/cli_routes.py | 18 ++++++++++++++++-- dokimi/tests/test_cli_routes.py | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 0659004d5..eec652b49 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -182,12 +182,24 @@ def summary_of(path: tuple[str, ...]) -> str | None: return leaf.summary if leaf is not None else None +def _canonical(path: tuple[str, ...]) -> tuple[str, ...]: + """``path`` with its first word replaced by the root name when that word is an alias.""" + if not path: + return path + head = next((root.name for root in ROOTS if path[0] in (root.name, *root.aliases)), path[0]) + return (head, *path[1:]) + + def moved_argv( argv: list[str], tombstones: tuple[Tombstone, ...] | None = None ) -> list[str] | None: - """The argv a tombstone rewrites ``argv`` to, or None when no tombstone matches its start.""" + """The argv a tombstone rewrites ``argv`` to, or None when no tombstone matches its start. + + An alias typed on the command line (``product`` for ``library``) matches the root's tombstones. + """ + canon = _canonical(tuple(argv)) for tomb in TOMBSTONES if tombstones is None else tombstones: - if tuple(argv[: len(tomb.old)]) == tomb.old: + if canon[: len(tomb.old)] == _canonical(tomb.old): return [*tomb.new, *argv[len(tomb.old) :]] return None @@ -227,6 +239,8 @@ def tombstone_problems(tombstones: tuple[Tombstone, ...] | None = None) -> list[ """Why a tombstone is wrong: its old path is still a live route, or it points nowhere.""" problems = [] for tomb in TOMBSTONES if tombstones is None else tombstones: + if not tomb.old or tomb.old[0].startswith("-"): # an empty path matches every argv + problems.append(f"{' '.join(tomb.old) or '(empty)'} is not a command path") if _is_route(tomb.old): problems.append(f"{' '.join(tomb.old)} is still a live route") if _extends_a_leaf(tomb.old): diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 2b8a9ce5a..232cb12e5 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -132,6 +132,28 @@ def test_the_tombstone_check_follows_groups_aliases_and_roots(): # compare takes positional files, so "compare x" is a live invocation, not a free old path extends = (cli_routes.Tombstone(("compare", "x"), ("compare",)),) assert cli_routes.tombstone_problems(extends) + # an empty old path matches every argv; an option is not a command + assert cli_routes.tombstone_problems((cli_routes.Tombstone((), ("data",)),)) + assert cli_routes.tombstone_problems((cli_routes.Tombstone(("--version",), ("data",)),)) + + +def test_a_tombstone_matches_an_alias_typed_on_the_command_line(): + tombs = (cli_routes.Tombstone(("library", "old-intents"), ("library", "intents")),) + assert cli_routes.moved_argv(["product", "old-intents"], tombs) == ["library", "intents"] + assert cli_routes.moved_argv(["library", "old-intents", "x"], tombs) == [ + "library", + "intents", + "x", + ] + assert cli_routes.moved_argv(["library", "list"], tombs) is None + + +def test_every_alias_points_at_its_own_root_parser(): + sub = _subparsers(build_parser()) + assert sub is not None + for root in cli_routes.ROOTS: + for alias in root.aliases: + assert sub.choices[alias] is sub.choices[root.name], (alias, root.name) def test_a_tombstone_rewrites_the_argv_and_exits_2(monkeypatch, capsys): From 3bbfc63a5d75a6a38de1343b54f32411860908e5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:39:54 -0700 Subject: [PATCH 05/19] fix(cli): add the two routes main gained to the route table After merging main, `repo tasks replay` and `context consistency` are parser leaves the table did not list, so test_route_table_matches_the_parser_leaves_and_roots and the cost-label test failed on the merged tree. Both are free: replay reads a recorded threads file and writes tasks (no LLM, no GitHub), consistency reads local trace files. Sync with main for PR 284; not a review item. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/cli_routes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index eec652b49..d6ae4daf2 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -126,6 +126,7 @@ class Tombstone: Leaf(("repo", "reach"), PAID), Leaf(("repo", "budget"), FREE), Leaf(("repo", "tasks", "check"), FREE), + Leaf(("repo", "tasks", "replay"), FREE), Leaf(("repo", "run"), PAID), Leaf(("repo", "report"), FREE), Leaf(("repo", "compare"), FREE), @@ -151,6 +152,7 @@ class Tombstone: Leaf(("ingest", "docling"), FREE), Leaf(("tools", "preference"), PAID), Leaf(("context", "trace"), FREE), + Leaf(("context", "consistency"), FREE), Leaf(("data", "schema", "list"), FREE), Leaf(("data", "schema", "show"), FREE), ) From 55601aef044419d26e2eae31be153509311dd4b4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:40:53 -0700 Subject: [PATCH 06/19] fix(cli): pin the cli_routes.v1 document with a schema dokimi data schema show cli_routes printed a document stamped cli_routes.v1 that no schema defined, so schema_compat could not catch a renamed leaves[].path or a dropped roots[].cost. Add schemas/cli_routes.v1.json, its fixture, register the kind, and test that document() validates and that a drifted shape is rejected. Thread: PRRT_kwDOVBg7g86xg7_0 Signed-off-by: Nikolay Petrov --- ...261028_120000_napetrov_cli-routes-table.md | 3 +- dokimi/dokimi/artifacts.py | 1 + dokimi/dokimi/schemas/cli_routes.v1.json | 64 +++++++++++++++++++ .../fixtures/artifacts/cli_routes.v1.json | 15 +++++ dokimi/tests/test_cli_routes.py | 23 +++++++ 5 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 dokimi/dokimi/schemas/cli_routes.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/cli_routes.v1.json diff --git a/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md b/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md index 62b1f3573..6528255cd 100644 --- a/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md +++ b/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md @@ -4,7 +4,8 @@ section (The loop, Subjects, Evals, Analysis and data), and every command's help line ends in `[free]` or `[paid]`: `paid` when the command can make a model call in some flag combination. The table lives in `dokimi/cli_routes.py`; - `dokimi data schema show cli_routes` prints it as JSON. + `dokimi data schema show cli_routes` prints it as JSON, and + `schemas/cli_routes.v1.json` pins its shape. - **`dokimi data schema list|show KIND`** — the new `data` command group; `show` prints a schema kind as JSON. `cli_routes` is the only kind for now. - **Tombstones** — the route table can name an old argv prefix that moved; running diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 5731c0f46..0088ff5eb 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -57,6 +57,7 @@ "result": "result.v1", "repo_budget": "repo_budget.v1", "task_inspect": "task_inspect.v1", + "cli_routes": "cli_routes.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/schemas/cli_routes.v1.json b/dokimi/dokimi/schemas/cli_routes.v1.json new file mode 100644 index 000000000..de1fd3141 --- /dev/null +++ b/dokimi/dokimi/schemas/cli_routes.v1.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/cli_routes.v1.json", + "title": "CLI route table v1", + "description": "The routes of the dokimi CLI as `dokimi data schema show cli_routes` prints them (dokimi.cli_routes.document): the help sections, every top-level command with its section, aliases and cost, every leaf operation with its cost, and the tombstones (old argv prefixes that moved). `cost` is `paid` when the route can make a model call in some flag combination, else `free`.", + "type": "object", + "required": ["schema_version", "sections", "roots", "leaves", "tombstones"], + "additionalProperties": false, + "$defs": { + "cost": {"enum": ["free", "paid"]}, + "path": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1}, + "description": "An argv prefix, e.g. [\"eval\", \"score\"]." + } + }, + "properties": { + "schema_version": {"const": "cli_routes.v1"}, + "sections": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + "description": "The headings of `dokimi --help`, in order." + }, + "roots": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "section", "aliases", "cost"], + "additionalProperties": false, + "properties": { + "name": {"type": "string", "minLength": 1}, + "section": {"type": "string", "minLength": 1, "description": "One of `sections`."}, + "aliases": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "cost": {"$ref": "#/$defs/cost", "description": "paid when any leaf under the root is paid."} + } + } + }, + "leaves": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "cost"], + "additionalProperties": false, + "properties": { + "path": {"$ref": "#/$defs/path"}, + "cost": {"$ref": "#/$defs/cost"} + } + } + }, + "tombstones": { + "type": "array", + "items": { + "type": "object", + "required": ["old", "new"], + "additionalProperties": false, + "properties": { + "old": {"$ref": "#/$defs/path"}, + "new": {"$ref": "#/$defs/path"} + } + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/cli_routes.v1.json b/dokimi/tests/fixtures/artifacts/cli_routes.v1.json new file mode 100644 index 000000000..270924b38 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/cli_routes.v1.json @@ -0,0 +1,15 @@ +{ + "schema_version": "cli_routes.v1", + "sections": ["The loop", "Subjects"], + "roots": [ + {"name": "evaluate", "section": "The loop", "aliases": [], "cost": "paid"}, + {"name": "library", "section": "Subjects", "aliases": ["product"], "cost": "free"} + ], + "leaves": [ + {"path": ["evaluate"], "cost": "paid"}, + {"path": ["library", "list"], "cost": "free"} + ], + "tombstones": [ + {"old": ["old-tool", "inspect"], "new": ["library", "list"]} + ] +} diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 232cb12e5..7cab523c1 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -12,6 +12,7 @@ import pytest from dokimi import cli_routes +from dokimi.artifacts import schema_errors from dokimi.cli import build_parser, parse_args SECTION_HEADING = re.compile(r"^(%s):$" % "|".join(re.escape(s) for s in cli_routes.SECTIONS)) @@ -187,3 +188,25 @@ def test_data_schema_list_names_the_kinds(capsys): args = parse_args(["data", "schema", "list"]) args.func(args) assert capsys.readouterr().out.split() == ["cli_routes"] + + +def test_the_route_document_validates_against_its_schema(): + assert schema_errors("cli_routes.v1", cli_routes.document()) == [] + + +@pytest.mark.parametrize( + "damage", + [ + lambda doc: doc["roots"][0].pop("cost"), + lambda doc: doc["leaves"][0].update(cost="cheap"), + lambda doc: doc["leaves"][0].pop("path"), + lambda doc: doc.pop("tombstones"), + lambda doc: doc.update(schema_version="cli_routes.v2"), + lambda doc: doc["roots"][0].update(extra=1), + ], + ids=["root-no-cost", "bad-cost", "leaf-no-path", "no-tombstones", "wrong-version", "extra-key"], +) +def test_the_route_schema_rejects_a_document_whose_shape_drifted(damage): + doc = cli_routes.document() + damage(doc) + assert schema_errors("cli_routes.v1", doc) From 1f26a763df4f06f80be8b616787d6d495b4d8277 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:20 -0700 Subject: [PATCH 07/19] fix(cli): say in the data help what data schema prints today `dokimi data --help` promised "Schemas of the artifacts and the CLI routes" and `data schema` "an artifact or route schema", but cli_routes is the only kind: `data schema show eval` is an invalid choice. Say what is there; widen the text when artifact kinds are added (the test pins the pairing). Thread: PRRT_kwDOVBg7g86xg7_2 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/data.py | 6 +++--- dokimi/tests/test_cli_routes.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/data.py b/dokimi/dokimi/commands/data.py index 077999bb5..87cd2ea31 100644 --- a/dokimi/dokimi/commands/data.py +++ b/dokimi/dokimi/commands/data.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""data: schemas of the artifacts and of the CLI routes. ``data schema show KIND`` prints one as JSON.""" +"""data: schemas. ``data schema show KIND`` prints one as JSON; ``cli_routes`` is the only kind so far.""" from __future__ import annotations @@ -28,10 +28,10 @@ def cmd_schema_show(args: argparse.Namespace) -> None: def register(sub, positive_int): # noqa: ARG001 - the registry passes it to every command - data_p = sub.add_parser("data", help="Schemas of the artifacts and the CLI routes") + data_p = sub.add_parser("data", help="Schemas: today the CLI route table") data_sub = data_p.add_subparsers(dest="data_cmd", required=True) - schema_p = data_sub.add_parser("schema", help="Print an artifact or route schema") + schema_p = data_sub.add_parser("schema", help="Print the CLI route table as JSON") schema_sub = schema_p.add_subparsers(dest="schema_cmd", required=True) list_p = schema_sub.add_parser("list", help="List the schema kinds") diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 7cab523c1..a14a378f3 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -210,3 +210,13 @@ def test_the_route_schema_rejects_a_document_whose_shape_drifted(damage): doc = cli_routes.document() damage(doc) assert schema_errors("cli_routes.v1", doc) + + +def test_the_data_help_promises_only_the_kinds_it_has(): + from dokimi.commands.data import SCHEMA_KINDS + + helps = {path: action.help for path, action in _help_actions(build_parser())} + assert set(SCHEMA_KINDS) == {"cli_routes"} # widen the help below when a kind is added + for path in (("data",), ("data", "schema")): + assert "artifact" not in helps[path].lower(), helps[path] + assert "route" in helps[path].lower(), helps[path] From f97bb3e331185184be4737c9a6d0192674c97246 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:42 -0700 Subject: [PATCH 08/19] refactor(cli): drop the routed flag, build subparsers as plain ArgumentParser add_subparsers builds its children with type(self), so every `dokimi ` parser was a _RoutedParser that relied on the class default routed = False. Pass parser_class=argparse.ArgumentParser instead and remove the flag; only the top-level parser lists its commands by section. Thread: PRRT_kwDOVBg7g86xg7_4 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/cli.py | 7 +------ dokimi/tests/test_cli_routes.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index affec419f..7e77d1823 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -75,11 +75,7 @@ def _label_routes(parser: argparse.ArgumentParser, prefix: tuple[str, ...] = ()) class _RoutedParser(argparse.ArgumentParser): """The top-level parser. Its ``--help`` lists the roots by section (``dokimi.cli_routes``).""" - routed = False # set on the top-level parser only; its subparsers keep argparse's layout - def format_help(self) -> str: - if not self.routed: - return super().format_help() sub = _subparsers(self) listed = {action.dest: action for action in sub._choices_actions} if sub else {} fmt = self._get_formatter() @@ -119,9 +115,8 @@ def positive_int(value: str) -> int: return ivalue p = _RoutedParser(prog="dokimi") - p.routed = True p.add_argument("--version", action="version", version=f"%(prog)s {__version__}") - sub = p.add_subparsers(dest="cmd", required=True) + sub = p.add_subparsers(dest="cmd", required=True, parser_class=argparse.ArgumentParser) # Order preserved to keep --help output stable. run.register(sub, positive_int) # run diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index a14a378f3..165bed31c 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -220,3 +220,13 @@ def test_the_data_help_promises_only_the_kinds_it_has(): for path in (("data",), ("data", "schema")): assert "artifact" not in helps[path].lower(), helps[path] assert "route" in helps[path].lower(), helps[path] + + +def test_only_the_top_level_parser_lists_its_commands_by_section(): + top = build_parser() + assert "The loop:" in top.format_help() + sub = _subparsers(top) + assert sub is not None + for name, child in sub.choices.items(): + assert type(child) is argparse.ArgumentParser, name + assert "The loop:" not in child.format_help(), name From 49df9a8df52872389f6ca20b1721b14c4579c24c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:42:50 -0700 Subject: [PATCH 09/19] test(cli): pin the cost labels, the route document and the tombstone rewrite to literals The cost-label, section and `data schema show` tests compared the output to cli_routes itself, so a wrong `[free]` passed. Mutations, each applied alone to cli_routes.py, all survived tests/test_cli_routes.py: evaluate and tools preference flipped to free, repo tasks check to paid, document() reporting every root free, no sections or no aliases, and moved_argv slicing by len(tomb.new). Now: the paid leaf paths are a literal set (the help labels are checked against it too), one root's section/aliases/cost and the sections are asserted from literals, and a tombstone whose old and new paths differ in length (old starting with `product`) is rewritten. All seven mutants now fail the file. Thread: PRRT_kwDOVBg7g86xh56h Signed-off-by: Nikolay Petrov --- dokimi/tests/test_cli_routes.py | 79 +++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 165bed31c..a4d825a95 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -74,6 +74,56 @@ def test_help_lists_each_root_under_its_section(): assert listed == {root.name: root.section for root in cli_routes.ROOTS} +# The commands that can make a model call. Written out here, not read from cli_routes, so a +# label that says `free` on a command that spends money (or the reverse) fails a test. +PAID_PATHS = { + ("evaluate",), + ("answers", "generate"), + ("benchmark", "run"), + ("benchmark", "batch"), + ("arms", "run"), + ("arms", "matrix-run"), + ("subjects", "run"), + ("tasks", "run"), + ("explore", "run"), + ("catalog", "trigger-set"), + ("catalog", "trigger-probe"), + ("novelty", "analyze"), + ("novelty", "atoms"), + ("novelty", "probe"), + ("novelty", "shorten"), + ("novelty", "routing"), + ("repo", "reach"), + ("repo", "run"), + ("personas", "discover"), + ("questions", "generate"), + ("questions", "analyze"), + ("questions", "refine"), + ("questions", "panel-review"), + ("eval", "score"), + ("eval", "ragas"), + ("eval", "judge-validity"), + ("eval", "checklist"), + ("eval", "judge-wording"), + ("eval", "cross-judge"), + ("eval", "pairwise"), + ("eval", "reference"), + ("eval", "panel-score"), + ("tools", "preference"), +} + + +def test_the_paid_leaves_are_exactly_the_ones_written_out_here(): + assert {leaf.path for leaf in cli_routes.LEAVES if leaf.cost == "paid"} == PAID_PATHS + + +def test_the_help_label_of_a_route_comes_from_its_literal_cost(): + top = build_parser() + for path, action in _help_actions(top): + paid = any(p[: len(path)] == path for p in PAID_PATHS) + assert action.help.endswith(" [paid]" if paid else " [free]"), (path, action.help) + + def test_every_route_help_line_ends_in_its_cost_label(): top = build_parser() checked = 0 @@ -149,6 +199,22 @@ def test_a_tombstone_matches_an_alias_typed_on_the_command_line(): assert cli_routes.moved_argv(["library", "list"], tombs) is None +def test_a_tombstone_keeps_the_arguments_after_the_old_path_whatever_the_lengths(): + longer_old = (cli_routes.Tombstone(("product", "old-intents", "all"), ("library", "intents")),) + assert cli_routes.moved_argv(["product", "old-intents", "all", "x"], longer_old) == [ + "library", + "intents", + "x", + ] + longer_new = (cli_routes.Tombstone(("product", "old"), ("library", "show", "intents")),) + assert cli_routes.moved_argv(["product", "old", "x"], longer_new) == [ + "library", + "show", + "intents", + "x", + ] + + def test_every_alias_points_at_its_own_root_parser(): sub = _subparsers(build_parser()) assert sub is not None @@ -184,6 +250,19 @@ def test_data_schema_show_prints_the_route_table_as_json(capsys): assert {"path": ["eval", "score"], "cost": "paid"} in doc["leaves"] +def test_the_route_document_reports_each_roots_section_aliases_and_cost(): + doc = cli_routes.document() + assert doc["sections"] == ["The loop", "Subjects", "Evals", "Analysis and data"] + roots = {root["name"]: root for root in doc["roots"]} + assert roots["library"] == { + "name": "library", "section": "Subjects", "aliases": ["product"], "cost": "free" + } + assert roots["repo"] == {"name": "repo", "section": "Subjects", "aliases": [], "cost": "paid"} + assert roots["data"]["section"] == "Analysis and data" + assert roots["evaluate"]["cost"] == "paid" + assert {leaf["cost"] for leaf in doc["leaves"]} == {"free", "paid"} + + def test_data_schema_list_names_the_kinds(capsys): args = parse_args(["data", "schema", "list"]) args.func(args) From fe1c6a7a459cfc857a4a3182792205d97904dded Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:43:06 -0700 Subject: [PATCH 10/19] test(cli): name the leaves whose help line was checked, not count the lines The cost-label test guarded with `checked >= len(LEAVES)`, but the walk also counts the root and group entries, which make up for a missing leaf; and a leaf parser added without help= is absent from _choices_actions, so it was skipped. Removing help= from `baseline save` left the file green (and `save` vanished from `dokimi baseline --help`). Compare the set of checked paths with the leaf paths instead; `run` and `compare`, whose help comes from the table's summary, are the exempt ones. Thread: PRRT_kwDOVBg7g86xlZDa Signed-off-by: Nikolay Petrov --- dokimi/tests/test_cli_routes.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index a4d825a95..edb180da4 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -126,13 +126,16 @@ def test_the_help_label_of_a_route_comes_from_its_literal_cost(): def test_every_route_help_line_ends_in_its_cost_label(): top = build_parser() - checked = 0 + seen = set() for path, action in _help_actions(top): cost = cli_routes.cost_of(path) assert cost in ("free", "paid"), path assert action.help and action.help.endswith(f" [{cost}]"), (path, action.help) - checked += 1 - assert checked >= len(cli_routes.LEAVES) + seen.add(path) + # A leaf whose parser has no help= is not in _choices_actions at all, so it must be named: + # a count would be made up by the root and group lines. `run` and `compare` take the + # table's summary in the top-level list instead (the test below). + assert {leaf.path for leaf in cli_routes.LEAVES if leaf.summary is None} <= seen def test_run_and_compare_have_a_one_line_help_in_the_top_level_list(): From c8d0ebaa847e0342fb5bd210e0aaaaf953c605db Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:45:11 -0700 Subject: [PATCH 11/19] test(cli): cover the empty argv in moved_argv Codecov listed line 190 of cli_routes.py (the empty-argv return in _canonical) as the patch lines without coverage. An empty argv matches no tombstone and argparse still asks for a command. Comment: IC_kwDOVBg7g88AAAABgsGHDA Signed-off-by: Nikolay Petrov --- dokimi/tests/test_cli_routes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index edb180da4..4f5855a02 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -218,6 +218,14 @@ def test_a_tombstone_keeps_the_arguments_after_the_old_path_whatever_the_lengths ] +def test_an_empty_argv_matches_no_tombstone(): + tombs = (cli_routes.Tombstone(("old-tool",), ("library",)),) + assert cli_routes.moved_argv([], tombs) is None + with pytest.raises(SystemExit) as exc: # argparse still asks for a command + parse_args([]) + assert exc.value.code == 2 + + def test_every_alias_points_at_its_own_root_parser(): sub = _subparsers(build_parser()) assert sub is not None From eb5e6127496657f70d479d1b2616b8471c3f8c11 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:45:25 -0700 Subject: [PATCH 12/19] Address review: the summary items are answered by the commits of their line comments Review bodies and the OpenCodeReview summary carry no claim of their own; their points are the line-comment threads fixed in the commits above (schema, data help, routed flag, test pins, help-less leaf), plus the Codecov line. Review: PRR_kwDOVBg7g88AAAABU4jUUQ Review: PRR_kwDOVBg7g88AAAABU5Daiw Review: PRR_kwDOVBg7g88AAAABU66xpw Comment: IC_kwDOVBg7g88AAAABg1oaiw Signed-off-by: Nikolay Petrov From 50a9ab93e5f6f68bd336518514149ef2ed8c442b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:52:12 -0700 Subject: [PATCH 13/19] test(cli): pin the help width the top-level cost label is read at argparse wraps help text at COLUMNS, or the terminal width, and 80 when neither is set. On a host narrower than about 77 columns the "[free]" label of `run` wraps onto a continuation line, so the one-line-help check fails although the label is there. Set COLUMNS=200 in that test, as test_bench_env does for the bench harness. Red: before this change the test fails at COLUMNS=76 and COLUMNS=60 (1 failed). Green: after it, the module passes at COLUMNS=60, 76, 80, 200 and unset. test_help_lists_each_root_under_its_section passes at COLUMNS 20-60 and is not changed. Review thread on dokimi/tests/test_cli_routes.py:142. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_cli_routes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 4f5855a02..c4a3e1259 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -138,7 +138,10 @@ def test_every_route_help_line_ends_in_its_cost_label(): assert {leaf.path for leaf in cli_routes.LEAVES if leaf.summary is None} <= seen -def test_run_and_compare_have_a_one_line_help_in_the_top_level_list(): +def test_run_and_compare_have_a_one_line_help_in_the_top_level_list(monkeypatch): + # argparse wraps help at COLUMNS (or the terminal width, else 80): a narrow host would + # wrap the cost label onto a continuation line, so pin the width the label is read at. + monkeypatch.setenv("COLUMNS", "200") text = build_parser().format_help() for name in ("run", "compare"): line = next(line for line in text.splitlines() if re.match(rf"^ {name}\s", line)) From c0c3f15b50ec08d6f8002d74438ebfbee654cf9e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:28:56 -0700 Subject: [PATCH 14/19] test(cli): fit the leaf-without-help comment in the two-line limit The comment block at the leaf check ran to three lines, which fails the Comment blocks CI gate (scripts/check_comment_blocks.py). Same reason, shorter: the leaf is named, not counted. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_cli_routes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index c4a3e1259..fe9d73ff1 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -132,9 +132,8 @@ def test_every_route_help_line_ends_in_its_cost_label(): assert cost in ("free", "paid"), path assert action.help and action.help.endswith(f" [{cost}]"), (path, action.help) seen.add(path) - # A leaf whose parser has no help= is not in _choices_actions at all, so it must be named: - # a count would be made up by the root and group lines. `run` and `compare` take the - # table's summary in the top-level list instead (the test below). + # A leaf with no help= is not in _choices_actions, so it is named, not counted: a count + # would include the root and group lines. `run` and `compare`: see the test below. assert {leaf.path for leaf in cli_routes.LEAVES if leaf.summary is None} <= seen From dd601101b391b710fb75b9ab07a76dc6728f76e0 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:17:11 +0000 Subject: [PATCH 15/19] test(cli): pin each root's help section with a literal Signed-off-by: Anton Volkov --- dokimi/tests/test_cli_routes.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index fe9d73ff1..6710dbe8b 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -71,7 +71,30 @@ def test_help_lists_each_root_under_its_section(): elif section and re.match(r"^ \S", line): listed[line.split()[0]] = section assert order == list(cli_routes.SECTIONS) - assert listed == {root.name: root.section for root in cli_routes.ROOTS} + loop = "The loop" + subjects = "Subjects" + evals = "Evals" + data = "Analysis and data" + assert listed == { + **dict.fromkeys( + ( + "evaluate", + "baseline", + "report", + "answers", + "benchmark", + "dashboard", + "arms", + "subjects", + "tasks", + "explore", + ), + loop, + ), + **dict.fromkeys(("run", "compare", "library", "catalog", "novelty", "repo"), subjects), + **dict.fromkeys(("personas", "questions", "eval"), evals), + **dict.fromkeys(("dataset", "ingest", "tools", "context", "data"), data), + } # The commands that can make a model call. Written out here, not read from cli_routes, so a From 3b224b65d9a50445b49eb73f10cb4f28f027a104 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 22:42:50 -0700 Subject: [PATCH 16/19] docs(cli): describe the data schema group, not its one kind Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/data.py b/dokimi/dokimi/commands/data.py index 87cd2ea31..a727d2fb9 100644 --- a/dokimi/dokimi/commands/data.py +++ b/dokimi/dokimi/commands/data.py @@ -31,7 +31,7 @@ def register(sub, positive_int): # noqa: ARG001 - the registry passes it to eve data_p = sub.add_parser("data", help="Schemas: today the CLI route table") data_sub = data_p.add_subparsers(dest="data_cmd", required=True) - schema_p = data_sub.add_parser("schema", help="Print the CLI route table as JSON") + schema_p = data_sub.add_parser("schema", help="List and print schemas: today the CLI route table") schema_sub = schema_p.add_subparsers(dest="schema_cmd", required=True) list_p = schema_sub.add_parser("list", help="List the schema kinds") From 1eeeae8098893f28a61681523db946397ff90780 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:13:13 -0700 Subject: [PATCH 17/19] test(cli): an unknown schema kind exits 2 before the handler runs Signed-off-by: Nikolay Petrov --- dokimi/tests/test_cli_routes.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 6710dbe8b..cf80f27b3 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -305,6 +305,12 @@ def test_data_schema_list_names_the_kinds(capsys): assert capsys.readouterr().out.split() == ["cli_routes"] +def test_data_schema_show_rejects_an_unknown_kind(): + with pytest.raises(SystemExit) as exc: + build_parser().parse_args(["data", "schema", "show", "nope"]) + assert exc.value.code == 2 + + def test_the_route_document_validates_against_its_schema(): assert schema_errors("cli_routes.v1", cli_routes.document()) == [] From f551d36236c051c8bdeb10230da87e3d94372b59 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:13:13 -0700 Subject: [PATCH 18/19] docs(cli): say what the data and schema groups hold, not one kind Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/data.py b/dokimi/dokimi/commands/data.py index a727d2fb9..db0601a65 100644 --- a/dokimi/dokimi/commands/data.py +++ b/dokimi/dokimi/commands/data.py @@ -28,10 +28,10 @@ def cmd_schema_show(args: argparse.Namespace) -> None: def register(sub, positive_int): # noqa: ARG001 - the registry passes it to every command - data_p = sub.add_parser("data", help="Schemas: today the CLI route table") + data_p = sub.add_parser("data", help="Schemas: list and print them (today the CLI route table)") data_sub = data_p.add_subparsers(dest="data_cmd", required=True) - schema_p = data_sub.add_parser("schema", help="List and print schemas: today the CLI route table") + schema_p = data_sub.add_parser("schema", help="List schema kinds and print one as JSON (today the CLI route table)") schema_sub = schema_p.add_subparsers(dest="schema_cmd", required=True) list_p = schema_sub.add_parser("list", help="List the schema kinds") From 9a7971366b4f3fbec3806246873f18804866ef6e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:13:13 -0700 Subject: [PATCH 19/19] chore(cli): regenerate the CLI snapshot on current main Signed-off-by: Nikolay Petrov --- dokimi/docs/reference/cli-interface.json | 116 +++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 1bbc56081..e881613cf 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -3512,6 +3512,122 @@ } } }, + "data": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi data", + "subcommand_required": true, + "subcommands": { + "schema": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi data schema", + "subcommand_required": true, + "subcommands": { + "list": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi data schema list", + "subcommand_required": null, + "subcommands": {} + }, + "show": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": [ + "'cli_routes'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "kind", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi data schema show", + "subcommand_required": null, + "subcommands": {} + } + } + } + } + }, "dataset": { "allow_abbrev": true, "params": [