diff --git a/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md b/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md new file mode 100644 index 000000000..6528255cd --- /dev/null +++ b/dokimi/changelog.d/20261028_120000_napetrov_cli-routes-table.md @@ -0,0 +1,18 @@ +### Added + +- **CLI route table** — the top-level `dokimi --help` lists the commands by + section (The loop, Subjects, Evals, Analysis and data), and every command's + help line ends in `[free]` or `[paid]`: `paid` when the command can make a + model call in some flag combination. The table lives in `dokimi/cli_routes.py`; + `dokimi data schema show cli_routes` prints it as JSON, and + `schemas/cli_routes.v1.json` pins its shape. +- **`dokimi data schema list|show KIND`** — the new `data` command group; `show` + prints a schema kind as JSON. `cli_routes` is the only kind for now. +- **Tombstones** — the route table can name an old argv prefix that moved; running + it prints `moved: dokimi ` and exits 2 without running anything. + No command has moved yet, so the table is empty. + +### Changed + +- `dokimi run` and `dokimi compare` have a one-line help in the top-level list. + Their parsers have no help text, so the route table supplies it. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 1bbc56081..e881613cf 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -3512,6 +3512,122 @@ } } }, + "data": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi data", + "subcommand_required": true, + "subcommands": { + "schema": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi data schema", + "subcommand_required": true, + "subcommands": { + "list": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi data schema list", + "subcommand_required": null, + "subcommands": {} + }, + "show": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": [ + "'cli_routes'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "kind", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi data schema show", + "subcommand_required": null, + "subcommands": {} + } + } + } + } + }, "dataset": { "allow_abbrev": true, "params": [ diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 5731c0f46..0088ff5eb 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -57,6 +57,7 @@ "result": "result.v1", "repo_budget": "repo_budget.v1", "task_inspect": "task_inspect.v1", + "cli_routes": "cli_routes.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index 32ee6e64c..7e77d1823 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -19,6 +19,7 @@ catalog, context, dashboard, + data, dataset, evaluate, explore, @@ -35,11 +36,71 @@ tasks, tools, ) -from dokimi import __version__, process +from dokimi import __version__, cli_routes, process from dokimi.console import tolerate_console_encoding from dokimi.defaults import JUDGE_MODEL_FOR_PROVIDER, default_judge_model +def _subparsers(parser: argparse.ArgumentParser) -> argparse._SubParsersAction | None: + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + return action + return None + + +def _route_help(text: str | None, path: tuple[str, ...]) -> str: + """The one-line help of a route with its cost label: the parser's text, else the table's.""" + base = text or cli_routes.summary_of(path) or "" + cost = cli_routes.cost_of(path) + return " ".join(part for part in (base, f"[{cost}]" if cost else "") if part) + + +def _label_routes(parser: argparse.ArgumentParser, prefix: tuple[str, ...] = ()) -> None: + """Append the cost label to every subcommand's help line, at every depth.""" + sub = _subparsers(parser) + if sub is None: + return + listed = {action.dest: action for action in sub._choices_actions} + visited: set[int] = set() + for name, child in sub.choices.items(): + if id(child) in visited: # an alias: the same parser under a second name + continue + visited.add(id(child)) + path = (*prefix, name) + if name in listed: + listed[name].help = _route_help(listed[name].help, path) + _label_routes(child, path) + + +class _RoutedParser(argparse.ArgumentParser): + """The top-level parser. Its ``--help`` lists the roots by section (``dokimi.cli_routes``).""" + + def format_help(self) -> str: + sub = _subparsers(self) + listed = {action.dest: action for action in sub._choices_actions} if sub else {} + fmt = self._get_formatter() + fmt.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) + # A parser root missing from cli_routes.ROOTS is left out of this list; + # test_route_table_matches_the_parser_leaves_and_roots fails on it. + for section in cli_routes.SECTIONS: + roots = [root for root in cli_routes.ROOTS if root.section == section] + if not roots: + continue + fmt.start_section(section) # the formatter adds the colon + for root in roots: + action = listed.get(root.name) + if action is None: # a route with no help= in its parser: listed from the table + action = argparse.Action([], root.name, help=_route_help(None, (root.name,))) + if root.aliases: + action.metavar = f"{root.name} ({', '.join(root.aliases)})" + fmt.add_argument(action) + fmt.end_section() + fmt.start_section("options") + fmt.add_arguments([action for action in self._actions if action.option_strings]) + fmt.end_section() + return fmt.format_help() + + def build_parser() -> argparse.ArgumentParser: """Build CLI argument parser.""" @@ -53,9 +114,9 @@ def positive_int(value: str) -> int: raise argparse.ArgumentTypeError(f"must be >= 1, got {value}") return ivalue - p = argparse.ArgumentParser(prog="dokimi") + p = _RoutedParser(prog="dokimi") p.add_argument("--version", action="version", version=f"%(prog)s {__version__}") - sub = p.add_subparsers(dest="cmd", required=True) + sub = p.add_subparsers(dest="cmd", required=True, parser_class=argparse.ArgumentParser) # Order preserved to keep --help output stable. run.register(sub, positive_int) # run @@ -81,7 +142,9 @@ def positive_int(value: str) -> int: repo.register(sub, positive_int) # repo (repositories as subjects) tools.register(sub, positive_int) # tools (tool-choice probes) context.register(sub, positive_int) # context (context window of a run) + data.register(sub, positive_int) # data (schema; M2 adds export, import and models) + _label_routes(p) return p @@ -92,7 +155,11 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: read once every flag is parsed, so the two flags can come in either order. """ parser = build_parser() - args = parser.parse_args(argv) + raw = list(sys.argv[1:] if argv is None else argv) + moved = cli_routes.moved_argv(raw) + if moved is not None: # a tombstone: say where the command went, do not run anything + parser.exit(2, cli_routes.moved_text(moved) + "\n") + args = parser.parse_args(raw) if getattr(args, "judge_model", None) is JUDGE_MODEL_FOR_PROVIDER: model = default_judge_model(args.judge_provider) if model is None: diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py new file mode 100644 index 000000000..d6ae4daf2 --- /dev/null +++ b/dokimi/dokimi/cli_routes.py @@ -0,0 +1,273 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The CLI route table: where each root and leaf is listed in ``dokimi --help``, whether it +can spend money, and which old paths are tombstones. + +``dokimi.cli`` reads the table to build the sectioned help and the ``[free]``/``[paid]`` +labels, and ``dokimi data schema show cli_routes`` prints it. Help text stays with each +command's parser; the table gives a one-line help only for a route whose parser has none. + +Cost: ``paid`` means the route can make a model call (an LLM, or an agent harness) in some +flag combination, and ``free`` means it never does. A root is ``paid`` when any leaf under +it is. The sections follow issue #94 §2.1. The table describes today's routes: a root sits in +the section of the operation it will hold once the renames of #94 §4 land. +""" + +from __future__ import annotations + +import shlex +from dataclasses import dataclass + +SECTIONS: tuple[str, ...] = ("The loop", "Subjects", "Evals", "Analysis and data") + +FREE = "free" +PAID = "paid" + + +@dataclass(frozen=True) +class Root: + """A top-level name. ``aliases`` are alternative names for it.""" + + name: str + section: str + aliases: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Leaf: + """An operation: a parser with no subcommands. ``path`` is its argv, e.g. ``("eval", "score")``.""" + + path: tuple[str, ...] + cost: str + summary: str | None = None # one-line help, used only when the parser has none + + +@dataclass(frozen=True) +class Tombstone: + """An old argv prefix that now moved: it answers with ``new`` and exits 2, never forwards.""" + + old: tuple[str, ...] + new: tuple[str, ...] + + +ROOTS: tuple[Root, ...] = ( + Root("evaluate", "The loop"), + Root("baseline", "The loop"), + Root("report", "The loop"), + Root("answers", "The loop"), + Root("benchmark", "The loop"), + Root("dashboard", "The loop"), + Root("arms", "The loop"), + Root("subjects", "The loop"), + Root("tasks", "The loop"), + Root("explore", "The loop"), + Root("run", "Subjects"), + Root("compare", "Subjects"), + Root("library", "Subjects", aliases=("product",)), + Root("catalog", "Subjects"), + Root("novelty", "Subjects"), + Root("repo", "Subjects"), + Root("personas", "Evals"), + Root("questions", "Evals"), + Root("eval", "Evals"), + Root("dataset", "Analysis and data"), + Root("ingest", "Analysis and data"), + Root("tools", "Analysis and data"), + Root("context", "Analysis and data"), + Root("data", "Analysis and data"), +) + +LEAVES: tuple[Leaf, ...] = ( + Leaf(("evaluate",), PAID), + Leaf(("baseline", "save"), FREE), + Leaf(("baseline", "list"), FREE), + Leaf(("baseline", "compare"), FREE), + Leaf(("report", "eval"), FREE), + Leaf(("report", "generate"), FREE), + Leaf(("report", "model-compare"), FREE), + Leaf(("answers", "generate"), PAID), + Leaf(("benchmark", "run"), PAID), + Leaf(("benchmark", "preflight"), FREE), + Leaf(("benchmark", "batch"), PAID), + Leaf(("dashboard", "generate"), FREE), + Leaf(("arms", "run"), PAID), + Leaf(("arms", "matrix-run"), PAID), + Leaf(("arms", "plugin-delta"), FREE), + Leaf(("subjects", "list"), FREE), + Leaf(("subjects", "show"), FREE), + Leaf(("subjects", "run"), PAID), + Leaf(("tasks", "list"), FREE), + Leaf(("tasks", "inspect"), FREE), + Leaf(("tasks", "harnesses"), FREE), + Leaf(("tasks", "run"), PAID), + Leaf(("explore", "list"), FREE), + Leaf(("explore", "run"), PAID), + Leaf(("run",), FREE, "Run the static benchmark spec and save a snapshot"), + Leaf(("compare",), FREE, "Compare two saved benchmark snapshots"), + Leaf(("library", "list"), FREE), + Leaf(("library", "show"), FREE), + Leaf(("library", "intents"), FREE), + Leaf(("catalog", "scan"), FREE), + Leaf(("catalog", "formats"), FREE), + Leaf(("catalog", "trigger-set"), PAID), + Leaf(("catalog", "trigger-probe"), PAID), + Leaf(("novelty", "analyze"), PAID), + Leaf(("novelty", "atoms"), PAID), + Leaf(("novelty", "probe"), PAID), + Leaf(("novelty", "shorten"), PAID), + Leaf(("novelty", "routing"), PAID), + Leaf(("novelty", "items"), FREE), + Leaf(("novelty", "view"), FREE), + Leaf(("repo", "validate"), FREE), + Leaf(("repo", "scan"), FREE), + Leaf(("repo", "inspect"), FREE), + Leaf(("repo", "fetch"), FREE), + Leaf(("repo", "reach"), PAID), + Leaf(("repo", "budget"), FREE), + Leaf(("repo", "tasks", "check"), FREE), + Leaf(("repo", "tasks", "replay"), FREE), + Leaf(("repo", "run"), PAID), + Leaf(("repo", "report"), FREE), + Leaf(("repo", "compare"), FREE), + Leaf(("personas", "discover"), PAID), + Leaf(("personas", "approve"), FREE), + Leaf(("questions", "generate"), PAID), + Leaf(("questions", "analyze"), PAID), + Leaf(("questions", "refine"), PAID), + Leaf(("questions", "panel-review"), PAID), + Leaf(("questions", "import"), FREE), + Leaf(("eval", "score"), PAID), + Leaf(("eval", "ragas"), PAID), + Leaf(("eval", "grounding"), FREE), + Leaf(("eval", "judge-validity"), PAID), + Leaf(("eval", "checklist"), PAID), + Leaf(("eval", "judge-wording"), PAID), + Leaf(("eval", "cross-judge"), PAID), + Leaf(("eval", "pairwise"), PAID), + Leaf(("eval", "reference"), PAID), + Leaf(("eval", "native-import"), FREE), + Leaf(("eval", "panel-score"), PAID), + Leaf(("dataset", "export"), FREE), + Leaf(("ingest", "docling"), FREE), + Leaf(("tools", "preference"), PAID), + Leaf(("context", "trace"), FREE), + Leaf(("context", "consistency"), FREE), + Leaf(("data", "schema", "list"), FREE), + Leaf(("data", "schema", "show"), FREE), +) + +# No command has moved yet: the renames of #94 §5 M1-M3 add entries here. +TOMBSTONES: tuple[Tombstone, ...] = () + +SCHEMA_VERSION = "cli_routes.v1" + + +def _leaf(path: tuple[str, ...]) -> Leaf | None: + return next((leaf for leaf in LEAVES if leaf.path == path), None) + + +def cost_of(path: tuple[str, ...]) -> str | None: + """``free`` or ``paid`` for a leaf or a root; None for a route the table does not know.""" + leaf = _leaf(path) + if leaf is not None: + return leaf.cost + under = [leaf for leaf in LEAVES if leaf.path[: len(path)] == path] + if not under: + return None + return PAID if any(leaf.cost == PAID for leaf in under) else FREE + + +def summary_of(path: tuple[str, ...]) -> str | None: + """The table's fallback one-line help for a route whose parser has none.""" + leaf = _leaf(path) + return leaf.summary if leaf is not None else None + + +def _canonical(path: tuple[str, ...]) -> tuple[str, ...]: + """``path`` with its first word replaced by the root name when that word is an alias.""" + if not path: + return path + head = next((root.name for root in ROOTS if path[0] in (root.name, *root.aliases)), path[0]) + return (head, *path[1:]) + + +def moved_argv( + argv: list[str], tombstones: tuple[Tombstone, ...] | None = None +) -> list[str] | None: + """The argv a tombstone rewrites ``argv`` to, or None when no tombstone matches its start. + + An alias typed on the command line (``product`` for ``library``) matches the root's tombstones. + """ + canon = _canonical(tuple(argv)) + for tomb in TOMBSTONES if tombstones is None else tombstones: + if canon[: len(tomb.old)] == _canonical(tomb.old): + return [*tomb.new, *argv[len(tomb.old) :]] + return None + + +def moved_text(argv: list[str]) -> str: + """The line a tombstone prints: the new command with the argv rewritten.""" + return "moved: " + shlex.join(["dokimi", *argv]) + + +def _resolve(path: tuple[str, ...]) -> tuple[str, ...] | None: + """``path`` with its first word resolved from an alias to its root; None if no root has it.""" + if not path: + return None + heads = {name: root.name for root in ROOTS for name in (root.name, *root.aliases)} + head = heads.get(path[0]) + return None if head is None else (head, *path[1:]) + + +def _is_route(path: tuple[str, ...]) -> bool: + """True when argv ``path`` names a root or an alias, or a group or leaf under one. + + ``moved_argv`` matches by prefix, so a tombstone on a group shadows every route under it. + """ + full = _resolve(path) + return full is not None and any(leaf.path[: len(full)] == full for leaf in LEAVES) + + +def _extends_a_leaf(path: tuple[str, ...]) -> bool: + """True when ``path`` is longer than a live leaf it starts with: the leaf takes it as arguments.""" + full = _resolve(path) + return full is not None and any( + len(leaf.path) < len(full) and full[: len(leaf.path)] == leaf.path for leaf in LEAVES + ) + + +def tombstone_problems(tombstones: tuple[Tombstone, ...] | None = None) -> list[str]: + """Why a tombstone is wrong: its old path is still a live route, or it points nowhere.""" + problems = [] + for tomb in TOMBSTONES if tombstones is None else tombstones: + if not tomb.old or tomb.old[0].startswith("-"): # an empty path matches every argv + problems.append(f"{' '.join(tomb.old) or '(empty)'} is not a command path") + if _is_route(tomb.old): + problems.append(f"{' '.join(tomb.old)} is still a live route") + if _extends_a_leaf(tomb.old): + problems.append(f"{' '.join(tomb.old)} extends a live route and would shadow it") + if not _is_route(tomb.new): + problems.append( + f"{' '.join(tomb.old)} points at {' '.join(tomb.new)}, which is not a route" + ) + return problems + + +def document() -> dict: + """The table as JSON, for ``dokimi data schema show cli_routes``.""" + return { + "schema_version": SCHEMA_VERSION, + "sections": list(SECTIONS), + "roots": [ + { + "name": root.name, + "section": root.section, + "aliases": list(root.aliases), + "cost": cost_of((root.name,)), + } + for root in ROOTS + ], + "leaves": [{"path": list(leaf.path), "cost": leaf.cost} for leaf in LEAVES], + "tombstones": [{"old": list(tomb.old), "new": list(tomb.new)} for tomb in TOMBSTONES], + } diff --git a/dokimi/dokimi/commands/data.py b/dokimi/dokimi/commands/data.py new file mode 100644 index 000000000..db0601a65 --- /dev/null +++ b/dokimi/dokimi/commands/data.py @@ -0,0 +1,42 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""data: schemas. ``data schema show KIND`` prints one as JSON; ``cli_routes`` is the only kind so far.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Callable + +from dokimi import cli_routes + +SCHEMA_KINDS: dict[str, Callable[[], dict]] = { + "cli_routes": cli_routes.document, +} + + +def cmd_schema_list(args: argparse.Namespace) -> None: + """Print the schema kinds, one per line.""" + for kind in SCHEMA_KINDS: + print(kind) + + +def cmd_schema_show(args: argparse.Namespace) -> None: + """Print one schema kind as JSON.""" + print(json.dumps(SCHEMA_KINDS[args.kind](), indent=2)) + + +def register(sub, positive_int): # noqa: ARG001 - the registry passes it to every command + data_p = sub.add_parser("data", help="Schemas: list and print them (today the CLI route table)") + data_sub = data_p.add_subparsers(dest="data_cmd", required=True) + + schema_p = data_sub.add_parser("schema", help="List schema kinds and print one as JSON (today the CLI route table)") + schema_sub = schema_p.add_subparsers(dest="schema_cmd", required=True) + + list_p = schema_sub.add_parser("list", help="List the schema kinds") + list_p.set_defaults(func=cmd_schema_list) + + show_p = schema_sub.add_parser("show", help="Print one schema kind as JSON") + show_p.add_argument("kind", choices=sorted(SCHEMA_KINDS)) + show_p.set_defaults(func=cmd_schema_show) diff --git a/dokimi/dokimi/schemas/cli_routes.v1.json b/dokimi/dokimi/schemas/cli_routes.v1.json new file mode 100644 index 000000000..de1fd3141 --- /dev/null +++ b/dokimi/dokimi/schemas/cli_routes.v1.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/cli_routes.v1.json", + "title": "CLI route table v1", + "description": "The routes of the dokimi CLI as `dokimi data schema show cli_routes` prints them (dokimi.cli_routes.document): the help sections, every top-level command with its section, aliases and cost, every leaf operation with its cost, and the tombstones (old argv prefixes that moved). `cost` is `paid` when the route can make a model call in some flag combination, else `free`.", + "type": "object", + "required": ["schema_version", "sections", "roots", "leaves", "tombstones"], + "additionalProperties": false, + "$defs": { + "cost": {"enum": ["free", "paid"]}, + "path": { + "type": "array", + "minItems": 1, + "items": {"type": "string", "minLength": 1}, + "description": "An argv prefix, e.g. [\"eval\", \"score\"]." + } + }, + "properties": { + "schema_version": {"const": "cli_routes.v1"}, + "sections": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + "description": "The headings of `dokimi --help`, in order." + }, + "roots": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "section", "aliases", "cost"], + "additionalProperties": false, + "properties": { + "name": {"type": "string", "minLength": 1}, + "section": {"type": "string", "minLength": 1, "description": "One of `sections`."}, + "aliases": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "cost": {"$ref": "#/$defs/cost", "description": "paid when any leaf under the root is paid."} + } + } + }, + "leaves": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "cost"], + "additionalProperties": false, + "properties": { + "path": {"$ref": "#/$defs/path"}, + "cost": {"$ref": "#/$defs/cost"} + } + } + }, + "tombstones": { + "type": "array", + "items": { + "type": "object", + "required": ["old", "new"], + "additionalProperties": false, + "properties": { + "old": {"$ref": "#/$defs/path"}, + "new": {"$ref": "#/$defs/path"} + } + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/cli_routes.v1.json b/dokimi/tests/fixtures/artifacts/cli_routes.v1.json new file mode 100644 index 000000000..270924b38 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/cli_routes.v1.json @@ -0,0 +1,15 @@ +{ + "schema_version": "cli_routes.v1", + "sections": ["The loop", "Subjects"], + "roots": [ + {"name": "evaluate", "section": "The loop", "aliases": [], "cost": "paid"}, + {"name": "library", "section": "Subjects", "aliases": ["product"], "cost": "free"} + ], + "leaves": [ + {"path": ["evaluate"], "cost": "paid"}, + {"path": ["library", "list"], "cost": "free"} + ], + "tombstones": [ + {"old": ["old-tool", "inspect"], "new": ["library", "list"]} + ] +} diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py new file mode 100644 index 000000000..cf80f27b3 --- /dev/null +++ b/dokimi/tests/test_cli_routes.py @@ -0,0 +1,353 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The CLI route table: sections, cost labels, tombstones, and ``data schema``.""" + +from __future__ import annotations + +import argparse +import json +import re + +import pytest + +from dokimi import cli_routes +from dokimi.artifacts import schema_errors +from dokimi.cli import build_parser, parse_args + +SECTION_HEADING = re.compile(r"^(%s):$" % "|".join(re.escape(s) for s in cli_routes.SECTIONS)) + + +def _subparsers(parser: argparse.ArgumentParser) -> argparse._SubParsersAction | None: + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + return action + return None + + +def _leaf_paths( + parser: argparse.ArgumentParser, prefix: tuple[str, ...] = () +) -> set[tuple[str, ...]]: + """Paths of every parser that has no subcommands, aliases resolved to their parser once.""" + found: set[tuple[str, ...]] = set() + sub = _subparsers(parser) + if sub is None: + return {prefix} if prefix else set() + seen: set[int] = set() + for name, child in sub.choices.items(): + if id(child) in seen: + continue + seen.add(id(child)) + found |= _leaf_paths(child, (*prefix, name)) + return found + + +def _help_actions(parser: argparse.ArgumentParser, prefix: tuple[str, ...] = ()): + """(path, pseudo action) for every subcommand that has a help line, at every depth.""" + sub = _subparsers(parser) + if sub is None: + return + for action in sub._choices_actions: + yield (*prefix, action.dest), action + seen: set[int] = set() + for name, child in sub.choices.items(): + if id(child) in seen: # an alias + continue + seen.add(id(child)) + yield from _help_actions(child, (*prefix, name)) + + +def test_help_lists_each_root_under_its_section(): + text = build_parser().format_help() + listed: dict[str, str] = {} + section = None + order: list[str] = [] + for line in text.splitlines(): + if SECTION_HEADING.match(line): + section = line[:-1] + order.append(section) + elif line == "options:": + section = None + elif section and re.match(r"^ \S", line): + listed[line.split()[0]] = section + assert order == list(cli_routes.SECTIONS) + loop = "The loop" + subjects = "Subjects" + evals = "Evals" + data = "Analysis and data" + assert listed == { + **dict.fromkeys( + ( + "evaluate", + "baseline", + "report", + "answers", + "benchmark", + "dashboard", + "arms", + "subjects", + "tasks", + "explore", + ), + loop, + ), + **dict.fromkeys(("run", "compare", "library", "catalog", "novelty", "repo"), subjects), + **dict.fromkeys(("personas", "questions", "eval"), evals), + **dict.fromkeys(("dataset", "ingest", "tools", "context", "data"), data), + } + + +# The commands that can make a model call. Written out here, not read from cli_routes, so a +# label that says `free` on a command that spends money (or the reverse) fails a test. +PAID_PATHS = { + ("evaluate",), + ("answers", "generate"), + ("benchmark", "run"), + ("benchmark", "batch"), + ("arms", "run"), + ("arms", "matrix-run"), + ("subjects", "run"), + ("tasks", "run"), + ("explore", "run"), + ("catalog", "trigger-set"), + ("catalog", "trigger-probe"), + ("novelty", "analyze"), + ("novelty", "atoms"), + ("novelty", "probe"), + ("novelty", "shorten"), + ("novelty", "routing"), + ("repo", "reach"), + ("repo", "run"), + ("personas", "discover"), + ("questions", "generate"), + ("questions", "analyze"), + ("questions", "refine"), + ("questions", "panel-review"), + ("eval", "score"), + ("eval", "ragas"), + ("eval", "judge-validity"), + ("eval", "checklist"), + ("eval", "judge-wording"), + ("eval", "cross-judge"), + ("eval", "pairwise"), + ("eval", "reference"), + ("eval", "panel-score"), + ("tools", "preference"), +} + + +def test_the_paid_leaves_are_exactly_the_ones_written_out_here(): + assert {leaf.path for leaf in cli_routes.LEAVES if leaf.cost == "paid"} == PAID_PATHS + + +def test_the_help_label_of_a_route_comes_from_its_literal_cost(): + top = build_parser() + for path, action in _help_actions(top): + paid = any(p[: len(path)] == path for p in PAID_PATHS) + assert action.help.endswith(" [paid]" if paid else " [free]"), (path, action.help) + + +def test_every_route_help_line_ends_in_its_cost_label(): + top = build_parser() + seen = set() + for path, action in _help_actions(top): + cost = cli_routes.cost_of(path) + assert cost in ("free", "paid"), path + assert action.help and action.help.endswith(f" [{cost}]"), (path, action.help) + seen.add(path) + # A leaf with no help= is not in _choices_actions, so it is named, not counted: a count + # would include the root and group lines. `run` and `compare`: see the test below. + assert {leaf.path for leaf in cli_routes.LEAVES if leaf.summary is None} <= seen + + +def test_run_and_compare_have_a_one_line_help_in_the_top_level_list(monkeypatch): + # argparse wraps help at COLUMNS (or the terminal width, else 80): a narrow host would + # wrap the cost label onto a continuation line, so pin the width the label is read at. + monkeypatch.setenv("COLUMNS", "200") + text = build_parser().format_help() + for name in ("run", "compare"): + line = next(line for line in text.splitlines() if re.match(rf"^ {name}\s", line)) + assert re.search(r"\S.* \[(free|paid)\]$", line), line + + +def test_route_table_matches_the_parser_leaves_and_roots(): + top = build_parser() + assert _leaf_paths(top) == {leaf.path for leaf in cli_routes.LEAVES} + sub = _subparsers(top) + assert sub is not None + names = set(sub.choices) + aliases = {alias for root in cli_routes.ROOTS for alias in root.aliases} + assert names - aliases == {root.name for root in cli_routes.ROOTS} + assert aliases == {"product"} + + +def test_a_root_costs_paid_when_any_of_its_leaves_is_paid(): + assert cli_routes.cost_of(("repo",)) == "paid" + assert cli_routes.cost_of(("library",)) == "free" + assert cli_routes.cost_of(("data",)) == "free" + assert cli_routes.cost_of(("eval", "judge-validity")) == "paid" + assert cli_routes.cost_of(("no", "such", "route")) is None + + +def test_the_tombstone_table_is_empty_and_its_check_finds_a_live_route(): + assert cli_routes.TOMBSTONES == () + assert cli_routes.tombstone_problems() == [] + bad = (cli_routes.Tombstone(("eval", "score"), ("eval", "inspect")),) + assert cli_routes.tombstone_problems(bad) # the old path is still a live route + + +def test_the_tombstone_check_follows_groups_aliases_and_roots(): + # moved_argv matches by prefix: a group tombstone shadows the live routes under it. + shadows = (cli_routes.Tombstone(("repo", "tasks"), ("tasks", "run")),) + assert cli_routes.tombstone_problems(shadows) + # an alias is a live route too + alias = (cli_routes.Tombstone(("product",), ("library", "list")),) + assert cli_routes.tombstone_problems(alias) + # a root is a valid destination, and an unknown name is a valid old path + root = (cli_routes.Tombstone(("old-tool",), ("library",)),) + assert cli_routes.tombstone_problems(root) == [] + nowhere = (cli_routes.Tombstone(("old-tool",), ("no-such-root",)),) + assert cli_routes.tombstone_problems(nowhere) + # compare takes positional files, so "compare x" is a live invocation, not a free old path + extends = (cli_routes.Tombstone(("compare", "x"), ("compare",)),) + assert cli_routes.tombstone_problems(extends) + # an empty old path matches every argv; an option is not a command + assert cli_routes.tombstone_problems((cli_routes.Tombstone((), ("data",)),)) + assert cli_routes.tombstone_problems((cli_routes.Tombstone(("--version",), ("data",)),)) + + +def test_a_tombstone_matches_an_alias_typed_on_the_command_line(): + tombs = (cli_routes.Tombstone(("library", "old-intents"), ("library", "intents")),) + assert cli_routes.moved_argv(["product", "old-intents"], tombs) == ["library", "intents"] + assert cli_routes.moved_argv(["library", "old-intents", "x"], tombs) == [ + "library", + "intents", + "x", + ] + assert cli_routes.moved_argv(["library", "list"], tombs) is None + + +def test_a_tombstone_keeps_the_arguments_after_the_old_path_whatever_the_lengths(): + longer_old = (cli_routes.Tombstone(("product", "old-intents", "all"), ("library", "intents")),) + assert cli_routes.moved_argv(["product", "old-intents", "all", "x"], longer_old) == [ + "library", + "intents", + "x", + ] + longer_new = (cli_routes.Tombstone(("product", "old"), ("library", "show", "intents")),) + assert cli_routes.moved_argv(["product", "old", "x"], longer_new) == [ + "library", + "show", + "intents", + "x", + ] + + +def test_an_empty_argv_matches_no_tombstone(): + tombs = (cli_routes.Tombstone(("old-tool",), ("library",)),) + assert cli_routes.moved_argv([], tombs) is None + with pytest.raises(SystemExit) as exc: # argparse still asks for a command + parse_args([]) + assert exc.value.code == 2 + + +def test_every_alias_points_at_its_own_root_parser(): + sub = _subparsers(build_parser()) + assert sub is not None + for root in cli_routes.ROOTS: + for alias in root.aliases: + assert sub.choices[alias] is sub.choices[root.name], (alias, root.name) + + +def test_a_tombstone_rewrites_the_argv_and_exits_2(monkeypatch, capsys): + # Test-only entry: no command moves in this release. + monkeypatch.setattr( + cli_routes, + "TOMBSTONES", + (cli_routes.Tombstone(("old-tool", "inspect"), ("eval", "inspect")),), + ) + with pytest.raises(SystemExit) as exc: + parse_args(["old-tool", "inspect", "data.json"]) + assert exc.value.code == 2 + assert "moved: dokimi eval inspect data.json" in capsys.readouterr().err + + +def test_an_argv_that_no_tombstone_matches_is_parsed_as_before(): + args = parse_args(["data", "schema", "list"]) + assert args.cmd == "data" + + +def test_data_schema_show_prints_the_route_table_as_json(capsys): + args = parse_args(["data", "schema", "show", "cli_routes"]) + args.func(args) + doc = json.loads(capsys.readouterr().out) + assert doc == cli_routes.document() + assert doc["schema_version"] == "cli_routes.v1" + assert {"path": ["eval", "score"], "cost": "paid"} in doc["leaves"] + + +def test_the_route_document_reports_each_roots_section_aliases_and_cost(): + doc = cli_routes.document() + assert doc["sections"] == ["The loop", "Subjects", "Evals", "Analysis and data"] + roots = {root["name"]: root for root in doc["roots"]} + assert roots["library"] == { + "name": "library", "section": "Subjects", "aliases": ["product"], "cost": "free" + } + assert roots["repo"] == {"name": "repo", "section": "Subjects", "aliases": [], "cost": "paid"} + assert roots["data"]["section"] == "Analysis and data" + assert roots["evaluate"]["cost"] == "paid" + assert {leaf["cost"] for leaf in doc["leaves"]} == {"free", "paid"} + + +def test_data_schema_list_names_the_kinds(capsys): + args = parse_args(["data", "schema", "list"]) + args.func(args) + assert capsys.readouterr().out.split() == ["cli_routes"] + + +def test_data_schema_show_rejects_an_unknown_kind(): + with pytest.raises(SystemExit) as exc: + build_parser().parse_args(["data", "schema", "show", "nope"]) + assert exc.value.code == 2 + + +def test_the_route_document_validates_against_its_schema(): + assert schema_errors("cli_routes.v1", cli_routes.document()) == [] + + +@pytest.mark.parametrize( + "damage", + [ + lambda doc: doc["roots"][0].pop("cost"), + lambda doc: doc["leaves"][0].update(cost="cheap"), + lambda doc: doc["leaves"][0].pop("path"), + lambda doc: doc.pop("tombstones"), + lambda doc: doc.update(schema_version="cli_routes.v2"), + lambda doc: doc["roots"][0].update(extra=1), + ], + ids=["root-no-cost", "bad-cost", "leaf-no-path", "no-tombstones", "wrong-version", "extra-key"], +) +def test_the_route_schema_rejects_a_document_whose_shape_drifted(damage): + doc = cli_routes.document() + damage(doc) + assert schema_errors("cli_routes.v1", doc) + + +def test_the_data_help_promises_only_the_kinds_it_has(): + from dokimi.commands.data import SCHEMA_KINDS + + helps = {path: action.help for path, action in _help_actions(build_parser())} + assert set(SCHEMA_KINDS) == {"cli_routes"} # widen the help below when a kind is added + for path in (("data",), ("data", "schema")): + assert "artifact" not in helps[path].lower(), helps[path] + assert "route" in helps[path].lower(), helps[path] + + +def test_only_the_top_level_parser_lists_its_commands_by_section(): + top = build_parser() + assert "The loop:" in top.format_help() + sub = _subparsers(top) + assert sub is not None + for name, child in sub.choices.items(): + assert type(child) is argparse.ArgumentParser, name + assert "The loop:" not in child.format_help(), name diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index ebeb4eb10..4a60f6315 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -44,7 +44,7 @@ #: its own layer and any layer below it. LAYERS: tuple[tuple[str, tuple[str, ...]], ...] = ( # The argparse front end: parses flags, calls into the tracks, prints. - ("frontend", ("dokimi.__main__", "dokimi.cli", "dokimi.commands")), + ("frontend", ("dokimi.__main__", "dokimi.cli", "dokimi.cli_routes", "dokimi.commands")), # Whole tracks and the tools beside them: static run, novelty, executable tasks, # repositories, subjects, reports, dashboards, the one-command pipeline. ("tracks", (