From 610b2ef3e7d690450e0f6f9695ce71451fb96469 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 21:04:15 -0700 Subject: [PATCH 1/2] test(cli): route the arms and novelty parser tests through the handler, not the subparser dest The two tests read args.arms_cmd and args.novelty_cmd, the dest names of the subparsers. Assert the command that runs instead (args.func, unwrapped for the novelty spend-cap wrapper), so a later move of a subcommand group keeps them green. Refs #94 Signed-off-by: Nikolay Petrov --- dokimi/tests/test_matrix_rollup.py | 4 ++-- dokimi/tests/test_novelty_cli.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/dokimi/tests/test_matrix_rollup.py b/dokimi/tests/test_matrix_rollup.py index 169a9000a..ca199ac1e 100644 --- a/dokimi/tests/test_matrix_rollup.py +++ b/dokimi/tests/test_matrix_rollup.py @@ -13,7 +13,7 @@ import dokimi.eval.arm_runner as arm_runner_mod from dokimi.artifacts import ArtifactValidationError, validate_artifact from dokimi.cli import build_parser -from dokimi.commands.arms import _load_questions +from dokimi.commands.arms import _load_questions, cmd_arms_matrix_run from dokimi.eval.matrix_rollup import build_matrix_rollup, strip_internal_payloads from dokimi.report.matrix_rollup_report import _md_cell, render_matrix_rollup_report @@ -160,7 +160,7 @@ def test_arms_matrix_run_parser_accepts_required_args(): "matrix.json", ]) - assert args.arms_cmd == "matrix-run" + assert args.func is cmd_arms_matrix_run assert args.matrix_cells == "matrix.json" assert args.plugin_deltas is True diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index 62b7f1f5f..9db901a78 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -5,6 +5,7 @@ from __future__ import annotations +import inspect import json import os import re @@ -27,12 +28,19 @@ def _parse(*argv): def test_every_subcommand_is_registered(): - for sub in ("analyze", "atoms", "probe", "shorten", "view"): + handlers = { + "analyze": novelty_cmd.cmd_novelty_analyze, + "atoms": novelty_cmd.cmd_novelty_atoms, + "probe": novelty_cmd.cmd_novelty_probe, + "shorten": novelty_cmd.cmd_novelty_shorten, + "view": novelty_cmd.cmd_novelty_view, + } + for sub, handler in handlers.items(): args = _parse("novelty", sub, *( ["--artifact", "skill:x"] if sub != "view" else ["--novelty", "n.json"] ), *(["--atoms", "a.json"] if sub in ("probe", "shorten", "view") else [])) - assert args.novelty_cmd == sub - assert callable(args.func) + # The command that runs is the subcommand's own handler (`_stops_at_cap` wraps it). + assert inspect.unwrap(args.func) is handler def test_the_judge_defaults_to_a_different_model_than_the_answerer(): From 26e9f43d4e4590e2d446656c6d6cb7f348f04be3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 21:04:15 -0700 Subject: [PATCH 2/2] test(cli): guard that no printed string in the package names python cli.py The hint tests cover four hints by name. This one scans every string literal in dokimi/ (docstrings excluded) so a new hint that copies the source shim fails here. Refs #94 Signed-off-by: Nikolay Petrov --- dokimi/tests/test_printed_hints_parse.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dokimi/tests/test_printed_hints_parse.py b/dokimi/tests/test_printed_hints_parse.py index 48e2a8d0f..7fff986a2 100644 --- a/dokimi/tests/test_printed_hints_parse.py +++ b/dokimi/tests/test_printed_hints_parse.py @@ -7,12 +7,15 @@ the argv is taken from the text, and the argv goes through the same parse_args the CLI uses. """ +import ast import json import re import shlex +from pathlib import Path import pytest +import dokimi from dokimi import cli from dokimi.commands import personas as personas_cmd @@ -90,3 +93,28 @@ def test_baseline_only_report_hint_parses(): # The report quotes the command with a "> " prefix on every line. _assert_hints_parse("\n".join(line.removeprefix("> ") for line in _BASELINE_ONLY_LIMITATION)) + + +def _docstring_nodes(tree: ast.AST) -> set[int]: + """ids of the string constants that are docstrings: prose, not text a user is shown or copies.""" + nodes = set() + for node in ast.walk(tree): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + body = node.body + if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant): + nodes.add(id(body[0].value)) + return nodes + + +def test_no_string_in_the_package_names_the_source_shim(): + """Any string literal in the package that names `python cli.py` is a hint a user may copy; none may.""" + package = Path(dokimi.__file__).parent + offenders = [] + for path in sorted(package.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + docstrings = _docstring_nodes(tree) + for node in ast.walk(tree): + if (isinstance(node, ast.Constant) and isinstance(node.value, str) + and id(node) not in docstrings and "python cli.py" in node.value): + offenders.append(f"{path.relative_to(package)}:{node.lineno}: {node.value.strip()}") + assert not offenders, "hints name the source shim, not `dokimi`:\n" + "\n".join(offenders)