test(cli): hint guard and behaviour-based subparser assertions (#94)

dokimi/tests/test_matrix_rollup.py

@@ -13,7 +13,7 @@
 import dokimi.eval.arm_runner as arm_runner_mod
 from dokimi.artifacts import ArtifactValidationError, validate_artifact
 from dokimi.cli import build_parser
-from dokimi.commands.arms import _load_questions
+from dokimi.commands.arms import _load_questions, cmd_arms_matrix_run
 from dokimi.eval.matrix_rollup import build_matrix_rollup, strip_internal_payloads
 from dokimi.report.matrix_rollup_report import _md_cell, render_matrix_rollup_report
 
@@ -160,7 +160,7 @@ def test_arms_matrix_run_parser_accepts_required_args():
         "matrix.json",
     ])
 
-    assert args.arms_cmd == "matrix-run"
+    assert args.func is cmd_arms_matrix_run
     assert args.matrix_cells == "matrix.json"
     assert args.plugin_deltas is True
 

dokimi/tests/test_novelty_cli.py

@@ -5,6 +5,7 @@
 
 from __future__ import annotations
 
+import inspect
 import json
 import os
 import re
@@ -27,12 +28,19 @@ def _parse(*argv):
 
 
 def test_every_subcommand_is_registered():
-    for sub in ("analyze", "atoms", "probe", "shorten", "view"):
+    handlers = {
+        "analyze": novelty_cmd.cmd_novelty_analyze,
+        "atoms": novelty_cmd.cmd_novelty_atoms,
+        "probe": novelty_cmd.cmd_novelty_probe,
+        "shorten": novelty_cmd.cmd_novelty_shorten,
+        "view": novelty_cmd.cmd_novelty_view,
+    }
+    for sub, handler in handlers.items():
         args = _parse("novelty", sub, *(
             ["--artifact", "skill:x"] if sub != "view" else ["--novelty", "n.json"]
         ), *(["--atoms", "a.json"] if sub in ("probe", "shorten", "view") else []))
-        assert args.novelty_cmd == sub
-        assert callable(args.func)
+        # The command that runs is the subcommand's own handler (`_stops_at_cap` wraps it).
+        assert inspect.unwrap(args.func) is handler
 
 
 def test_the_judge_defaults_to_a_different_model_than_the_answerer():

dokimi/tests/test_printed_hints_parse.py

@@ -7,12 +7,15 @@
 the argv is taken from the text, and the argv goes through the same parse_args the CLI uses.
 """
 
+import ast
 import json
 import re
 import shlex
+from pathlib import Path
 
 import pytest
 
+import dokimi
 from dokimi import cli
 from dokimi.commands import personas as personas_cmd
 
@@ -90,3 +93,28 @@ def test_baseline_only_report_hint_parses():
 
     # The report quotes the command with a "> " prefix on every line.
     _assert_hints_parse("\n".join(line.removeprefix("> ") for line in _BASELINE_ONLY_LIMITATION))
+
+
+def _docstring_nodes(tree: ast.AST) -> set[int]:
+    """ids of the string constants that are docstrings: prose, not text a user is shown or copies."""
+    nodes = set()
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+            body = node.body
+            if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant):
+                nodes.add(id(body[0].value))
+    return nodes
+
+
+def test_no_string_in_the_package_names_the_source_shim():
+    """Any string literal in the package that names `python cli.py` is a hint a user may copy; none may."""
+    package = Path(dokimi.__file__).parent
+    offenders = []
+    for path in sorted(package.rglob("*.py")):
+        tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+        docstrings = _docstring_nodes(tree)
+        for node in ast.walk(tree):
+            if (isinstance(node, ast.Constant) and isinstance(node.value, str)
+                    and id(node) not in docstrings and "python cli.py" in node.value):
+                offenders.append(f"{path.relative_to(package)}:{node.lineno}: {node.value.strip()}")
+    assert not offenders, "hints name the source shim, not `dokimi`:\n" + "\n".join(offenders)