diff --git a/dokimi/changelog.d/20261029_150000_antonwolfy_scan-incomplete-reasons.md b/dokimi/changelog.d/20261029_150000_antonwolfy_scan-incomplete-reasons.md new file mode 100644 index 000000000..7095b27a4 --- /dev/null +++ b/dokimi/changelog.d/20261029_150000_antonwolfy_scan-incomplete-reasons.md @@ -0,0 +1,11 @@ +### Fixed + +- **`repo scan` says why a scan is `incomplete`** — the text output used to print a missing + `also_scan` file like any glob that matches nothing, and `--fail-on-fail` exited 1 with + `fail 0` and no message. A scan that is incomplete now prints an `incomplete: ...` line after + the warnings, with the warnings that make it so; `--fail-on-fail` prints them on stderr as + `✗ scan incomplete: ...` when it exits 1 on such a scan; and `repo_readiness.v1` lists them in + `incomplete_reasons` (`[]` when the scan is complete). A complete scan prints what it did + before. The guidance-only manifest in `docs/reviewing.md` no longer lists + `also_scan: ["CONTRIBUTING.md"]`, which made every scan of a repository without that file + incomplete. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 38d9c96d8..4c5767910 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1136,6 +1136,11 @@ when any claim fails, an `also_scan` file is missing, a guidance file cannot be read, a Bazel build input (`BUILD`, `.bzl`, `.bazelrc`, ...) cannot be read, or no guidance file matched (`incomplete`). A glob that matches nothing does not fail the gate, so a file the gate must see belongs in `also_scan`. +`incomplete_reasons` lists the warnings that make the scan `incomplete`, as +they appear in `warnings` and in its order (`[]` for a complete scan). The text +output prints them after the warnings, on one `incomplete: ...` line, and when +`--fail-on-fail` exits 1 on an incomplete scan it prints them on stderr as +`✗ scan incomplete: ...`. Statuses are those of §4.4. Only `fail` counts against the guidance; `unresolved` needs a human look. Not implemented yet: the pre-commit↔CI diff --git a/dokimi/docs/reviewing.md b/dokimi/docs/reviewing.md index 8a0f8982b..966cb4b70 100644 --- a/dokimi/docs/reviewing.md +++ b/dokimi/docs/reviewing.md @@ -102,9 +102,13 @@ schema: repo_manifest.v1 id: my-repo guidance: globs: ["**/AGENTS.md", "**/CLAUDE.md", ".github/copilot-instructions.md"] - also_scan: ["CONTRIBUTING.md"] ``` +A glob that matches no file is only a warning. To scan a file the globs leave out, +add `also_scan: ["CONTRIBUTING.md"]` under `guidance`: `also_scan` names files the +repository has and the gate must see, so a missing one makes the scan `incomplete` +and `repo scan --fail-on-fail` exits 1. + The `id` must be lowercase: start with a letter or digit, then use only letters, digits, `.`, `_` and `-`. `reach` always adds the harness's own guidance file names (for example `CLAUDE.md`, `CLAUDE.local.md` or `AGENTS.md`). diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 159a606c9..bca0e00ab 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -107,6 +107,8 @@ def cmd_repo_scan(args: argparse.Namespace) -> None: print(f" claims: {c['total']} (ok {c['ok']}, unresolved {c['unresolved']}, fail {c['fail']})") for warning in payload["warnings"]: print(f" warning: {warning}") + if payload["incomplete"]: + print(f" incomplete: {'; '.join(payload['incomplete_reasons'])} (repo scan --fail-on-fail exits 1)") if static_info is not None and not static_errors: r = payload["by_source"]["repository"] print(f" repository static: {r['total']} claims, {r['fail']} fail; " @@ -117,10 +119,18 @@ def cmd_repo_scan(args: argparse.Namespace) -> None: f"{str(claim.get('claim'))[:90]} -- {claim.get('why')}") if args.out: print(f"wrote {args.out}") + _scan_gate(payload, args.fail_on_fail) # runs first: an invalid --static still gets the incomplete reason if static_errors: sys.exit(1) - if args.fail_on_fail and (payload["counts"]["fail"] or payload["incomplete"]): - sys.exit(1) + + +def _scan_gate(payload: dict, fail_on_fail: bool) -> None: + """--fail-on-fail: exit 1 on a failed claim or an incomplete scan, saying on stderr what the scan missed.""" + if not fail_on_fail or not (payload["counts"]["fail"] or payload["incomplete"]): + return + if payload["incomplete"]: + print(f"✗ scan incomplete: {'; '.join(payload['incomplete_reasons'])}", file=sys.stderr) + sys.exit(1) def cmd_repo_inspect(args: argparse.Namespace) -> None: diff --git a/dokimi/dokimi/repos/l0/__init__.py b/dokimi/dokimi/repos/l0/__init__.py index c67d9c84c..796eb510d 100644 --- a/dokimi/dokimi/repos/l0/__init__.py +++ b/dokimi/dokimi/repos/l0/__init__.py @@ -29,8 +29,8 @@ def guidance_files(tree: RepoTree, manifest: RepoManifest) -> list[str]: return [f for f in tree.files() if any(match_glob(p, f) for p in patterns)] -def _coverage_warnings(tree: RepoTree, manifest: RepoManifest, files: list[str]) -> tuple[list[str], bool]: - """Warnings about guidance the manifest names but the tree lacks, and whether a gate should fail. +def _coverage_warnings(tree: RepoTree, manifest: RepoManifest, files: list[str]) -> tuple[list[str], list[str]]: + """Warnings about guidance the manifest names but the tree lacks, and those of them that fail a gate. A glob that matches nothing is only a warning (it may name a file a later base adds); a missing ``also_scan`` file, or no guidance at all, means the scan checked less than it says. @@ -38,11 +38,11 @@ def _coverage_warnings(tree: RepoTree, manifest: RepoManifest, files: list[str]) tree_files = tree.files() warnings = [f"guidance glob matches no file: {p}" for p in manifest.guidance_globs if not any(match_glob(p, f) for f in tree_files)] - missing = [p for p in manifest.also_scan if not any(match_glob(p, f) for f in tree_files)] - warnings += [f"guidance/also_scan matches no file: {p}" for p in missing] + reasons = [f"guidance/also_scan matches no file: {p}" for p in manifest.also_scan + if not any(match_glob(p, f) for f in tree_files)] if not files: - warnings.append("no guidance file to scan") - return warnings, bool(missing) or not files + reasons.append("no guidance file to scan") + return warnings + reasons, reasons def _is_build_input(path: str) -> bool: @@ -88,13 +88,14 @@ def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | N static_info["superseded"] = len(framework) - len(kept) claims = sorted(repository + kept, key=lambda c: (c["file"], c["line"], c["source"])) - warnings, incomplete = _coverage_warnings(tree, manifest, files) - warnings += [f"guidance file not scanned: {f}: {why}" for f, why in skipped.items()] + warnings, reasons = _coverage_warnings(tree, manifest, files) rc_files = scan.cached("bazel", dict).get("unread", {}) # `.bazelrc` imports, whatever their names unread = {f: why for f, why in sorted(scan.unread.items()) if "bazel" in scan.adapters and f not in skipped and (_is_build_input(f) or f in rc_files)} - warnings += [f"build file not scanned: {f}: {why}" for f, why in unread.items()] - incomplete = incomplete or bool(skipped) or bool(unread) + unscanned = [f"guidance file not scanned: {f}: {why}" for f, why in skipped.items()] + unscanned += [f"build file not scanned: {f}: {why}" for f, why in unread.items()] + warnings += unscanned + reasons += unscanned # every warning that makes the scan incomplete, in the order of `warnings` sha = getattr(tree, "sha", None) return { "repo": {"id": manifest.id, "tree": tree.label, "sha": sha, "manifest": manifest.path}, @@ -108,7 +109,8 @@ def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | N "by_file": _grouped(claims, "file"), "static": static_info, "warnings": warnings, - "incomplete": incomplete, + "incomplete": bool(reasons), + "incomplete_reasons": reasons, "claims": claims, } diff --git a/dokimi/dokimi/schemas/repo_readiness.v1.json b/dokimi/dokimi/schemas/repo_readiness.v1.json index 96e2a19dc..6340840af 100644 --- a/dokimi/dokimi/schemas/repo_readiness.v1.json +++ b/dokimi/dokimi/schemas/repo_readiness.v1.json @@ -67,6 +67,7 @@ }, "warnings": {"type": "array", "items": {"type": "string"}, "description": "Guidance the manifest names but the tree lacks: a glob with no match, a missing also_scan file, no guidance at all."}, "incomplete": {"type": "boolean", "description": "True when an also_scan file is missing, a guidance file or Bazel build input cannot be read, or no guidance file was scanned; `repo scan --fail-on-fail` then exits 1."}, + "incomplete_reasons": {"type": "array", "items": {"type": "string"}, "description": "The warnings that make the scan incomplete, as they appear in `warnings` and in its order; empty when the scan is complete. Absent from reports written before it was added."}, "claims": { "type": "array", "items": { diff --git a/dokimi/tests/test_repo_l0.py b/dokimi/tests/test_repo_l0.py index 8e500f6a0..87cc600c4 100644 --- a/dokimi/tests/test_repo_l0.py +++ b/dokimi/tests/test_repo_l0.py @@ -240,28 +240,98 @@ def test_scan_command_writes_a_valid_artifact(toy, tmp_path, capsys): with pytest.raises(SystemExit) as exc: args.func(args) assert exc.value.code == 1 + assert capsys.readouterr().err == "" # the scan is complete: the FAIL lines say why @pytest.mark.outcome("cjm-repo-scan") -@pytest.mark.parametrize("files, exits", [ +@pytest.mark.parametrize("files, err", [ # every claim passes and the guidance is there: the gate passes - ({"AGENTS.md": "# Clean\n\nNothing here to check.\n", "CONTRIBUTING.md": "Read first.\n"}, False), - # no guidance file: nothing is checked, so the gate fails with zero failed claims - ({"README.md": "Read first.\n"}, True), + ({"AGENTS.md": "# Clean\n\nNothing here to check.\n", "CONTRIBUTING.md": "Read first.\n"}, None), + # no guidance file: nothing is checked, so the gate fails with zero failed claims, and says why + ({"README.md": "Read first.\n"}, + "✗ scan incomplete: guidance/also_scan matches no file: CONTRIBUTING.md; no guidance file to scan\n"), ]) -def test_fail_on_fail_follows_the_result_not_the_flag(tmp_path, files, exits): +def test_fail_on_fail_follows_the_result_not_the_flag(tmp_path, capsys, files, err): root = tmp_path / "clean" sha = _repo(root, files) manifest_file = tmp_path / "m.json" manifest_file.write_text(json.dumps(MANIFEST), encoding="utf-8") args = build_parser().parse_args(["repo", "scan", f"{root}@{sha}", "--manifest-file", str(manifest_file), "--fail-on-fail"]) - if exits: + if err: with pytest.raises(SystemExit) as exc: args.func(args) assert exc.value.code == 1 + assert capsys.readouterr().err == err else: args.func(args) + assert capsys.readouterr().err == "" + + +def _scan_argv(tmp_path: Path, files: dict[str, str]) -> list[str]: + """`repo scan` of a new repository holding ``files``, with guidance globs for AGENTS.md and + CLAUDE.md and CONTRIBUTING.md in ``also_scan``.""" + root = tmp_path / "r" + sha = _repo(root, files) + manifest_file = tmp_path / "m.json" + guidance = {"globs": ["**/AGENTS.md", "**/CLAUDE.md"], "also_scan": ["CONTRIBUTING.md"]} + manifest_file.write_text(json.dumps(dict(MANIFEST, guidance=guidance)), encoding="utf-8") + return ["repo", "scan", f"{root}@{sha}", "--manifest-file", str(manifest_file)] + + +def _exit_code(argv: list[str]) -> int | str | None: + args = build_parser().parse_args(argv) + try: + args.func(args) + except SystemExit as exc: + return exc.code + return None + + +CLEAN = "# Clean\n\nNothing here to check.\n" +NO_CLAUDE = "guidance glob matches no file: **/CLAUDE.md" + + +def test_a_missing_also_scan_file_is_named_as_the_cause(tmp_path, capsys): + argv = _scan_argv(tmp_path, {"AGENTS.md": CLEAN}) + reason = "guidance/also_scan matches no file: CONTRIBUTING.md" + out = tmp_path / "readiness.json" + assert _exit_code(argv + ["--out", str(out)]) is None + captured = capsys.readouterr() + assert captured.err == "" # without --fail-on-fail the reason is only in the summary + lines = captured.out.splitlines() + assert lines[1:5] == [" claims: 0 (ok 0, unresolved 0, fail 0)", f" warning: {NO_CLAUDE}", + f" warning: {reason}", f" incomplete: {reason} (repo scan --fail-on-fail exits 1)"] + data = load_artifact("repo_readiness", out) + assert data["warnings"] == [NO_CLAUDE, reason] + assert data["incomplete"] is True and data["incomplete_reasons"] == [reason] + + assert _exit_code(argv + ["--fail-on-fail"]) == 1 + assert capsys.readouterr().err == f"✗ scan incomplete: {reason}\n" + + +def test_an_invalid_static_file_still_says_why_an_incomplete_scan_fails(tmp_path, capsys): + argv = _scan_argv(tmp_path, {"AGENTS.md": CLEAN}) + static = tmp_path / "bad.json" + static.write_text(json.dumps({"schema": "nope"}), encoding="utf-8") + reason = "guidance/also_scan matches no file: CONTRIBUTING.md" + assert _exit_code(argv + ["--static", str(static), "--fail-on-fail"]) == 1 + err = capsys.readouterr().err.splitlines() + assert any(line.startswith("✗ --static output not merged: ") for line in err) + assert f"✗ scan incomplete: {reason}" in err + + +def test_a_glob_that_matches_nothing_alone_keeps_the_scan_complete(tmp_path, capsys): + argv = _scan_argv(tmp_path, {"AGENTS.md": CLEAN, "CONTRIBUTING.md": "Read first.\n"}) + out = tmp_path / "readiness.json" + assert _exit_code(argv + ["--out", str(out), "--fail-on-fail"]) is None + captured = capsys.readouterr() + lines = captured.out.splitlines() + assert f" warning: {NO_CLAUDE}" in lines + assert not [line for line in lines if line.startswith(" incomplete:")] and captured.err == "" + data = load_artifact("repo_readiness", out) + assert data["warnings"] == [NO_CLAUDE] + assert data["incomplete"] is False and data["incomplete_reasons"] == [] def test_read_many_fetches_missing_blobs_of_a_partial_clone(toy, tmp_path): diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index c8763ffcf..1f557b722 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -649,7 +649,7 @@ def test_dangling_guidance_symlink_reported(tmp_path, link, why): for payload in _both(tmp_path, {"AGENTS.md": "x\n"}, {"CLAUDE.md": link}, also_scan=["CLAUDE.md"]): warning = next(w for w in payload["warnings"] if "CLAUDE.md" in w) assert warning.startswith("guidance file not scanned: CLAUDE.md") and why in warning, warning - assert payload["incomplete"] + assert payload["incomplete"] and payload["incomplete_reasons"] == [warning] assert payload["counts"]["total"] == 0 diff --git a/dokimi/tests/test_repo_l0_parity.py b/dokimi/tests/test_repo_l0_parity.py index 2ac7276e9..1785ea9c3 100644 --- a/dokimi/tests/test_repo_l0_parity.py +++ b/dokimi/tests/test_repo_l0_parity.py @@ -32,6 +32,7 @@ def _scan( build_systems=ALL, links: dict[str, str] | None = None, also_scan: list[str] | None = None, + globs: tuple[str, ...] = ("**/AGENTS.md",), ): root = tmp_path / "r" for rel, text in {"AGENTS.md": agents, "ev/tasks/t/task.json": "{}", **files}.items(): @@ -55,7 +56,7 @@ def _scan( "id": "case", "source": {"url": "https://github.com/acme/case"}, "eval_dir": "ev", - "guidance": {"globs": ["**/AGENTS.md"], **({"also_scan": also_scan} if also_scan else {})}, + "guidance": {"globs": list(globs), **({"also_scan": also_scan} if also_scan else {})}, "entry_points": {"grade": "true"}, "tasks": "ev/tasks", "build_systems": list(build_systems), @@ -590,10 +591,16 @@ def test_unreadable_build_inputs_warn_and_mark_the_scan_incomplete(tmp_path): "defs.bzl": "def m(name):\n pass\n\0", "third_party/zlib.BUILD": 'cc_library(name = "zlib")\n\0', } - payload = _scan(tmp_path, agents, files) + payload = _scan(tmp_path, agents, files, globs=("**/AGENTS.md", "**/CLAUDE.md")) assert payload["incomplete"] is True assert any(w.startswith("build file not scanned: defs.bzl: ") and "binary" in w for w in payload["warnings"]) assert any(w.startswith("build file not scanned: third_party/zlib.BUILD: ") for w in payload["warnings"]) + # the glob with no match is a warning but not a reason: it may name a file a later base adds + assert "guidance glob matches no file: **/CLAUDE.md" in payload["warnings"] + assert payload["incomplete_reasons"] == [ + "build file not scanned: defs.bzl: binary (a NUL byte)", + "build file not scanned: third_party/zlib.BUILD: binary (a NUL byte)", + ] @pytest.mark.parametrize("name", ["zlib.BUILD", "BUILD.zlib", "BUILD.zlib.bazel"])