test: state each fixture's expectations in its header and check them

CMakeLists.txt

@@ -286,9 +286,15 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
   if(BUILD_CLI)
     find_package(Python3 COMPONENTS Interpreter)
     if(Python3_Interpreter_FOUND)
-      add_test(NAME findings
-        COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/findings/check_findings.py"
-                $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_BINARY_DIR}/findings")
+      # Every fixture under test/ states what it proves in its header; a fixture without an
+      # expectation fails.
+      add_test(NAME fixtures
+        COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/check_fixtures.py"
+                $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_SOURCE_DIR}/test"
+                "${CMAKE_CURRENT_BINARY_DIR}/fixtures")
+      add_test(NAME cli
+        COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/cli/check_cli.py"
+                $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_BINARY_DIR}/cli")
       add_test(NAME timeout
         COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/timeout/check_timeout.py"
                 $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_BINARY_DIR}/timeout")
@@ -305,15 +311,10 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
       COMMAND sh -c "\"$<TARGET_FILE:${TOOL_NAME}>\" -o \"${CMAKE_CURRENT_BINARY_DIR}/hello\" -- test/examples/fixtures/hello.c > \"${_hello_out}\" && grep -q 'entry_events=[1-9]' \"${_hello_out}\" && grep -q 'exit_events=[1-9]' \"${_hello_out}\""
       WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
 
-    # Every source under test/ compiles: the sweep exits 0 only when no test fails to build.
-    add_test(NAME sweep_compiles
-      COMMAND $<TARGET_FILE:${TOOL_NAME}> --test-dir test --no-run
-              --output-dir "${CMAKE_CURRENT_BINARY_DIR}/sweep"
-      WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}")
-
-    # Every binary runs to completion within the timeout. Several fixtures report findings on
-    # purpose, so the exit status is 1: the pass expression makes CTest ignore it, and any
-    # [TIMEOUT] or [COMPILE] record fails the test.
+    # Batch mode over the whole tree, with its per-source heuristics (C sources that are C++,
+    # DEBUG): every fixture builds and runs to completion within the timeout. Several report
+    # findings on purpose, so the exit status is 1: the pass expression makes CTest ignore it,
+    # and any [TIMEOUT] or [COMPILE] record fails the test.
     add_test(NAME sweep_runs
       COMMAND $<TARGET_FILE:${TOOL_NAME}> --test-dir test --timeout 10
               --output-dir "${CMAKE_CURRENT_BINARY_DIR}/sweep-run" -- --ct-modules=alloc

README.md

@@ -221,17 +221,29 @@ ctest --test-dir build --output-on-failure
 | `findings_unit`, `process_unit` | the report parser and the process runner (`unittests/`) |
 | `api_unit` | the library API: `Run` and `RunBatch` on real programs, a heap overflow, a compile-only run, a hang against the timeout, a batch, the refused and empty cases |
 | `consumer` | a project fetching this repository the way ctrace does (`unittests/consumer/`) configures, builds, links `coretrace::runtime-analyzer_lib` and analyzes a program |
-| `findings` | one fixture per finding kind (`test/findings/`) and the vtable fixtures: rule, CWE, location, exit status, SARIF and text output, batch mode |
+| `fixtures` | every source under `test/` states what it proves in its header (`// args:`, `// expect:`), and the analyzer proves it: findings with rule, CWE, level, faulting line and allocation site, or none; the program's exit status and output when stated. A fixture without an expectation fails (`test/check_fixtures.py`) |
+| `cli` | the text output, the SARIF log of a batch, and exit `2` for what cannot be analyzed (`test/cli/check_cli.py`) |
 | `timeout` | hung, stdin-reading and forking programs, alone and in a batch |
 | `compile_failure` | a source that fails code generation fails its own analysis only |
 | `hello_runs` | `test/examples/fixtures/hello.c` runs and its entry/exit events are collected |
-| `sweep_compiles` | every source under `test/` compiles (`--test-dir test --no-run`) |
-| `sweep_runs` | every binary runs without a timeout (`--test-dir test --timeout 10 -- --ct-modules=alloc`) |
+| `sweep_runs` | batch mode over the whole tree: every fixture builds, with the per-source heuristics, and runs without a timeout (`--test-dir test --timeout 10 -- --ct-modules=alloc`) |
 | `version` | `--version` prints the `project()` version |
 | `llvm_floor` | configuration fails against an LLVM below the floor |
 | `shipped_toolchain_unit` | the CLI points coretrace-compiler at the Clang headers shipped next to it, unless `CT_CLANG` is set |
 | `install_layout` | `cmake --install` lays out the CLI, library, headers and Clang headers, and the installed CLI builds and runs a program |
 
+A new fixture is a source under `test/` whose header states its compiler arguments and what the
+analyzer must report, for example:
+
+```c
+// args: --ct-modules=alloc,vtable --ct-vtable-diag
+// expect: rule=vtable-null-this cwe=CWE-476 level=error line=11
+```
+
+`expect: none` states that nothing is reported; `expect: program-exit=3` and
+`expect: output="..."` check the program itself; `expect[linux]:` and `expect[darwin]:` keep a
+line to one platform. `test/check_fixtures.py` documents the keys.
+
 The `Build` workflow runs this suite on Ubuntu 24.04 and macOS against LLVM 20 for every push and
 pull request. `clang-format` (version 17) is checked by the `clang-format` workflow.
 

test/check_fixtures.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Every fixture under test/ states what it proves, and the analyzer proves it.
+
+Usage: check_fixtures.py <runtime-analyzer> <test dir> <work dir>
+
+A fixture is a C or C++ source under the test directory. The comments at its top declare:
+
+    // args: --ct-modules=alloc,vtable --ct-vtable-diag
+    // expect: rule=heap-buffer-overflow cwe=CWE-122 level=error line=8 alloc=7
+    // expect: none
+    // expect: program-exit=3
+    // expect: output="auto-free ptr="
+
+`args` are the compiler arguments, `--ct-modules=alloc,bounds` by default. One `expect: rule=`
+line per finding: `line` is the faulting access and `alloc` the allocation site, both in the
+fixture; `cwe` and `level` are checked when given. `expect: none` states that nothing is
+reported. `program-exit` is the program's own exit status and `output` a substring of what it
+printed. `expect[linux]:` and `expect[darwin]:` keep a line to one platform. The analyzer's exit
+status must be 1 with findings and 0 without. A fixture without `expect:` fails: nobody adds
+one without saying what it proves.
+"""
+
+from __future__ import annotations
+
+import json
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+DEFAULT_ARGS = ["--ct-modules=alloc,bounds"]
+PLATFORM = "darwin" if sys.platform == "darwin" else "linux"
+SOURCE_SUFFIXES = {".c", ".cc", ".cpp", ".cxx"}
+KEYS = {"rule", "cwe", "level", "line", "alloc", "program-exit", "output"}
+
+failures = 0
+
+
+def fail(message: str) -> None:
+    global failures
+    failures += 1
+    print(f"[FAIL] {message}")
+
+
+def read_header(fixture: Path) -> tuple[list[str], list[dict[str, str]], list[str]]:
+    """The compiler arguments, the expectations and the parse errors of a fixture."""
+    args, expectations, errors = list(DEFAULT_ARGS), [], []
+    for line in fixture.read_text().splitlines():
+        if not line.startswith("//"):
+            if line.strip():
+                break
+            continue
+        text = line[2:].strip()
+        if text.startswith("args:"):
+            args = shlex.split(text[len("args:"):])
+            continue
+        if not text.startswith("expect"):
+            continue
+        directive, _, rest = text.partition(":")
+        platform = directive[len("expect"):].strip("[]")
+        if platform not in ("", PLATFORM):
+            continue
+        tokens = shlex.split(rest)
+        if tokens == ["none"]:
+            expectations.append({"none": ""})
+            continue
+        expectation = dict(token.partition("=")[::2] for token in tokens)
+        unknown = set(expectation) - KEYS
+        if unknown or any("=" not in token for token in tokens):
+            errors.append(f"cannot read expectation: {rest.strip()}")
+            continue
+        expectations.append(expectation)
+    return args, expectations, errors
+
+
+def sarif_findings(stdout: str) -> list[dict[str, object]] | None:
+    try:
+        log = json.loads(stdout)
+    except json.JSONDecodeError:
+        return None
+    findings = []
+    for result in log["runs"][0]["results"]:
+        finding: dict[str, object] = {"rule": result["ruleId"], "cwe": result["properties"]["cwe"],
+                                      "level": result["level"]}
+        for key, locations in (("line", result.get("locations")),
+                               ("alloc", result.get("relatedLocations"))):
+            if locations:
+                physical = locations[0]["physicalLocation"]
+                finding[key] = str(physical["region"]["startLine"])
+                finding[key + "_uri"] = physical["artifactLocation"]["uri"]
+                finding[key + "_column"] = physical["region"].get("startColumn", 0)
+        related = (result.get("relatedLocations") or [{}])[0]
+        finding["alloc_ok"] = related.get("id") == 0 and \
+            related.get("message", {}).get("text") == "allocated here"
+        findings.append(finding)
+    return findings
+
+
+def matches(expected: dict[str, str], finding: dict[str, object], uri: str) -> bool:
+    for key in ("rule", "cwe", "level", "line", "alloc"):
+        if key in expected and finding.get(key) != expected[key]:
+            return False
+    if "line" in expected and (finding["line_uri"] != uri or not finding["line_column"]):
+        return False
+    if "alloc" in expected and (finding["alloc_uri"] != uri or not finding["alloc_ok"]):
+        return False
+    return True
+
+
+def describe(finding: dict[str, object]) -> str:
+    return " ".join(f"{key}={finding[key]}" for key in ("rule", "cwe", "level", "line", "alloc")
+                    if key in finding)
+
+
+def check(analyzer: str, test_dir: Path, work: Path, fixture: Path) -> None:
+    rel = fixture.relative_to(test_dir).as_posix()
+    args, expectations, errors = read_header(fixture)
+    for error in errors:
+        fail(f"{rel}: {error}")
+    findings_expected = [e for e in expectations if "rule" in e]
+    if not findings_expected and not any("none" in e for e in expectations):
+        fail(f"{rel}: states no expectation (expect: rule=... or expect: none)")
+        return
+
+    binary = work / fixture.stem
+    command = [analyzer, "-o", str(binary), "--timeout", "30"]
+    completed = subprocess.run([*command, "--format=sarif", "--", *args, rel], cwd=test_dir,
+                               capture_output=True, text=True, timeout=120)
+    findings = sarif_findings(completed.stdout)
+    if findings is None:
+        fail(f"{rel}: no SARIF log on stdout\n{completed.stdout}{completed.stderr}")
+        return
+    verdict = 1 if findings_expected else 0
+    if completed.returncode != verdict:
+        fail(f"{rel}: exit {verdict} expected, got {completed.returncode}\n{completed.stderr}")
+
+    remaining = list(findings)
+    for expected in findings_expected:
+        found = next((f for f in remaining if matches(expected, f, rel)), None)
+        if found is None:
+            fail(f"{rel}: no finding matches {describe(expected)}")
+        else:
+            remaining.remove(found)
+    for finding in remaining:
+        fail(f"{rel}: unexpected finding {describe(finding)}")
+
+    program = [e for e in expectations if "program-exit" in e or "output" in e]
+    if program:
+        completed = subprocess.run([*command, "--show-output", "--", *args, rel], cwd=test_dir,
+                                   capture_output=True, text=True, timeout=120)
+        text = completed.stdout + completed.stderr
+        for expected in program:
+            if "program-exit" in expected:
+                wanted = f"runtime-analyzer: exit_code={expected['program-exit']}"
+                if wanted not in completed.stdout:
+                    fail(f"{rel}: program exit {expected['program-exit']} expected\n{text}")
+            if "output" in expected and expected["output"] not in text:
+                fail(f"{rel}: program output lacks {expected['output']!r}\n{text}")
+
+
+def main() -> int:
+    analyzer = str(Path(sys.argv[1]).resolve())
+    test_dir = Path(sys.argv[2]).resolve()
+    work = Path(sys.argv[3]).resolve()
+    work.mkdir(parents=True, exist_ok=True)
+    fixtures = sorted(p for p in test_dir.rglob("*") if p.suffix in SOURCE_SUFFIXES)
+    if not fixtures:
+        fail(f"no fixture under {test_dir}")
+    for fixture in fixtures:
+        before = failures
+        check(analyzer, test_dir, work, fixture)
+        if failures == before:
+            print(f"[PASS] {fixture.relative_to(test_dir).as_posix()}")
+    if failures:
+        print(f"{failures} fixture check(s) failed", file=sys.stderr)
+        return 1
+    print(f"check_fixtures: {len(fixtures)} fixtures proved what they state")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

test/cli/check_cli.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""The CLI's output formats, verdicts and batch mode over the findings fixtures.
+
+Usage: check_cli.py <runtime-analyzer> <work dir>
+
+What each fixture proves is stated in the fixture and checked by check_fixtures.py; this script
+checks how the CLI reports it: the text output, the SARIF log of a batch, and the exit status
+of the runs that cannot be analyzed.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+TEST_DIR = HERE.parent
+MEMORY_ARGS = ["--ct-modules=alloc,bounds"]
+
+failures = 0
+
+
+def expect(condition: bool, message: str) -> None:
+    global failures
+    print(f"[{'PASS' if condition else 'FAIL'}] {message}")
+    if not condition:
+        failures += 1
+
+
+def run(*args: str, timeout: int = 120) -> subprocess.CompletedProcess:
+    # From test/, with relative sources: findings carry the path the source was compiled from.
+    return subprocess.run(list(args), cwd=TEST_DIR, capture_output=True, text=True,
+                          timeout=timeout)
+
+
+def analyze(analyzer: str, work: Path, fixture: str, *options: str) -> subprocess.CompletedProcess:
+    return run(analyzer, "-o", str(work / Path(fixture).stem), *options, "--", *MEMORY_ARGS,
+               f"findings/{fixture}")
+
+
+def sarif_rules(completed: subprocess.CompletedProcess) -> list[str] | None:
+    try:
+        log = json.loads(completed.stdout)
+    except json.JSONDecodeError:
+        return None
+    return sorted(result["ruleId"] for result in log["runs"][0]["results"])
+
+
+def main() -> int:
+    analyzer, work = str(Path(sys.argv[1]).resolve()), Path(sys.argv[2]).resolve()
+    work.mkdir(parents=True, exist_ok=True)
+
+    # The program's exit status is reported, but only runtime errors decide the verdict.
+    completed = analyze(analyzer, work, "no_findings.c")
+    expect(completed.returncode == 0 and "runtime-analyzer: exit_code=3" in completed.stdout,
+           "no_findings.c: the text summary keeps the program's exit code")
+    expect("runtime-analyzer: findings=0" in completed.stdout, "no_findings.c: findings=0 in text")
+
+    completed = analyze(analyzer, work, "heap_overflow_write.c")
+    expect(completed.returncode == 1, "text output: exit 1 with findings")
+    expect("runtime-analyzer: findings=1" in completed.stdout, "text output: findings=1")
+    expect("findings/heap_overflow_write.c:8:" in completed.stdout
+           and "error: heap-buffer-overflow WRITE of size 4 [heap-buffer-overflow, CWE-122]"
+           in completed.stdout,
+           "text output: one compiler-style line per finding")
+    expect("findings/heap_overflow_write.c:7:" in completed.stdout
+           and "note: allocated here" in completed.stdout,
+           "text output: the allocation site as a note")
+
+    completed = analyze(analyzer, work, "no_findings.c", "--no-run", "--format=sarif")
+    expect(completed.returncode == 0 and sarif_rules(completed) == [],
+           "--no-run: exit 0 and an empty log after a successful build")
+
+    # --test-dir runs every fixture and reports all their findings in one log.
+    completed = run(analyzer, "--test-dir", "findings", "--output-dir", str(work / "batch"),
+                    "--format=sarif", "--", *MEMORY_ARGS, timeout=300)
+    expect(completed.returncode == 1, "batch: exit 1 when any program reports findings")
+    expect(sarif_rules(completed) == ["double-free", "heap-buffer-overflow", "heap-buffer-overflow",
+                                      "heap-use-after-free", "memory-leak", "stack-buffer-overflow"],
+           "batch: one log with the findings of every program")
+    completed = run(analyzer, "--test-dir", "findings", "--output-dir", str(work / "batch"),
+                    "--", *MEMORY_ARGS, timeout=300)
+    expect(completed.returncode == 1 and "runtime-analyzer: [PASS] findings/no_findings.c"
+           not in completed.stdout and "findings/heap_overflow_write.c:8:" in completed.stdout,
+           "batch text: findings are listed under each test's line")
+
+    completed = analyze(analyzer, work, "missing.c")
+    expect(completed.returncode == 2, "exit 2 when the program cannot be built")
+    completed = analyze(analyzer, work, "no_findings.c", "--format=xml")
+    expect(completed.returncode == 2, "exit 2 on an unknown output format")
+    completed = run(analyzer, "--test-dir", "findings", "--", "-c", timeout=60)
+    expect(completed.returncode == 2, "batch: exit 2 on compile-only flags")
+
+    if failures:
+        print(f"{failures} CLI check(s) failed", file=sys.stderr)
+        return 1
+    print("check_cli: all checks passed")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

test/ct_alloc_basic.c

@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
+// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=18
 #include <stdlib.h>
 
 int main(void)

test/ct_alloc_growth.c

@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
+// expect: none
 #include <stdlib.h>
 #include <stdio.h>
 

test/ct_autofree_aligned_alloc.c

@@ -1,4 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
+// args: --ct-modules=trace,alloc --ct-autofree
+// expect: none
+// expect: output="auto-free ptr="
 #include <stdlib.h>
 
 int main(void)

test/ct_autofree_brk.c

@@ -1,4 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
+// args: --ct-modules=trace,alloc --ct-autofree
+// expect: none
 #include <unistd.h>
 
 int main(void)

test/ct_autofree_inttoptr.c

@@ -1,4 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
+// args: --ct-modules=trace,alloc --ct-autofree
+// expect: none
+// expect: output="auto-free ptr="
 #include <stdlib.h>
 #include <stdint.h>