diff --git a/CMakeLists.txt b/CMakeLists.txt index a34cfc1..f35f1b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -286,9 +286,15 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) if(BUILD_CLI) find_package(Python3 COMPONENTS Interpreter) if(Python3_Interpreter_FOUND) - add_test(NAME findings - COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/findings/check_findings.py" - $ "${CMAKE_CURRENT_BINARY_DIR}/findings") + # Every fixture under test/ states what it proves in its header; a fixture without an + # expectation fails. + add_test(NAME fixtures + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/check_fixtures.py" + $ "${CMAKE_CURRENT_SOURCE_DIR}/test" + "${CMAKE_CURRENT_BINARY_DIR}/fixtures") + add_test(NAME cli + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/cli/check_cli.py" + $ "${CMAKE_CURRENT_BINARY_DIR}/cli") add_test(NAME timeout COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/timeout/check_timeout.py" $ "${CMAKE_CURRENT_BINARY_DIR}/timeout") @@ -305,15 +311,10 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) COMMAND sh -c "\"$\" -o \"${CMAKE_CURRENT_BINARY_DIR}/hello\" -- test/examples/fixtures/hello.c > \"${_hello_out}\" && grep -q 'entry_events=[1-9]' \"${_hello_out}\" && grep -q 'exit_events=[1-9]' \"${_hello_out}\"" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") - # Every source under test/ compiles: the sweep exits 0 only when no test fails to build. - add_test(NAME sweep_compiles - COMMAND $ --test-dir test --no-run - --output-dir "${CMAKE_CURRENT_BINARY_DIR}/sweep" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") - - # Every binary runs to completion within the timeout. Several fixtures report findings on - # purpose, so the exit status is 1: the pass expression makes CTest ignore it, and any - # [TIMEOUT] or [COMPILE] record fails the test. + # Batch mode over the whole tree, with its per-source heuristics (C sources that are C++, + # DEBUG): every fixture builds and runs to completion within the timeout. Several report + # findings on purpose, so the exit status is 1: the pass expression makes CTest ignore it, + # and any [TIMEOUT] or [COMPILE] record fails the test. add_test(NAME sweep_runs COMMAND $ --test-dir test --timeout 10 --output-dir "${CMAKE_CURRENT_BINARY_DIR}/sweep-run" -- --ct-modules=alloc diff --git a/README.md b/README.md index fa87478..78d6a24 100644 --- a/README.md +++ b/README.md @@ -221,17 +221,29 @@ ctest --test-dir build --output-on-failure | `findings_unit`, `process_unit` | the report parser and the process runner (`unittests/`) | | `api_unit` | the library API: `Run` and `RunBatch` on real programs, a heap overflow, a compile-only run, a hang against the timeout, a batch, the refused and empty cases | | `consumer` | a project fetching this repository the way ctrace does (`unittests/consumer/`) configures, builds, links `coretrace::runtime-analyzer_lib` and analyzes a program | -| `findings` | one fixture per finding kind (`test/findings/`) and the vtable fixtures: rule, CWE, location, exit status, SARIF and text output, batch mode | +| `fixtures` | every source under `test/` states what it proves in its header (`// args:`, `// expect:`), and the analyzer proves it: findings with rule, CWE, level, faulting line and allocation site, or none; the program's exit status and output when stated. A fixture without an expectation fails (`test/check_fixtures.py`) | +| `cli` | the text output, the SARIF log of a batch, and exit `2` for what cannot be analyzed (`test/cli/check_cli.py`) | | `timeout` | hung, stdin-reading and forking programs, alone and in a batch | | `compile_failure` | a source that fails code generation fails its own analysis only | | `hello_runs` | `test/examples/fixtures/hello.c` runs and its entry/exit events are collected | -| `sweep_compiles` | every source under `test/` compiles (`--test-dir test --no-run`) | -| `sweep_runs` | every binary runs without a timeout (`--test-dir test --timeout 10 -- --ct-modules=alloc`) | +| `sweep_runs` | batch mode over the whole tree: every fixture builds, with the per-source heuristics, and runs without a timeout (`--test-dir test --timeout 10 -- --ct-modules=alloc`) | | `version` | `--version` prints the `project()` version | | `llvm_floor` | configuration fails against an LLVM below the floor | | `shipped_toolchain_unit` | the CLI points coretrace-compiler at the Clang headers shipped next to it, unless `CT_CLANG` is set | | `install_layout` | `cmake --install` lays out the CLI, library, headers and Clang headers, and the installed CLI builds and runs a program | +A new fixture is a source under `test/` whose header states its compiler arguments and what the +analyzer must report, for example: + +```c +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: rule=vtable-null-this cwe=CWE-476 level=error line=11 +``` + +`expect: none` states that nothing is reported; `expect: program-exit=3` and +`expect: output="..."` check the program itself; `expect[linux]:` and `expect[darwin]:` keep a +line to one platform. `test/check_fixtures.py` documents the keys. + The `Build` workflow runs this suite on Ubuntu 24.04 and macOS against LLVM 20 for every push and pull request. `clang-format` (version 17) is checked by the `clang-format` workflow. diff --git a/test/check_fixtures.py b/test/check_fixtures.py new file mode 100644 index 0000000..e8465bb --- /dev/null +++ b/test/check_fixtures.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Every fixture under test/ states what it proves, and the analyzer proves it. + +Usage: check_fixtures.py + +A fixture is a C or C++ source under the test directory. The comments at its top declare: + + // args: --ct-modules=alloc,vtable --ct-vtable-diag + // expect: rule=heap-buffer-overflow cwe=CWE-122 level=error line=8 alloc=7 + // expect: none + // expect: program-exit=3 + // expect: output="auto-free ptr=" + +`args` are the compiler arguments, `--ct-modules=alloc,bounds` by default. One `expect: rule=` +line per finding: `line` is the faulting access and `alloc` the allocation site, both in the +fixture; `cwe` and `level` are checked when given. `expect: none` states that nothing is +reported. `program-exit` is the program's own exit status and `output` a substring of what it +printed. `expect[linux]:` and `expect[darwin]:` keep a line to one platform. The analyzer's exit +status must be 1 with findings and 0 without. A fixture without `expect:` fails: nobody adds +one without saying what it proves. +""" + +from __future__ import annotations + +import json +import shlex +import subprocess +import sys +from pathlib import Path + +DEFAULT_ARGS = ["--ct-modules=alloc,bounds"] +PLATFORM = "darwin" if sys.platform == "darwin" else "linux" +SOURCE_SUFFIXES = {".c", ".cc", ".cpp", ".cxx"} +KEYS = {"rule", "cwe", "level", "line", "alloc", "program-exit", "output"} + +failures = 0 + + +def fail(message: str) -> None: + global failures + failures += 1 + print(f"[FAIL] {message}") + + +def read_header(fixture: Path) -> tuple[list[str], list[dict[str, str]], list[str]]: + """The compiler arguments, the expectations and the parse errors of a fixture.""" + args, expectations, errors = list(DEFAULT_ARGS), [], [] + for line in fixture.read_text().splitlines(): + if not line.startswith("//"): + if line.strip(): + break + continue + text = line[2:].strip() + if text.startswith("args:"): + args = shlex.split(text[len("args:"):]) + continue + if not text.startswith("expect"): + continue + directive, _, rest = text.partition(":") + platform = directive[len("expect"):].strip("[]") + if platform not in ("", PLATFORM): + continue + tokens = shlex.split(rest) + if tokens == ["none"]: + expectations.append({"none": ""}) + continue + expectation = dict(token.partition("=")[::2] for token in tokens) + unknown = set(expectation) - KEYS + if unknown or any("=" not in token for token in tokens): + errors.append(f"cannot read expectation: {rest.strip()}") + continue + expectations.append(expectation) + return args, expectations, errors + + +def sarif_findings(stdout: str) -> list[dict[str, object]] | None: + try: + log = json.loads(stdout) + except json.JSONDecodeError: + return None + findings = [] + for result in log["runs"][0]["results"]: + finding: dict[str, object] = {"rule": result["ruleId"], "cwe": result["properties"]["cwe"], + "level": result["level"]} + for key, locations in (("line", result.get("locations")), + ("alloc", result.get("relatedLocations"))): + if locations: + physical = locations[0]["physicalLocation"] + finding[key] = str(physical["region"]["startLine"]) + finding[key + "_uri"] = physical["artifactLocation"]["uri"] + finding[key + "_column"] = physical["region"].get("startColumn", 0) + related = (result.get("relatedLocations") or [{}])[0] + finding["alloc_ok"] = related.get("id") == 0 and \ + related.get("message", {}).get("text") == "allocated here" + findings.append(finding) + return findings + + +def matches(expected: dict[str, str], finding: dict[str, object], uri: str) -> bool: + for key in ("rule", "cwe", "level", "line", "alloc"): + if key in expected and finding.get(key) != expected[key]: + return False + if "line" in expected and (finding["line_uri"] != uri or not finding["line_column"]): + return False + if "alloc" in expected and (finding["alloc_uri"] != uri or not finding["alloc_ok"]): + return False + return True + + +def describe(finding: dict[str, object]) -> str: + return " ".join(f"{key}={finding[key]}" for key in ("rule", "cwe", "level", "line", "alloc") + if key in finding) + + +def check(analyzer: str, test_dir: Path, work: Path, fixture: Path) -> None: + rel = fixture.relative_to(test_dir).as_posix() + args, expectations, errors = read_header(fixture) + for error in errors: + fail(f"{rel}: {error}") + findings_expected = [e for e in expectations if "rule" in e] + if not findings_expected and not any("none" in e for e in expectations): + fail(f"{rel}: states no expectation (expect: rule=... or expect: none)") + return + + binary = work / fixture.stem + command = [analyzer, "-o", str(binary), "--timeout", "30"] + completed = subprocess.run([*command, "--format=sarif", "--", *args, rel], cwd=test_dir, + capture_output=True, text=True, timeout=120) + findings = sarif_findings(completed.stdout) + if findings is None: + fail(f"{rel}: no SARIF log on stdout\n{completed.stdout}{completed.stderr}") + return + verdict = 1 if findings_expected else 0 + if completed.returncode != verdict: + fail(f"{rel}: exit {verdict} expected, got {completed.returncode}\n{completed.stderr}") + + remaining = list(findings) + for expected in findings_expected: + found = next((f for f in remaining if matches(expected, f, rel)), None) + if found is None: + fail(f"{rel}: no finding matches {describe(expected)}") + else: + remaining.remove(found) + for finding in remaining: + fail(f"{rel}: unexpected finding {describe(finding)}") + + program = [e for e in expectations if "program-exit" in e or "output" in e] + if program: + completed = subprocess.run([*command, "--show-output", "--", *args, rel], cwd=test_dir, + capture_output=True, text=True, timeout=120) + text = completed.stdout + completed.stderr + for expected in program: + if "program-exit" in expected: + wanted = f"runtime-analyzer: exit_code={expected['program-exit']}" + if wanted not in completed.stdout: + fail(f"{rel}: program exit {expected['program-exit']} expected\n{text}") + if "output" in expected and expected["output"] not in text: + fail(f"{rel}: program output lacks {expected['output']!r}\n{text}") + + +def main() -> int: + analyzer = str(Path(sys.argv[1]).resolve()) + test_dir = Path(sys.argv[2]).resolve() + work = Path(sys.argv[3]).resolve() + work.mkdir(parents=True, exist_ok=True) + fixtures = sorted(p for p in test_dir.rglob("*") if p.suffix in SOURCE_SUFFIXES) + if not fixtures: + fail(f"no fixture under {test_dir}") + for fixture in fixtures: + before = failures + check(analyzer, test_dir, work, fixture) + if failures == before: + print(f"[PASS] {fixture.relative_to(test_dir).as_posix()}") + if failures: + print(f"{failures} fixture check(s) failed", file=sys.stderr) + return 1 + print(f"check_fixtures: {len(fixtures)} fixtures proved what they state") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/cli/check_cli.py b/test/cli/check_cli.py new file mode 100644 index 0000000..6ca5e46 --- /dev/null +++ b/test/cli/check_cli.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""The CLI's output formats, verdicts and batch mode over the findings fixtures. + +Usage: check_cli.py + +What each fixture proves is stated in the fixture and checked by check_fixtures.py; this script +checks how the CLI reports it: the text output, the SARIF log of a batch, and the exit status +of the runs that cannot be analyzed. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +TEST_DIR = HERE.parent +MEMORY_ARGS = ["--ct-modules=alloc,bounds"] + +failures = 0 + + +def expect(condition: bool, message: str) -> None: + global failures + print(f"[{'PASS' if condition else 'FAIL'}] {message}") + if not condition: + failures += 1 + + +def run(*args: str, timeout: int = 120) -> subprocess.CompletedProcess: + # From test/, with relative sources: findings carry the path the source was compiled from. + return subprocess.run(list(args), cwd=TEST_DIR, capture_output=True, text=True, + timeout=timeout) + + +def analyze(analyzer: str, work: Path, fixture: str, *options: str) -> subprocess.CompletedProcess: + return run(analyzer, "-o", str(work / Path(fixture).stem), *options, "--", *MEMORY_ARGS, + f"findings/{fixture}") + + +def sarif_rules(completed: subprocess.CompletedProcess) -> list[str] | None: + try: + log = json.loads(completed.stdout) + except json.JSONDecodeError: + return None + return sorted(result["ruleId"] for result in log["runs"][0]["results"]) + + +def main() -> int: + analyzer, work = str(Path(sys.argv[1]).resolve()), Path(sys.argv[2]).resolve() + work.mkdir(parents=True, exist_ok=True) + + # The program's exit status is reported, but only runtime errors decide the verdict. + completed = analyze(analyzer, work, "no_findings.c") + expect(completed.returncode == 0 and "runtime-analyzer: exit_code=3" in completed.stdout, + "no_findings.c: the text summary keeps the program's exit code") + expect("runtime-analyzer: findings=0" in completed.stdout, "no_findings.c: findings=0 in text") + + completed = analyze(analyzer, work, "heap_overflow_write.c") + expect(completed.returncode == 1, "text output: exit 1 with findings") + expect("runtime-analyzer: findings=1" in completed.stdout, "text output: findings=1") + expect("findings/heap_overflow_write.c:8:" in completed.stdout + and "error: heap-buffer-overflow WRITE of size 4 [heap-buffer-overflow, CWE-122]" + in completed.stdout, + "text output: one compiler-style line per finding") + expect("findings/heap_overflow_write.c:7:" in completed.stdout + and "note: allocated here" in completed.stdout, + "text output: the allocation site as a note") + + completed = analyze(analyzer, work, "no_findings.c", "--no-run", "--format=sarif") + expect(completed.returncode == 0 and sarif_rules(completed) == [], + "--no-run: exit 0 and an empty log after a successful build") + + # --test-dir runs every fixture and reports all their findings in one log. + completed = run(analyzer, "--test-dir", "findings", "--output-dir", str(work / "batch"), + "--format=sarif", "--", *MEMORY_ARGS, timeout=300) + expect(completed.returncode == 1, "batch: exit 1 when any program reports findings") + expect(sarif_rules(completed) == ["double-free", "heap-buffer-overflow", "heap-buffer-overflow", + "heap-use-after-free", "memory-leak", "stack-buffer-overflow"], + "batch: one log with the findings of every program") + completed = run(analyzer, "--test-dir", "findings", "--output-dir", str(work / "batch"), + "--", *MEMORY_ARGS, timeout=300) + expect(completed.returncode == 1 and "runtime-analyzer: [PASS] findings/no_findings.c" + not in completed.stdout and "findings/heap_overflow_write.c:8:" in completed.stdout, + "batch text: findings are listed under each test's line") + + completed = analyze(analyzer, work, "missing.c") + expect(completed.returncode == 2, "exit 2 when the program cannot be built") + completed = analyze(analyzer, work, "no_findings.c", "--format=xml") + expect(completed.returncode == 2, "exit 2 on an unknown output format") + completed = run(analyzer, "--test-dir", "findings", "--", "-c", timeout=60) + expect(completed.returncode == 2, "batch: exit 2 on compile-only flags") + + if failures: + print(f"{failures} CLI check(s) failed", file=sys.stderr) + return 1 + print("check_cli: all checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/ct_alloc_basic.c b/test/ct_alloc_basic.c index 3ec388d..4ef686d 100644 --- a/test/ct_alloc_basic.c +++ b/test/ct_alloc_basic.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=18 #include int main(void) diff --git a/test/ct_alloc_growth.c b/test/ct_alloc_growth.c index e63c658..993cfa7 100644 --- a/test/ct_alloc_growth.c +++ b/test/ct_alloc_growth.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: none #include #include diff --git a/test/ct_autofree_aligned_alloc.c b/test/ct_autofree_aligned_alloc.c index 0e27946..fd97936 100644 --- a/test/ct_autofree_aligned_alloc.c +++ b/test/ct_autofree_aligned_alloc.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: none +// expect: output="auto-free ptr=" #include int main(void) diff --git a/test/ct_autofree_brk.c b/test/ct_autofree_brk.c index cb088e5..141eb1c 100644 --- a/test/ct_autofree_brk.c +++ b/test/ct_autofree_brk.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: none #include int main(void) diff --git a/test/ct_autofree_inttoptr.c b/test/ct_autofree_inttoptr.c index f595ff0..ffc273d 100644 --- a/test/ct_autofree_inttoptr.c +++ b/test/ct_autofree_inttoptr.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: none +// expect: output="auto-free ptr=" #include #include diff --git a/test/ct_autofree_inttoptr_escape.c b/test/ct_autofree_inttoptr_escape.c index f608094..4cb5292 100644 --- a/test/ct_autofree_inttoptr_escape.c +++ b/test/ct_autofree_inttoptr_escape.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=13 +// expect: program-exit=1 #include #include diff --git a/test/ct_autofree_local.c b/test/ct_autofree_local.c index 9cc73c2..6bcf556 100644 --- a/test/ct_autofree_local.c +++ b/test/ct_autofree_local.c @@ -1,4 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=17 +// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=18 +// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=19 +// expect: output="auto-free ptr=" #include #include #include diff --git a/test/ct_autofree_mmap.c b/test/ct_autofree_mmap.c index 0fda063..22a90ed 100644 --- a/test/ct_autofree_mmap.c +++ b/test/ct_autofree_mmap.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: none +// expect: output="auto-free ptr=" #include #include diff --git a/test/ct_autofree_new_nothrow.cpp b/test/ct_autofree_new_nothrow.cpp index b8c4ad3..37322e0 100644 --- a/test/ct_autofree_new_nothrow.cpp +++ b/test/ct_autofree_new_nothrow.cpp @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: none +// expect: output="auto-free ptr=" #include int main() diff --git a/test/ct_autofree_posix_memalign.c b/test/ct_autofree_posix_memalign.c index 3d61090..22091d3 100644 --- a/test/ct_autofree_posix_memalign.c +++ b/test/ct_autofree_posix_memalign.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: none +// expect: output="auto-free ptr=" #include int main(void) diff --git a/test/ct_autofree_ptrtoint.c b/test/ct_autofree_ptrtoint.c index 8c045a5..31633d3 100644 --- a/test/ct_autofree_ptrtoint.c +++ b/test/ct_autofree_ptrtoint.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: none +// expect: output="auto-free ptr=" #include #include diff --git a/test/ct_autofree_ptrtoint_escape.c b/test/ct_autofree_ptrtoint_escape.c index 65ace11..8050f54 100644 --- a/test/ct_autofree_ptrtoint_escape.c +++ b/test/ct_autofree_ptrtoint_escape.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=12 +// expect: program-exit=1 #include #include diff --git a/test/ct_autofree_return_unused.c b/test/ct_autofree_return_unused.c index cdfd445..365a769 100644 --- a/test/ct_autofree_return_unused.c +++ b/test/ct_autofree_return_unused.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: none +// expect: output="auto-free ptr=" #include void* bar(void) diff --git a/test/ct_autofree_sbrk.c b/test/ct_autofree_sbrk.c index bd5ab71..111d7a5 100644 --- a/test/ct_autofree_sbrk.c +++ b/test/ct_autofree_sbrk.c @@ -1,4 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// sbrk is a real allocation on Linux; macOS emulates it outside the runtime's view. +// expect: none +// expect[linux]: output="auto-free ptr=" #include int main(void) diff --git a/test/ct_autofree_select.c b/test/ct_autofree_select.c index cddc8de..6f1adb0 100644 --- a/test/ct_autofree_select.c +++ b/test/ct_autofree_select.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: none +// expect: output="auto-free ptr=" #include int main(void) diff --git a/test/ct_autofree_select_escape.c b/test/ct_autofree_select_escape.c index be85226..ba11431 100644 --- a/test/ct_autofree_select_escape.c +++ b/test/ct_autofree_select_escape.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=trace,alloc --ct-autofree +// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=12 +// expect: program-exit=1 #include #include diff --git a/test/ct_new_delete.cpp b/test/ct_new_delete.cpp index d4c1436..4db138b 100644 --- a/test/ct_new_delete.cpp +++ b/test/ct_new_delete.cpp @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=11 int main() { int* p = new int(7); diff --git a/test/ct_new_delete_sized.cpp b/test/ct_new_delete_sized.cpp index a3c1c58..87356e7 100644 --- a/test/ct_new_delete_sized.cpp +++ b/test/ct_new_delete_sized.cpp @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: none #include #include diff --git a/test/ct_new_delete_variants.cpp b/test/ct_new_delete_variants.cpp index 153d7d9..1a98b8b 100644 --- a/test/ct_new_delete_variants.cpp +++ b/test/ct_new_delete_variants.cpp @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: none #include #include diff --git a/test/ct_realloc_zero.c b/test/ct_realloc_zero.c index 114a852..ebff89d 100644 --- a/test/ct_realloc_zero.c +++ b/test/ct_realloc_zero.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +// realloc(p, 0) frees on glibc; macOS returns a new block that is never freed. +// expect[linux]: none +// expect[darwin]: rule=memory-leak cwe=CWE-401 level=warning alloc=10 #include int main(void) diff --git a/test/ct_shadow_pages.c b/test/ct_shadow_pages.c index df26f04..dddff68 100644 --- a/test/ct_shadow_pages.c +++ b/test/ct_shadow_pages.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: none #include #include #include diff --git a/test/ct_vtable_basic.cpp b/test/ct_vtable_basic.cpp index fc41561..f4be235 100644 --- a/test/ct_vtable_basic.cpp +++ b/test/ct_vtable_basic.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: none #include struct Base diff --git a/test/ct_vtable_diag_fake.cpp b/test/ct_vtable_diag_fake.cpp index 978e060..b8b0f56 100644 --- a/test/ct_vtable_diag_fake.cpp +++ b/test/ct_vtable_diag_fake.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: rule=vtable-corrupted cwe=CWE-843 level=error line=31 #include #include @@ -26,7 +28,7 @@ int main() FakeObject obj{}; obj.vptr = &table[2]; // vtable pointer points to heap (unresolvable module) - __ct_vtable_dump(&obj, "ct_vtable_diag_fake.cpp:22:5", "FakeObject"); + __ct_vtable_dump(&obj, "ct_vtable_diag_fake.cpp:31:5", "FakeObject"); std::free(table); return 0; diff --git a/test/ct_vtable_diag_freed.cpp b/test/ct_vtable_diag_freed.cpp index 448a4c6..4a66b53 100644 --- a/test/ct_vtable_diag_freed.cpp +++ b/test/ct_vtable_diag_freed.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: rule=vtable-use-after-free cwe=CWE-416 level=error line=33 #include extern "C" void __ct_vtable_dump(void* this_ptr, const char* site, const char* static_type); @@ -28,7 +30,7 @@ int main() delete ptr; // UB: intentionally using a freed pointer to trigger the diagnostic. - __ct_vtable_dump(ptr, "ct_vtable_diag_freed.cpp:22:5", "Derived"); + __ct_vtable_dump(ptr, "ct_vtable_diag_freed.cpp:33:5", "Derived"); return 0; } diff --git a/test/ct_vtable_diag_mismatch.cpp b/test/ct_vtable_diag_mismatch.cpp index cda1d05..760d7f6 100644 --- a/test/ct_vtable_diag_mismatch.cpp +++ b/test/ct_vtable_diag_mismatch.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: rule=vtable-type-mismatch cwe=CWE-843 level=warning line=41 #include #include @@ -36,7 +38,7 @@ int main() return 1; } - __ct_vcall_trace(base, target, "ct_vtable_diag_mismatch.cpp:24:5", "Base"); + __ct_vcall_trace(base, target, "ct_vtable_diag_mismatch.cpp:41:5", "Base"); return 0; } diff --git a/test/ct_vtable_diag_null.cpp b/test/ct_vtable_diag_null.cpp index c315d49..6cde5fc 100644 --- a/test/ct_vtable_diag_null.cpp +++ b/test/ct_vtable_diag_null.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: rule=vtable-null-this cwe=CWE-476 level=error line=11 #include extern "C" void __ct_vtable_dump(void* this_ptr, const char* site, const char* static_type); @@ -6,6 +8,6 @@ extern "C" void __ct_vtable_dump(void* this_ptr, const char* site, const char* s int main() { std::puts("ct_vtable_diag_null"); - __ct_vtable_dump(nullptr, "ct_vtable_diag_null.cpp:7:5", "Base"); + __ct_vtable_dump(nullptr, "ct_vtable_diag_null.cpp:11:5", "Base"); return 0; } diff --git a/test/ct_vtable_diag_stack_target.cpp b/test/ct_vtable_diag_stack_target.cpp index 3a600bf..fa81731 100644 --- a/test/ct_vtable_diag_stack_target.cpp +++ b/test/ct_vtable_diag_stack_target.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: rule=vcall-invalid-target cwe=CWE-843 level=error line=34 #include extern "C" void __ct_vcall_trace(void* this_ptr, void* target, const char* site, @@ -29,7 +31,7 @@ int main() Base* base = &obj; int local = 42; - __ct_vcall_trace(base, &local, "ct_vtable_diag_stack_target.cpp:25:5", "Derived"); + __ct_vcall_trace(base, &local, "ct_vtable_diag_stack_target.cpp:34:5", "Derived"); return 0; } diff --git a/test/ct_vtable_interface.cpp b/test/ct_vtable_interface.cpp index ab4320e..9fd0846 100644 --- a/test/ct_vtable_interface.cpp +++ b/test/ct_vtable_interface.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: none #include struct IFace diff --git a/test/ct_vtable_multi.cpp b/test/ct_vtable_multi.cpp index 66f3a37..02ec6c6 100644 --- a/test/ct_vtable_multi.cpp +++ b/test/ct_vtable_multi.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: none #include struct Base1 diff --git a/test/ct_vtable_uaf.cpp b/test/ct_vtable_uaf.cpp index f632d23..85c5536 100644 --- a/test/ct_vtable_uaf.cpp +++ b/test/ct_vtable_uaf.cpp @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 -// test/ct_vtable_uaf.cpp +// The access faults before the vtable module sees the freed vptr: the alloc module +// reports the use after free. +// expect: rule=heap-use-after-free cwe=CWE-416 level=error line=27 alloc=24 #include struct Base { diff --git a/test/ct_vtable_virtual_base.cpp b/test/ct_vtable_virtual_base.cpp index b82a595..0877fe3 100644 --- a/test/ct_vtable_virtual_base.cpp +++ b/test/ct_vtable_virtual_base.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,vtable --ct-vtable-diag +// expect: none #include struct VBase diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile deleted file mode 100644 index 935bd12..0000000 --- a/test/docker/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM ubuntu:24.04 - -ARG DEBIAN_FRONTEND=noninteractive -ARG LLVM_VERSION=20 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - gnupg \ - lsb-release \ - software-properties-common \ - build-essential \ - cmake \ - ninja-build \ - python3-venv \ - python3 \ - git \ - && rm -rf /var/lib/apt/lists/* - -# Install LLVM/Clang toolchain -RUN curl -fsSL https://apt.llvm.org/llvm.sh -o /tmp/llvm.sh \ - && chmod +x /tmp/llvm.sh \ - && /tmp/llvm.sh ${LLVM_VERSION} \ - && rm -f /tmp/llvm.sh \ - && apt-get update \ - && apt-get install -y --no-install-recommends libclang-20-dev \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /repo -COPY . /repo - -RUN rm -rf build \ - && cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_DIR=/usr/lib/llvm-${LLVM_VERSION}/lib/cmake/llvm \ - -DClang_DIR=/usr/lib/llvm-${LLVM_VERSION}/lib/cmake/clang \ - -DCLANG_LINK_CLANG_DYLIB=ON \ - -DLLVM_LINK_LLVM_DYLIB=ON \ - -DUSE_SHARED_LIB=OFF \ - && cmake --build build -j"$(nproc)" - -RUN bash test/scripts/linux_compile.sh - -RUN python3 -m venv .venv \ - && . .venv/bin/activate \ - && python -m pip install --upgrade pip \ - && python -m pip install git+https://github.com/CoreTrace/coretrace-testkit.git \ - && python test/examples/test_smoke.py \ - && python test/examples/test_extern_project.py \ - && python test/examples/test_help_smoke.py diff --git a/test/examples/fixtures/cpp_as_c.c b/test/examples/fixtures/cpp_as_c.c index e054a7e..9ec5bf1 100644 --- a/test/examples/fixtures/cpp_as_c.c +++ b/test/examples/fixtures/cpp_as_c.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,bounds -x c++ +// expect: none #include int main() diff --git a/test/examples/fixtures/debug.c b/test/examples/fixtures/debug.c index c038b0e..ee88b58 100644 --- a/test/examples/fixtures/debug.c +++ b/test/examples/fixtures/debug.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// args: --ct-modules=alloc,bounds -DDEBUG +// expect: none #ifndef DEBUG #error "DEBUG not defined" #endif diff --git a/test/examples/fixtures/hello.c b/test/examples/fixtures/hello.c index 05a5c82..9b7cf70 100644 --- a/test/examples/fixtures/hello.c +++ b/test/examples/fixtures/hello.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: none #include int main() { diff --git a/test/examples/fixtures/hello.cpp b/test/examples/fixtures/hello.cpp index e054a7e..717051b 100644 --- a/test/examples/fixtures/hello.cpp +++ b/test/examples/fixtures/hello.cpp @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: none #include int main() diff --git a/test/examples/fixtures/vtable.cpp b/test/examples/fixtures/vtable.cpp index 3041f99..7ee9415 100644 --- a/test/examples/fixtures/vtable.cpp +++ b/test/examples/fixtures/vtable.cpp @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: none +// expect: program-exit=1 struct Base { virtual ~Base() = default; diff --git a/test/examples/test_extern_project.py b/test/examples/test_extern_project.py deleted file mode 100644 index 02111e0..0000000 --- a/test/examples/test_extern_project.py +++ /dev/null @@ -1,306 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations -from pathlib import Path -import os -import shutil -import subprocess -import tempfile - -from ctestfw.runner import CompilerRunner, RunnerConfig -from ctestfw.plan import CompilePlan -from ctestfw.framework.testcase import TestCase -from ctestfw.framework.reporter import ConsoleReporter -from ctestfw.assertions.core import Assertion, require -from ctestfw.assertions.compiler import ( - assert_exit_code, - assert_output_name, - assert_output_exists, - assert_native_binary_kind, -) - -ROOT = Path(__file__).resolve().parents[2] -EXTERN = ROOT / "extern-project" -FIXTURES = EXTERN / "tests" -WORK = EXTERN / ".work" -BUILD = EXTERN / ".build-python" - - -def _platform_executable(path: Path) -> Path: - if os.name == "nt" and path.suffix.lower() != ".exe": - return path.with_suffix(".exe") - return path - - -def _powershell_quote(value: str) -> str: - return "'" + value.replace("'", "''") + "'" - - -def _read_artifact_bytes(res, path: str) -> bytes: - artifact = Path(path) - if not artifact.is_absolute(): - artifact = res.run.cwd / artifact - require(artifact.exists(), f"output does not exist: {artifact}") - return artifact.read_bytes() - - -def _is_windows_native_artifact(data: bytes) -> bool: - if data.startswith(b"MZ"): - return True - if len(data) < 2: - return False - return data[:2] in {b"\x64\x86", b"\x4c\x01", b"\x64\xaa"} - - -def assert_windows_native_artifact(path: str) -> Assertion: - def _check(res) -> None: - data = _read_artifact_bytes(res, path) - require( - _is_windows_native_artifact(data), - f"expected PE/COFF artifact at {path}, got unrecognized header", - ) - return Assertion(name=f"windows_native_artifact_{Path(path).name}", check=_check) - - -def run_cmd(argv: list[str], cwd: Path) -> tuple[int, str, str]: - try: - p = subprocess.run( - argv, - cwd=str(cwd), - capture_output=True, - text=True, - ) - return p.returncode, p.stdout or "", p.stderr or "" - except FileNotFoundError: - return 127, "", f"command not found: {argv[0]}" - - -def find_windows_dev_shell() -> str | None: - program_files_x86 = os.environ.get("ProgramFiles(x86)") - if not program_files_x86: - return None - - vswhere = Path(program_files_x86) / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" - if not vswhere.exists(): - return None - - rc, out, _ = run_cmd([ - str(vswhere), - "-latest", - "-products", "*", - "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", - "-property", "installationPath", - ], ROOT) - if rc != 0 or not out.strip(): - return None - - dev_shell = Path(out.strip()) / "Common7" / "Tools" / "Launch-VsDevShell.ps1" - if not dev_shell.exists(): - return None - return str(dev_shell) - - -def run_cmd_in_vsdev(argv: list[str], cwd: Path) -> tuple[int, str, str]: - dev_shell = find_windows_dev_shell() - if dev_shell is None: - return 127, "", "Visual Studio developer shell not found" - - quoted_args = ", ".join(_powershell_quote(arg) for arg in argv) - script = ( - "$ErrorActionPreference = 'Stop'; " - f". {_powershell_quote(dev_shell)} -Arch amd64 -HostArch amd64 | Out-Null; " - f"$cmd = @({quoted_args}); " - "& $cmd[0] @($cmd[1..($cmd.Length - 1)])" - ) - return run_cmd( - ["powershell", "-ExecutionPolicy", "Bypass", "-Command", script], - cwd, - ) - - -def read_cache_var(cache_path: Path, key: str) -> str | None: - if not cache_path.exists(): - return None - prefix = f"{key}:" - try: - for line in cache_path.read_text().splitlines(): - if line.startswith(prefix) and "=" in line: - return line.split("=", 1)[1].strip() - except OSError: - return None - return None - - -def detect_llvm_clang_dirs() -> tuple[str | None, str | None]: - llvm_dir = os.environ.get("LLVM_DIR") - clang_dir = os.environ.get("Clang_DIR") - - # Try to reuse the main build configuration if present. - if not llvm_dir or not clang_dir: - for cache in (ROOT / "build" / "CMakeCache.txt", ROOT / "build-win" / "CMakeCache.txt"): - llvm_dir = llvm_dir or read_cache_var(cache, "LLVM_DIR") - clang_dir = clang_dir or read_cache_var(cache, "Clang_DIR") - if llvm_dir and clang_dir: - break - - # Try llvm-config if still missing. - if not llvm_dir: - rc, out, _ = run_cmd(["llvm-config", "--cmakedir"], ROOT) - if rc == 0 and out.strip(): - llvm_dir = out.strip() - - # Derive clang dir from llvm dir if possible. - if llvm_dir and not clang_dir: - llvm_path = Path(llvm_dir) - if llvm_path.name == "llvm": - candidate = llvm_path.parent / "clang" - if candidate.exists(): - clang_dir = str(candidate) - - # Brew fallback (macOS) if still missing. - if not llvm_dir or not clang_dir: - for formula in ("llvm@20", "llvm@19", "llvm"): - rc, out, _ = run_cmd(["brew", "--prefix", formula], ROOT) - if rc == 0 and out.strip(): - prefix = Path(out.strip()) - llvm_candidate = prefix / "lib" / "cmake" / "llvm" - clang_candidate = prefix / "lib" / "cmake" / "clang" - if not llvm_dir and llvm_candidate.exists(): - llvm_dir = str(llvm_candidate) - if not clang_dir and clang_candidate.exists(): - clang_dir = str(clang_candidate) - if llvm_dir and clang_dir: - break - - return llvm_dir, clang_dir - - -def configure_and_build() -> Path: - if os.name == "nt" and BUILD.exists(): - shutil.rmtree(BUILD, ignore_errors=True) - - if BUILD.exists(): - cache = BUILD / "CMakeCache.txt" - cached_src = read_cache_var(cache, "CMAKE_HOME_DIRECTORY") - if cached_src and Path(cached_src).resolve() != EXTERN.resolve(): - shutil.rmtree(BUILD, ignore_errors=True) - - cmake_args = [ - "cmake", - "-S", str(EXTERN), - "-B", str(BUILD), - "-DCMAKE_BUILD_TYPE=Release", - f"-DFETCHCONTENT_SOURCE_DIR_CC={ROOT}", - ] - llvm_dir, clang_dir = detect_llvm_clang_dirs() - runner = run_cmd - if os.name == "nt": - llvm_root = Path(llvm_dir).parents[2] if llvm_dir else None - clang_cl = llvm_root / "bin" / "clang-cl.exe" if llvm_root else None - cmake_args.extend([ - "-G", "Ninja Multi-Config", - ]) - if clang_cl and clang_cl.exists(): - cmake_args.append(f"-DCMAKE_C_COMPILER={clang_cl}") - cmake_args.append(f"-DCMAKE_CXX_COMPILER={clang_cl}") - runner = run_cmd_in_vsdev - if llvm_dir: - cmake_args.append(f"-DLLVM_DIR={llvm_dir}") - if clang_dir: - cmake_args.append(f"-DClang_DIR={clang_dir}") - logger_source_dir = os.environ.get("CORETRACE_LOGGER_SOURCE_DIR") - if logger_source_dir: - cmake_args.append(f"-DFETCHCONTENT_SOURCE_DIR_CORETRACE_LOGGER={Path(logger_source_dir).resolve()}") - if not llvm_dir or not clang_dir: - print("LLVM/Clang CMake dirs not found. Set LLVM_DIR and Clang_DIR, or build the main project first.") - - rc, out, err = runner(cmake_args, ROOT) - if rc != 0: - print("cmake configure failed") - print(out) - print(err) - raise SystemExit(1) - - build_args = ["cmake", "--build", str(BUILD), "--config", "Release"] - if os.name != "nt": - build_args.append("-j") - rc, out, err = runner(build_args, ROOT) - if rc != 0: - print("cmake build failed") - print(out) - print(err) - raise SystemExit(1) - - cc1 = _platform_executable(BUILD / "cc1") - if not cc1.exists(): - cc1 = _platform_executable(BUILD / "Release" / "cc1") - if not cc1.exists(): - cc1 = _platform_executable(BUILD / "Debug" / "cc1") - if not cc1.exists(): - print(f"cc1 binary not found after build: {cc1}") - raise SystemExit(1) - return cc1.resolve() - - -def copy_fixtures(ws: Path, files: list[Path]) -> None: - for f in files: - dst = ws / f.name - shutil.copy2(f, dst) - - -def main() -> int: - cc1 = configure_and_build() - runner = CompilerRunner(RunnerConfig(executable=cc1)) - - src_c = FIXTURES / "hello.c" - src_cpp = FIXTURES / "hello.cpp" - for src in (src_c, src_cpp): - if not src.exists(): - print(f"fixture not found: {src}") - return 1 - - tc_c = TestCase( - name="extern_project_compile_c", - plan=CompilePlan( - name="extern_project_compile_c", - sources=[Path("hello.c")], - out=Path("hello_ext_c.o"), - extra_args=["-c"], - ), - assertions=[ - assert_exit_code(0), - assert_output_name("hello_ext_c.o"), - assert_output_exists(), - assert_windows_native_artifact("hello_ext_c.o") if os.name == "nt" else assert_native_binary_kind(), - ], - ) - - tc_cpp = TestCase( - name="extern_project_compile_cpp", - plan=CompilePlan( - name="extern_project_compile_cpp", - sources=[Path("hello.cpp")], - out=Path("hello_ext_cpp.o"), - extra_args=["-c"], - ), - assertions=[ - assert_exit_code(0), - assert_output_name("hello_ext_cpp.o"), - assert_output_exists(), - assert_windows_native_artifact("hello_ext_cpp.o") if os.name == "nt" else assert_native_binary_kind(), - ], - ) - - WORK.mkdir(parents=True, exist_ok=True) - reports = [] - for tc in (tc_c, tc_cpp): - with tempfile.TemporaryDirectory(prefix=f"{tc.name}_", dir=str(WORK)) as d: - ws = Path(d) - copy_fixtures(ws, [src_c, src_cpp]) - reports.append(tc.run(runner, ws)) - - rep = type("Tmp", (), {"name": "extern_project", "reports": reports})() - return ConsoleReporter().render(rep) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test/examples/test_help_smoke.py b/test/examples/test_help_smoke.py deleted file mode 100644 index e8cfe50..0000000 --- a/test/examples/test_help_smoke.py +++ /dev/null @@ -1,66 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -from pathlib import Path -import shutil -import tempfile - -from ctestfw.runner import CompilerRunner, RunnerConfig -from ctestfw.plan import CompilePlan -from ctestfw.framework.testcase import TestCase -from ctestfw.framework.reporter import ConsoleReporter -from ctestfw.assertions.compiler import ( - assert_exit_code, - assert_stdout_contains, -) - -ROOT = Path(__file__).resolve().parents[2] -FIXTURES = ROOT / "test" / "examples" / "fixtures" -WORK = ROOT / "test" / "examples" / ".work" - - -def copy_fixtures(ws: Path, files: list[Path]) -> None: - for f in files: - dst = ws / f.name - shutil.copy2(f, dst) - - -def main() -> int: - cc_bin = (ROOT / "build" / "cc").resolve() - runner = CompilerRunner(RunnerConfig(executable=cc_bin)) - if not cc_bin.exists(): - print(f"cc binary not found: {cc_bin}") - return 1 - - src = FIXTURES / "hello.c" - if not src.exists(): - print(f"fixture not found: {src}") - return 1 - - tc_help = TestCase( - name="help_smoke", - plan=CompilePlan( - name="help_smoke", - sources=[Path("hello.c")], - out=None, - extra_args=["--help"], - ), - assertions=[ - assert_exit_code(0), - assert_stdout_contains("Usage:"), - assert_stdout_contains("Core options:"), - assert_stdout_contains("--instrument"), - assert_stdout_contains("Exit codes:"), - ], - ) - - WORK.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=f"{tc_help.name}_", dir=str(WORK)) as d: - ws = Path(d) - copy_fixtures(ws, [src]) - report = tc_help.run(runner, ws) - - rep = type("Tmp", (), {"name": "help_smoke", "reports": [report]})() - return ConsoleReporter().render(rep) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test/examples/test_smoke.py b/test/examples/test_smoke.py deleted file mode 100644 index 9434a5a..0000000 --- a/test/examples/test_smoke.py +++ /dev/null @@ -1,571 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -import os -from pathlib import Path -import shutil - -from ctestfw.runner import CompilerRunner, RunnerConfig -from ctestfw.plan import CompilePlan -from ctestfw.framework.testcase import TestCase -from ctestfw.framework.suite import TestSuite -from ctestfw.framework.reporter import ConsoleReporter -from ctestfw.assertions.core import Assertion, require -from ctestfw.assertions.compiler import ( - assert_exit_code, - assert_argv_contains, - assert_output_exists, - assert_output_name, - assert_output_kind, - assert_native_binary_kind, - assert_output_exists_at, - assert_native_binary_kind_at, - assert_output_kind_at, - assert_output_nonempty_at, - assert_stdout_contains -) -from ctestfw.inspect.filetype import ArtifactKind -from ctestfw.platform import detect_platform, OS - -ROOT = Path(__file__).resolve().parents[2] -FIXTURES = ROOT / "test" / "examples" / "fixtures" -WORK = ROOT / "test" / "examples" / ".work" - -def copy_fixtures(ws: Path, files: list[Path]) -> None: - for f in files: - src = f - dst = ws / f.name - shutil.copy2(src, dst) - -def assert_file_contains(path: str, text: str) -> Assertion: - def _check(res) -> None: - p = Path(path) - if not p.is_absolute(): - p = res.run.cwd / p - require(p.exists(), f"output does not exist: {p}") - data = p.read_text(encoding="utf-8", errors="ignore") - require(text in data, f"file does not contain '{text}': {p}") - return Assertion(name=f"file_contains_{Path(path).name}", check=_check) - -def assert_stderr_contains(text: str) -> Assertion: - def _check(res) -> None: - require(text in (res.run.stderr or ""), - f"stderr does not contain '{text}'\nstderr:\n{res.run.stderr}") - return Assertion(name=f"stderr_contains_{text}", check=_check) - -def _read_artifact_bytes(res, path: str) -> bytes: - artifact = Path(path) - if not artifact.is_absolute(): - artifact = res.run.cwd / artifact - require(artifact.exists(), f"output does not exist: {artifact}") - return artifact.read_bytes() - -def _is_windows_native_artifact(data: bytes) -> bool: - if data.startswith(b"MZ"): - return True - if len(data) < 2: - return False - return data[:2] in {b"\x64\x86", b"\x4c\x01", b"\x64\xaa"} - -def assert_windows_native_artifact_at(path: str) -> Assertion: - def _check(res) -> None: - data = _read_artifact_bytes(res, path) - require( - _is_windows_native_artifact(data), - f"expected PE/COFF artifact at {path}, got unrecognized header", - ) - return Assertion(name=f"windows_native_artifact_{Path(path).name}", check=_check) - -def native_artifact_assert_at(path: str, platform_os: OS) -> Assertion: - if platform_os == OS.WINDOWS: - return assert_windows_native_artifact_at(path) - return assert_native_binary_kind_at(path) - -def resolve_compiler_binary() -> Path | None: - candidates: list[Path] = [] - - env_override = os.environ.get("CORETRACE_COMPILER_TEST_CC") - if env_override: - candidates.append(Path(env_override)) - - candidates.extend([ - ROOT / "dist" / "windows" / "bin" / "cc.exe", - ROOT / "build" / "cc", - ROOT / "build" / "Release" / "cc.exe", - ROOT / "build-win" / "cc.exe", - ROOT / "build-win" / "Release" / "cc.exe", - ]) - - for candidate in candidates: - if candidate.exists(): - return candidate.resolve() - - return None - -def main() -> int: - platform = detect_platform() - cc_bin = resolve_compiler_binary() - if cc_bin is None: - print("cc binary not found. Tried:") - for candidate in [ - os.environ.get("CORETRACE_COMPILER_TEST_CC", ""), - str(ROOT / "dist" / "windows" / "bin" / "cc.exe"), - str(ROOT / "build" / "cc"), - str(ROOT / "build" / "Release" / "cc.exe"), - str(ROOT / "build-win" / "cc.exe"), - str(ROOT / "build-win" / "Release" / "cc.exe"), - ]: - if candidate: - print(f" - {candidate}") - return 1 - - runner = CompilerRunner(RunnerConfig(executable=cc_bin)) - - # Fixtures (ex: hello.c) - src = FIXTURES / "hello.c" - debug_src = FIXTURES / "debug.c" - cpp_src = FIXTURES / "hello.cpp" - cpp_as_c_src = FIXTURES / "cpp_as_c.c" - vtable_src = FIXTURES / "vtable.cpp" - - def base_out_assertions(out_name: str): - assertions = [ - assert_exit_code(0), - assert_argv_contains(["-o"]), # check args passed - assert_output_name(out_name), # check binary name respected - assert_output_exists(), - ] - if platform.os == OS.WINDOWS: - assertions.append(assert_windows_native_artifact_at(out_name)) - return assertions - - tc_macho = TestCase( - name="compile_macho_hello", - plan=CompilePlan( - name="compile_macho_hello", - sources=[Path("hello.c")], # sera copié dans workspace - out=Path("hello.out"), - extra_args=[], - ), - assertions=base_out_assertions("hello.out") + [ - assert_output_kind(ArtifactKind.MACHO), - ], - ) - - tc_elf = TestCase( - name="compile_elf_hello", - plan=CompilePlan( - name="compile_elf_hello", - sources=[Path("hello.c")], - out=Path("hello.out"), - extra_args=[], - ), - assertions=base_out_assertions("hello.out") + [ - assert_output_kind(ArtifactKind.ELF), - ], - ) - - tc_native = TestCase( - name="compile_native_hello", - plan=CompilePlan( - name="compile_native_hello", - sources=[Path("hello.c")], - out=Path("hello.out"), - extra_args=[], - ), - assertions=base_out_assertions("hello.out") + [ - assert_windows_native_artifact_at("hello.out") if platform.os == OS.WINDOWS else assert_native_binary_kind(), - ], - ) - - tc_cpp = TestCase( - name="compile_cpp_hello", - plan=CompilePlan( - name="compile_cpp_hello", - sources=[Path("hello.cpp")], - out=Path("hello_cpp.out"), - extra_args=[], - ), - assertions=base_out_assertions("hello_cpp.out") + [ - assert_windows_native_artifact_at("hello_cpp.out") if platform.os == OS.WINDOWS else assert_native_binary_kind(), - ], - ) - - tc_o_eq = TestCase( - name="compile_o_equals", - plan=CompilePlan( - name="compile_o_equals", - sources=[Path("hello.c")], - out=None, - extra_args=["-o=main"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["-o=main"]), - assert_output_exists_at("main"), - native_artifact_assert_at("main", platform.os), - ], - ) - - tc_d_space = TestCase( - name="compile_define_space", - plan=CompilePlan( - name="compile_define_space", - sources=[Path("debug.c")], - out=None, - extra_args=["-D", "DEBUG", "-o=debug_space"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["-D", "DEBUG"]), - assert_output_exists_at("debug_space"), - native_artifact_assert_at("debug_space", platform.os), - ], - ) - - tc_d_compact = TestCase( - name="compile_define_compact", - plan=CompilePlan( - name="compile_define_compact", - sources=[Path("debug.c")], - out=None, - extra_args=["-DDEBUG", "-o=debug_compact"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["-DDEBUG"]), - assert_output_exists_at("debug_compact"), - native_artifact_assert_at("debug_compact", platform.os), - ], - ) - - tc_x_cxx = TestCase( - name="compile_x_cxx", - plan=CompilePlan( - name="compile_x_cxx", - sources=[], - out=None, - extra_args=["-x=c++", "cpp_as_c.c", "-o=hello_xcxx.out"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["-x=c++"]), - assert_output_exists_at("hello_xcxx.out"), - native_artifact_assert_at("hello_xcxx.out", platform.os), - ], - ) - - tc_instrument_c = TestCase( - name="compile_instrument_c", - plan=CompilePlan( - name="compile_instrument_c", - sources=[Path("hello.c")], - out=Path("hello_instr_c.out"), - extra_args=["--instrument"], - ), - assertions=base_out_assertions("hello_instr_c.out") + [ - assert_argv_contains(["--instrument"]), - assert_windows_native_artifact_at("hello_instr_c.out") if platform.os == OS.WINDOWS else assert_native_binary_kind(), - ], - ) - - tc_instrument_cpp = TestCase( - name="compile_instrument_cpp", - plan=CompilePlan( - name="compile_instrument_cpp", - sources=[Path("hello.cpp")], - out=Path("hello_instr_cpp.out"), - extra_args=["--instrument"], - ), - assertions=base_out_assertions("hello_instr_cpp.out") + [ - assert_argv_contains(["--instrument"]), - assert_windows_native_artifact_at("hello_instr_cpp.out") if platform.os == OS.WINDOWS else assert_native_binary_kind(), - ], - ) - - tc_instrument_x_cxx = TestCase( - name="compile_instrument_x_cxx", - plan=CompilePlan( - name="compile_instrument_x_cxx", - sources=[], - out=None, - extra_args=["--instrument", "-x=c++", "cpp_as_c.c", "-o=hello_instr_xcxx.out"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--instrument", "-x=c++"]), - assert_output_exists_at("hello_instr_xcxx.out"), - native_artifact_assert_at("hello_instr_xcxx.out", platform.os), - ], - ) - - tc_instrument_emit_llvm = TestCase( - name="compile_instrument_emit_llvm", - plan=CompilePlan( - name="compile_instrument_emit_llvm", - sources=[Path("hello.c")], - out=None, - extra_args=["--instrument", "-S", "-emit-llvm", "-o=hello_instr.ll"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--instrument", "-S", "-emit-llvm"]), - assert_output_exists_at("hello_instr.ll"), - assert_output_kind_at("hello_instr.ll", ArtifactKind.LLVM_IR_TEXT), - assert_output_nonempty_at("hello_instr.ll"), - ], - ) - - tc_instrument_emit_bc = TestCase( - name="compile_instrument_emit_bc", - plan=CompilePlan( - name="compile_instrument_emit_bc", - sources=[Path("hello.c")], - out=None, - extra_args=["--instrument", "-c", "-emit-llvm", "-o=hello_instr.bc"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--instrument", "-c", "-emit-llvm"]), - assert_output_exists_at("hello_instr.bc"), - assert_output_nonempty_at("hello_instr.bc"), - ], - ) - - tc_readme_emit_llvm = TestCase( - name="readme_emit_llvm", - plan=CompilePlan( - name="readme_emit_llvm", - sources=[Path("hello.cpp")], - out=None, - extra_args=["-S", "-emit-llvm"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["-S", "-emit-llvm"]), - assert_output_kind_at("hello.ll", ArtifactKind.LLVM_IR_TEXT), - ], - ) - - tc_readme_asm = TestCase( - name="readme_asm", - plan=CompilePlan( - name="readme_asm", - sources=[Path("hello.cpp")], - out=None, - extra_args=["-S"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["-S"]), - assert_output_nonempty_at("hello.s"), - ], - ) - - tc_readme_c_obj = TestCase( - name="readme_c_obj", - plan=CompilePlan( - name="readme_c_obj", - sources=[Path("hello.c")], - out=None, - extra_args=["-c"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["-c"]), - native_artifact_assert_at("hello.o", platform.os), - ], - ) - - tc_readme_c_obj_o2 = TestCase( - name="readme_c_obj_o2", - plan=CompilePlan( - name="readme_c_obj_o2", - sources=[Path("hello.c")], - out=None, - extra_args=["-c", "-O2"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["-c", "-O2"]), - native_artifact_assert_at("hello.o", platform.os), - ], - ) - - tc_readme_instrument = TestCase( - name="readme_instrument", - plan=CompilePlan( - name="readme_instrument", - sources=[Path("hello.c")], - out=None, - extra_args=["--instrument", "-o", "app"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--instrument", "-o", "app"]), - assert_output_exists_at("app"), - native_artifact_assert_at("app", platform.os), - ], - ) - - tc_readme_shadow = TestCase( - name="readme_shadow", - plan=CompilePlan( - name="readme_shadow", - sources=[Path("hello.c")], - out=None, - extra_args=["--instrument", "--ct-shadow", "-o", "app_shadow"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--instrument", "--ct-shadow"]), - assert_output_exists_at("app_shadow"), - native_artifact_assert_at("app_shadow", platform.os), - ], - ) - - tc_readme_shadow_aggr = TestCase( - name="readme_shadow_aggr", - plan=CompilePlan( - name="readme_shadow_aggr", - sources=[Path("hello.c")], - out=None, - extra_args=["--instrument", "--ct-shadow-aggressive", "--ct-bounds-no-abort", "-o", "app_shadow_aggr"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--instrument", "--ct-shadow-aggressive", "--ct-bounds-no-abort"]), - assert_output_exists_at("app_shadow_aggr"), - native_artifact_assert_at("app_shadow_aggr", platform.os), - ], - ) - - tc_readme_vtable = TestCase( - name="readme_vtable", - plan=CompilePlan( - name="readme_vtable", - sources=[Path("vtable.cpp")], - out=None, - extra_args=["--instrument", "--ct-modules=vtable", "--ct-vcall-trace", "-o", "app_vtable"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--instrument", "--ct-modules=vtable", "--ct-vcall-trace"]), - assert_output_exists_at("app_vtable"), - native_artifact_assert_at("app_vtable", platform.os), - ], - ) - - tc_readme_inmem = TestCase( - name="readme_inmem", - plan=CompilePlan( - name="readme_inmem", - sources=[Path("hello.c")], - out=None, - extra_args=["--in-mem", "-S", "-emit-llvm"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--in-mem", "-S", "-emit-llvm"]), - assert_stdout_contains("target triple"), - ], - ) - - tc_optnone_emit_llvm = TestCase( - name="compile_optnone_emit_llvm", - plan=CompilePlan( - name="compile_optnone_emit_llvm", - sources=[Path("hello.c")], - out=None, - extra_args=["--ct-optnone", "-O1", "-S", "-emit-llvm", "-o=hello_optnone.ll"], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--ct-optnone", "-O1", "-S", "-emit-llvm"]), - assert_output_exists_at("hello_optnone.ll"), - assert_output_kind_at("hello_optnone.ll", ArtifactKind.LLVM_IR_TEXT), - assert_output_nonempty_at("hello_optnone.ll"), - assert_file_contains("hello_optnone.ll", "optnone"), - ], - ) - - tc_optnone_disable_o0 = TestCase( - name="compile_optnone_disable_o0", - plan=CompilePlan( - name="compile_optnone_disable_o0", - sources=[Path("hello.c")], - out=None, - extra_args=[ - "--ct-optnone", - "-O0", - "-Xclang", - "-disable-O0-optnone", - "-S", - "-emit-llvm", - "-o", - "-", - ], - ), - assertions=[ - assert_exit_code(0), - assert_argv_contains(["--ct-optnone", "-O0", "-Xclang", "-disable-O0-optnone"]), - assert_stdout_contains("optnone"), - assert_stderr_contains( - "warning: ct: -disable-O0-optnone ignored because --ct-optnone is enabled" - ), - ], - ) - - common_cases = [tc_o_eq, tc_d_space, tc_d_compact, tc_cpp, tc_x_cxx] - instrument_cases = [ - tc_instrument_c, - tc_instrument_cpp, - tc_instrument_x_cxx, - tc_instrument_emit_llvm, - tc_instrument_emit_bc, - ] - readme_cases = [ - tc_readme_emit_llvm, - tc_readme_asm, - tc_readme_c_obj, - tc_readme_c_obj_o2, - tc_readme_instrument, - tc_readme_shadow, - tc_readme_shadow_aggr, - tc_readme_vtable, - tc_readme_inmem, - tc_optnone_emit_llvm, - tc_optnone_disable_o0, - ] - if platform.os == OS.MACOS: - cases = [tc_macho, *common_cases, *instrument_cases, *readme_cases] - elif platform.os == OS.LINUX: - cases = [tc_elf, *common_cases, *instrument_cases, *readme_cases] - else: - windows_readme_cases = [ - tc_readme_emit_llvm, - tc_readme_c_obj, - tc_readme_c_obj_o2, - tc_readme_instrument, - tc_readme_shadow, - tc_readme_shadow_aggr, - tc_readme_inmem, - tc_optnone_emit_llvm, - tc_optnone_disable_o0, - ] - cases = [tc_native, *common_cases, *instrument_cases, *windows_readme_cases] - - suite = TestSuite(name="compiler_smoke", cases=cases) - - reports = [] - WORK.mkdir(parents=True, exist_ok=True) - for case in suite.cases: - import tempfile - with tempfile.TemporaryDirectory(prefix=f"{case.name}_", dir=str(WORK)) as d: - ws = Path(d) - copy_fixtures(ws, [src, debug_src, cpp_src, cpp_as_c_src, vtable_src]) - reports.append(case.run(runner, ws)) - - rep = type("Tmp", (), {"name": suite.name, "reports": reports})() - return ConsoleReporter().render(rep) - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test/findings/check_findings.py b/test/findings/check_findings.py deleted file mode 100755 index ddd647f..0000000 --- a/test/findings/check_findings.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -"""Each runtime memory error becomes one finding with its rule, CWE and source location. - -Usage: check_findings.py -""" - -from __future__ import annotations - -import json -import subprocess -import sys -from pathlib import Path - -HERE = Path(__file__).resolve().parent - -# fixture: (rule, CWE, SARIF level, faulting access line, allocation line). A stack object's -# allocation line is its function's; a leak has no faulting access. -MEMORY_ARGS = ["--ct-modules=alloc,bounds"] -EXPECTED = { - "heap_overflow_write.c": ("heap-buffer-overflow", "CWE-122", "error", 7, 6), - "heap_overflow_read.c": ("heap-buffer-overflow", "CWE-125", "error", 7, 6), - "stack_overflow_write.c": ("stack-buffer-overflow", "CWE-121", "error", 7, 3), - "use_after_free.c": ("heap-use-after-free", "CWE-416", "error", 8, 6), - "double_free.c": ("double-free", "CWE-415", "error", 8, 6), - "memory_leak.c": ("memory-leak", "CWE-401", "warning", None, 6), -} - -# The vtable module's diagnostics, over the fixtures at the root of test/. The boxes carry no -# allocation site. -VTABLE_ARGS = ["--ct-modules=alloc,vtable", "--ct-vtable-diag"] -VTABLE_EXPECTED = { - "ct_vtable_diag_null.cpp": ("vtable-null-this", "CWE-476", "error", 7), - "ct_vtable_diag_freed.cpp": ("vtable-use-after-free", "CWE-416", "error", 22), - "ct_vtable_diag_stack_target.cpp": ("vcall-invalid-target", "CWE-843", "error", 25), - "ct_vtable_diag_mismatch.cpp": ("vtable-type-mismatch", "CWE-843", "warning", 24), - "ct_vtable_diag_fake.cpp": ("vtable-corrupted", "CWE-843", "error", 22), -} - -failures = 0 - - -def expect(condition: bool, message: str) -> None: - global failures - print(("[PASS] " if condition else "[FAIL] ") + message) - if not condition: - failures += 1 - - -def analyze(analyzer: str, work: Path, fixture: str, *options: str, - compiler_args: list[str] = MEMORY_ARGS, source: str | None = None - ) -> subprocess.CompletedProcess: - # Run from test/ with a relative source: the finding must carry the path the source was - # compiled from, as the static analyzers report it. - return subprocess.run( - [analyzer, "-o", str(work / Path(fixture).stem), *options, "--", *compiler_args, - source or f"findings/{fixture}"], - cwd=HERE.parent, - capture_output=True, - text=True, - timeout=120, - ) - - -def region(location: dict) -> tuple[str, int, int]: - physical = location["physicalLocation"] - return ( - physical["artifactLocation"]["uri"], - physical["region"]["startLine"], - physical["region"].get("startColumn", 0), - ) - - -def sarif_results(completed: subprocess.CompletedProcess, fixture: str) -> list[dict] | None: - try: - log = json.loads(completed.stdout) - except json.JSONDecodeError: - expect(False, f"{fixture}: stdout is a JSON document\n{completed.stdout}{completed.stderr}") - return None - expect(log.get("version") == "2.1.0", f"{fixture}: SARIF 2.1.0") - expect(log["runs"][0]["tool"]["driver"]["name"] == "coretrace-runtime-analyzer", - f"{fixture}: tool name") - return log["runs"][0]["results"] - - -def main() -> int: - analyzer, work = str(Path(sys.argv[1]).resolve()), Path(sys.argv[2]).resolve() - work.mkdir(parents=True, exist_ok=True) - - for fixture, (rule, cwe, level, access_line, alloc_line) in EXPECTED.items(): - completed = analyze(analyzer, work, fixture, "--format=sarif") - expect(completed.returncode == 1, f"{fixture}: exit 1 when findings are reported") - results = sarif_results(completed, fixture) or [] - expect(len(results) == 1, f"{fixture}: exactly one finding (got {len(results)})") - if len(results) != 1: - continue - result = results[0] - expect(result.get("ruleId") == rule, f"{fixture}: rule {rule}") - expect(result.get("properties", {}).get("cwe") == cwe, f"{fixture}: {cwe}") - expect(result.get("level") == level, f"{fixture}: level {level}") - expect(bool(result.get("message", {}).get("text")), f"{fixture}: has a message") - if access_line is None: - expect("locations" not in result, f"{fixture}: no faulting access") - else: - uri, line, column = region(result["locations"][0]) - expect(uri == f"findings/{fixture}", f"{fixture}: located in the compiled source") - expect(line == access_line and column > 0, - f"{fixture}: faulting access at line {access_line} (got {line}:{column})") - related = result.get("relatedLocations", [{}])[0] - uri, line, _ = region(related) if related else ("", 0, 0) - expect(uri == f"findings/{fixture}" and line == alloc_line, - f"{fixture}: allocation site at line {alloc_line} (got {uri}:{line})") - expect(related.get("id") == 0 and related.get("message", {}).get("text") == "allocated here", - f"{fixture}: allocation site is related location 0, 'allocated here'") - - for fixture, (rule, cwe, level, line) in VTABLE_EXPECTED.items(): - completed = analyze(analyzer, work, fixture, "--format=sarif", - compiler_args=VTABLE_ARGS, source=fixture) - expect(completed.returncode == 1, f"{fixture}: exit 1 when findings are reported") - results = sarif_results(completed, fixture) or [] - expect(len(results) == 1, f"{fixture}: exactly one finding (got {len(results)})") - if len(results) != 1: - continue - result = results[0] - expect(result.get("ruleId") == rule, f"{fixture}: rule {rule}") - expect(result.get("properties", {}).get("cwe") == cwe, f"{fixture}: {cwe}") - expect(result.get("level") == level, f"{fixture}: level {level}") - uri, got_line, column = region(result["locations"][0]) - expect(uri == fixture and got_line == line and column > 0, - f"{fixture}: site at line {line} (got {uri}:{got_line}:{column})") - expect("relatedLocations" not in result, f"{fixture}: no allocation site") - - completed = analyze(analyzer, work, "ct_vtable_basic.cpp", "--format=sarif", - compiler_args=VTABLE_ARGS, source="ct_vtable_basic.cpp") - expect(completed.returncode == 0 and sarif_results(completed, "ct_vtable_basic.cpp") == [], - "ct_vtable_basic.cpp: tracing boxes are not findings") - - # The program's exit status is reported, but only runtime errors decide the verdict. - completed = analyze(analyzer, work, "no_findings.c", "--format=sarif") - expect(completed.returncode == 0, "no_findings.c: exit 0 without findings") - expect(sarif_results(completed, "no_findings.c") == [], "no_findings.c: no finding") - completed = analyze(analyzer, work, "no_findings.c") - expect(completed.returncode == 0 and "runtime-analyzer: exit_code=3" in completed.stdout, - "no_findings.c: the text summary keeps the program's exit code") - expect("runtime-analyzer: findings=0" in completed.stdout, "no_findings.c: findings=0 in text") - - completed = analyze(analyzer, work, "heap_overflow_write.c") - expect(completed.returncode == 1, "text output: exit 1 with findings") - expect("runtime-analyzer: findings=1" in completed.stdout, "text output: findings=1") - expect("findings/heap_overflow_write.c:7:" in completed.stdout - and "error: heap-buffer-overflow WRITE of size 4 [heap-buffer-overflow, CWE-122]" - in completed.stdout, - "text output: one compiler-style line per finding") - expect("findings/heap_overflow_write.c:6:" in completed.stdout - and "note: allocated here" in completed.stdout, - "text output: the allocation site as a note") - - completed = analyze(analyzer, work, "no_findings.c", "--no-run", "--format=sarif") - expect(completed.returncode == 0, "--no-run: exit 0 after a successful build") - - # --test-dir runs every fixture and reports all their findings in one log. - completed = subprocess.run( - [analyzer, "--test-dir", "findings", "--output-dir", str(work / "batch"), - "--format=sarif", "--", *MEMORY_ARGS], - cwd=HERE.parent, capture_output=True, text=True, timeout=300, - ) - expect(completed.returncode == 1, "batch: exit 1 when any program reports findings") - results = sarif_results(completed, "batch") or [] - expect(sorted(result["ruleId"] for result in results) == - sorted(expected[0] for expected in EXPECTED.values()), - "batch: one log with the findings of every program") - completed = subprocess.run( - [analyzer, "--test-dir", "findings", "--output-dir", str(work / "batch"), - "--", *MEMORY_ARGS], - cwd=HERE.parent, capture_output=True, text=True, timeout=300, - ) - expect(completed.returncode == 1 and "runtime-analyzer: [PASS] findings/no_findings.c" - not in completed.stdout and "findings/heap_overflow_write.c:7:" in completed.stdout, - "batch text: findings are listed under each test's line") - - completed = analyze(analyzer, work, "missing.c") - expect(completed.returncode == 2, "exit 2 when the program cannot be built") - completed = analyze(analyzer, work, "no_findings.c", "--format=xml") - expect(completed.returncode == 2, "exit 2 on an unknown output format") - completed = subprocess.run( - [analyzer, "--test-dir", "findings", "--", "-c"], - cwd=HERE.parent, capture_output=True, text=True, timeout=60, - ) - expect(completed.returncode == 2, "batch: exit 2 on compile-only flags") - - if failures: - print(f"{failures} finding check(s) failed", file=sys.stderr) - return 1 - print("check_findings: all checks passed") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/test/findings/double_free.c b/test/findings/double_free.c index ae8ca5c..eff47be 100644 --- a/test/findings/double_free.c +++ b/test/findings/double_free.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: rule=double-free cwe=CWE-415 level=error line=9 alloc=7 #include int main(void) diff --git a/test/findings/heap_overflow_read.c b/test/findings/heap_overflow_read.c index 73422d2..3adb683 100644 --- a/test/findings/heap_overflow_read.c +++ b/test/findings/heap_overflow_read.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: rule=heap-buffer-overflow cwe=CWE-125 level=error line=8 alloc=7 #include int main(void) diff --git a/test/findings/heap_overflow_write.c b/test/findings/heap_overflow_write.c index 4ee3191..66f3cbc 100644 --- a/test/findings/heap_overflow_write.c +++ b/test/findings/heap_overflow_write.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: rule=heap-buffer-overflow cwe=CWE-122 level=error line=8 alloc=7 #include int main(void) diff --git a/test/findings/memory_leak.c b/test/findings/memory_leak.c index 54525dc..0e00db6 100644 --- a/test/findings/memory_leak.c +++ b/test/findings/memory_leak.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: rule=memory-leak cwe=CWE-401 level=warning alloc=7 #include int main(void) diff --git a/test/findings/no_findings.c b/test/findings/no_findings.c index 6a0e3f4..33e6b06 100644 --- a/test/findings/no_findings.c +++ b/test/findings/no_findings.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: none +// expect: program-exit=3 #include // Exits non-zero without any memory error: the program's status is not a finding. diff --git a/test/findings/stack_overflow_write.c b/test/findings/stack_overflow_write.c index e29c1cf..be92a87 100644 --- a/test/findings/stack_overflow_write.c +++ b/test/findings/stack_overflow_write.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: rule=stack-buffer-overflow cwe=CWE-121 level=error line=8 alloc=4 // Writes one element past a local array, at an index the compiler cannot see. int main(void) { diff --git a/test/findings/use_after_free.c b/test/findings/use_after_free.c index a69332d..a911b35 100644 --- a/test/findings/use_after_free.c +++ b/test/findings/use_after_free.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +// expect: rule=heap-use-after-free cwe=CWE-416 level=error line=9 alloc=7 #include int main(void) diff --git a/test/release/check-analysis.sh b/test/release/check-analysis.sh index f9d426a..e961468 100755 --- a/test/release/check-analysis.sh +++ b/test/release/check-analysis.sh @@ -31,6 +31,6 @@ fail() { exit 1 } [ "$status" -eq 1 ] || fail "expected exit status 1 (findings), got $status" -printf '%s\n' "$output" | grep -q 'heap_overflow_write.c:7:15: error: heap-buffer-overflow WRITE of size 4' || +printf '%s\n' "$output" | grep -q 'heap_overflow_write.c:8:15: error: heap-buffer-overflow WRITE of size 4' || fail "the heap overflow of heap_overflow_write.c is not reported" echo "PASS: runtime-analyzer analyzes C from $prefix without an installed clang" diff --git a/test/run_autofree_tests.sh b/test/run_autofree_tests.sh deleted file mode 100755 index 92a9b4e..0000000 --- a/test/run_autofree_tests.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -CC_BIN="${ROOT_DIR}/build/cc" -OUT_DIR="${1:-/tmp/ct_autofree_tests}" - -if command -v rg >/dev/null 2>&1; then - MATCH_TOOL="rg" -else - MATCH_TOOL="grep" -fi - -has_match() { - local pattern="$1" - local file="$2" - if [[ "${MATCH_TOOL}" == "rg" ]]; then - rg -q "${pattern}" "${file}" - else - grep -q "${pattern}" "${file}" - fi -} - -if [[ ! -x "${CC_BIN}" ]]; then - echo "ERROR: ${CC_BIN} not found or not executable." - echo "Build coretrace-compiler first (cmake --build build)." - exit 1 -fi - -mkdir -p "${OUT_DIR}" - -expect_leak() { - case "$1" in - ct_autofree_local.c|ct_autofree_select_escape.c|ct_autofree_ptrtoint_escape.c|ct_autofree_inttoptr_escape.c) - return 0 - ;; - *) - return 1 - ;; - esac -} - -expect_autofree() { - case "$1" in - ct_autofree_return_unused.c|ct_autofree_select.c|ct_autofree_ptrtoint.c|ct_autofree_inttoptr.c|ct_autofree_new_nothrow.cpp|ct_autofree_posix_memalign.c|ct_autofree_aligned_alloc.c|ct_autofree_mmap.c|ct_autofree_sbrk.c) - return 0 - ;; - *) - return 1 - ;; - esac -} - -expect_nonzero_exit() { - case "$1" in - ct_autofree_select_escape.c|ct_autofree_ptrtoint_escape.c|ct_autofree_inttoptr_escape.c) - return 0 - ;; - *) - return 1 - ;; - esac -} - -TESTS=( - ct_autofree_local.c - ct_autofree_return_unused.c - ct_autofree_select.c - ct_autofree_select_escape.c - ct_autofree_ptrtoint.c - ct_autofree_ptrtoint_escape.c - ct_autofree_inttoptr.c - ct_autofree_inttoptr_escape.c - ct_autofree_new_nothrow.cpp - ct_autofree_posix_memalign.c - ct_autofree_aligned_alloc.c - ct_autofree_mmap.c - ct_autofree_sbrk.c - ct_autofree_brk.c -) - -PASS=0 -FAIL=0 - -run_one() { - local test_file="$1" - local test_path="${ROOT_DIR}/test/${test_file}" - local base="${test_file%.*}" - local bin="${OUT_DIR}/${base}" - local compile_log="${OUT_DIR}/${base}.compile.log" - local run_log="${OUT_DIR}/${base}.run.log" - - echo "==> ${test_file}" - - "${CC_BIN}" --instrument --ct-modules=trace,alloc --ct-autofree \ - "${test_path}" -o "${bin}" >"${compile_log}" 2>&1 || { - echo " FAIL: compile (see ${compile_log})" - return 1 - } - - set +e - "${bin}" >"${run_log}" 2>&1 - local run_rc=$? - set -e - - if expect_nonzero_exit "${test_file}"; then - if [[ "${run_rc}" -eq 0 ]]; then - echo " FAIL: expected non-zero exit, got 0" - return 1 - fi - else - if [[ "${run_rc}" -ne 0 ]]; then - echo " FAIL: run (see ${run_log})" - return 1 - fi - fi - - if expect_leak "${test_file}"; then - if ! has_match "ct: leaks detected" "${run_log}"; then - echo " FAIL: expected leak, none found" - return 1 - fi - else - if has_match "ct: leaks detected" "${run_log}"; then - echo " FAIL: unexpected leak detected" - return 1 - fi - fi - - if expect_autofree "${test_file}"; then - if ! has_match "auto-free ptr=" "${run_log}"; then - echo " FAIL: expected auto-free log, none found" - return 1 - fi - fi - - echo " OK" - return 0 -} - -for t in "${TESTS[@]}"; do - if run_one "${t}"; then - PASS=$((PASS + 1)) - else - FAIL=$((FAIL + 1)) - fi -done - -echo "" -echo "Summary: ${PASS} passed, ${FAIL} failed" -[[ "${FAIL}" -eq 0 ]] diff --git a/test/scripts/linux_compile.sh b/test/scripts/linux_compile.sh deleted file mode 100644 index 7176528..0000000 --- a/test/scripts/linux_compile.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -BUILD_DIR="${BUILD_DIR:-$ROOT/build}" -CC_BIN="${CC_BIN:-$BUILD_DIR/cc}" - -if [ ! -x "$CC_BIN" ]; then - echo "cc binary not found at $CC_BIN" - echo "Build it first (see README.md)." - exit 1 -fi - -TMP_DIR="$(mktemp -d)" -trap 'rm -rf "$TMP_DIR"' EXIT - -cat >"$TMP_DIR/foo.cpp" <<'EOF' -#include -int foo() { - std::string s = "foo"; - return static_cast(s.size()); -} -EOF - -cat >"$TMP_DIR/bar.cpp" <<'EOF' -#include -extern int foo(); -int bar() { - std::vector v{1, 2, 3}; - return static_cast(v.size()); -} -int main() { - return (foo() + bar() == 6) ? 0 : 1; -} -EOF - -cat >"$TMP_DIR/baz.c" <<'EOF' -#include -int baz() { - std::string s = "baz"; - return static_cast(s.size()); -} -EOF - -pushd "$TMP_DIR" >/dev/null - -"$CC_BIN" --instrument foo.cpp bar.cpp -o app_instrumented -./app_instrumented - -"$CC_BIN" --instrument -x c++ -c foo.cpp bar.cpp -"$CC_BIN" --instrument -x=c++ -c baz.c -o=baz_inst.o -"$CC_BIN" --instrument foo.o bar.o -o app_instrumented_obj -./app_instrumented_obj -"$CC_BIN" --instrument foo.o bar.o baz_inst.o -o=app_instrumented_obj_eq -./app_instrumented_obj_eq - -rm -f foo.o bar.o baz_inst.o - -"$CC_BIN" foo.cpp bar.cpp -o app_plain -./app_plain - -"$CC_BIN" -x c++ -c foo.cpp bar.cpp -"$CC_BIN" foo.o bar.o -o app_plain_obj -./app_plain_obj -"$CC_BIN" -x=c++ -c baz.c -o=baz_eq.o -"$CC_BIN" foo.o bar.o baz_eq.o -o=app_plain_obj_eq -./app_plain_obj_eq - -rm -f foo.o bar.o baz_eq.o - -popd >/dev/null diff --git a/test/scripts/macos_compile.sh b/test/scripts/macos_compile.sh deleted file mode 100644 index 7176528..0000000 --- a/test/scripts/macos_compile.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -BUILD_DIR="${BUILD_DIR:-$ROOT/build}" -CC_BIN="${CC_BIN:-$BUILD_DIR/cc}" - -if [ ! -x "$CC_BIN" ]; then - echo "cc binary not found at $CC_BIN" - echo "Build it first (see README.md)." - exit 1 -fi - -TMP_DIR="$(mktemp -d)" -trap 'rm -rf "$TMP_DIR"' EXIT - -cat >"$TMP_DIR/foo.cpp" <<'EOF' -#include -int foo() { - std::string s = "foo"; - return static_cast(s.size()); -} -EOF - -cat >"$TMP_DIR/bar.cpp" <<'EOF' -#include -extern int foo(); -int bar() { - std::vector v{1, 2, 3}; - return static_cast(v.size()); -} -int main() { - return (foo() + bar() == 6) ? 0 : 1; -} -EOF - -cat >"$TMP_DIR/baz.c" <<'EOF' -#include -int baz() { - std::string s = "baz"; - return static_cast(s.size()); -} -EOF - -pushd "$TMP_DIR" >/dev/null - -"$CC_BIN" --instrument foo.cpp bar.cpp -o app_instrumented -./app_instrumented - -"$CC_BIN" --instrument -x c++ -c foo.cpp bar.cpp -"$CC_BIN" --instrument -x=c++ -c baz.c -o=baz_inst.o -"$CC_BIN" --instrument foo.o bar.o -o app_instrumented_obj -./app_instrumented_obj -"$CC_BIN" --instrument foo.o bar.o baz_inst.o -o=app_instrumented_obj_eq -./app_instrumented_obj_eq - -rm -f foo.o bar.o baz_inst.o - -"$CC_BIN" foo.cpp bar.cpp -o app_plain -./app_plain - -"$CC_BIN" -x c++ -c foo.cpp bar.cpp -"$CC_BIN" foo.o bar.o -o app_plain_obj -./app_plain_obj -"$CC_BIN" -x=c++ -c baz.c -o=baz_eq.o -"$CC_BIN" foo.o bar.o baz_eq.o -o=app_plain_obj_eq -./app_plain_obj_eq - -rm -f foo.o bar.o baz_eq.o - -popd >/dev/null diff --git a/unittests/api_test.cpp b/unittests/api_test.cpp index fb5b9f5..f282aed 100644 --- a/unittests/api_test.cpp +++ b/unittests/api_test.cpp @@ -81,10 +81,10 @@ namespace // relative path: only its tail is checked. Expect(finding.location && finding.location->file.ends_with("findings/heap_overflow_write.c") && - finding.location->line == 7 && finding.location->column > 0, + finding.location->line == 8 && finding.location->column > 0, "overflow: located at the faulting write in the source"); Expect(finding.allocation && finding.allocation->file == finding.location->file && - finding.allocation->line == 6, + finding.allocation->line == 7, "overflow: the allocation site, in the same source"); }