diff --git a/README.md b/README.md index 78d6a24..d2bb7b1 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,7 @@ ctest --test-dir build --output-on-failure | `api_unit` | the library API: `Run` and `RunBatch` on real programs, a heap overflow, a compile-only run, a hang against the timeout, a batch, the refused and empty cases | | `consumer` | a project fetching this repository the way ctrace does (`unittests/consumer/`) configures, builds, links `coretrace::runtime-analyzer_lib` and analyzes a program | | `fixtures` | every source under `test/` states what it proves in its header (`// args:`, `// expect:`), and the analyzer proves it: findings with rule, CWE, level, faulting line and allocation site, or none; the program's exit status and output when stated. A fixture without an expectation fails (`test/check_fixtures.py`) | -| `cli` | the text output, the SARIF log of a batch, and exit `2` for what cannot be analyzed (`test/cli/check_cli.py`) | +| `cli` | every option as the table above states it, the text output, the SARIF log of a batch, and exit `2` with a message for invalid arguments and for what cannot be analyzed (`test/cli/check_cli.py`) | | `timeout` | hung, stdin-reading and forking programs, alone and in a batch | | `compile_failure` | a source that fails code generation fails its own analysis only | | `hello_runs` | `test/examples/fixtures/hello.c` runs and its entry/exit events are collected | diff --git a/test/cli/check_cli.py b/test/cli/check_cli.py index 6ca5e46..d8a9a37 100644 --- a/test/cli/check_cli.py +++ b/test/cli/check_cli.py @@ -1,17 +1,19 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""The CLI's output formats, verdicts and batch mode over the findings fixtures. +"""The CLI's options, output formats and verdicts, as the README states them. Usage: check_cli.py What each fixture proves is stated in the fixture and checked by check_fixtures.py; this script -checks how the CLI reports it: the text output, the SARIF log of a batch, and the exit status -of the runs that cannot be analyzed. +checks how the CLI drives a run and reports it: every option, the text output, the SARIF log of +a batch, and exit 2 with a message for what cannot be analyzed. The programs it needs are +written into the work directory: they must not live under test/, which the sweeps run in full. """ from __future__ import annotations import json +import os import subprocess import sys from pathlib import Path @@ -20,6 +22,20 @@ TEST_DIR = HERE.parent MEMORY_ARGS = ["--ct-modules=alloc,bounds"] +PROGRAMS = { + "args.c": "#include \nint main(int argc, char** argv)\n{\n printf(\"argc=%d\\n\", argc);\n" + " for (int i = 1; i < argc; ++i)\n puts(argv[i]);\n return 0;\n}\n", + "env.c": "#include \n#include \nint main(void)\n{\n" + " const char* value = getenv(\"CT_CHECK\");\n" + " printf(\"CT_CHECK=%s\\n\", value ? value : \"(unset)\");\n return 0;\n}\n", + "cwd.c": "#include \n#include \nint main(void)\n{\n char buffer[4096];\n" + " puts(getcwd(buffer, sizeof buffer));\n return 0;\n}\n", + "chatty.c": "#include \nint main(void)\n{\n puts(\"to-stdout\");\n" + " fputs(\"to-stderr\\n\", stderr);\n return 0;\n}\n", + "exit_one.c": "int main(void)\n{\n return 1;\n}\n", + "normal.c": "int main(void)\n{\n return 0;\n}\n", +} + failures = 0 @@ -49,9 +65,145 @@ def sarif_rules(completed: subprocess.CompletedProcess) -> list[str] | None: return sorted(result["ruleId"] for result in log["runs"][0]["results"]) +def program(analyzer: str, work: Path, name: str, *options: str) -> subprocess.CompletedProcess: + return run(analyzer, "-o", str(work / Path(name).stem), *options, "--", *MEMORY_ARGS, + str(work / name)) + + +def check_options(analyzer: str, work: Path) -> None: + completed = program(analyzer, work, "args.c", "--show-output", "--run-arg", "one", + "--run-arg", "two words") + expect(completed.returncode == 0 and "argc=3\none\ntwo words\n" in completed.stdout, + "--run-arg: each value is an argument of the program, in order") + + completed = program(analyzer, work, "env.c", "--show-output", "--env", "CT_CHECK=hello") + expect(completed.returncode == 0 and "CT_CHECK=hello\n" in completed.stdout, + "--env: the variable is set for the program") + completed = program(analyzer, work, "env.c", "--show-output") + expect(completed.returncode == 0 and "CT_CHECK=(unset)\n" in completed.stdout, + "--env: nothing leaks into the program without it") + + elsewhere = work / "elsewhere" + elsewhere.mkdir(exist_ok=True) + completed = program(analyzer, work, "cwd.c", "--show-output", "--cwd", str(elsewhere)) + expect(completed.returncode == 0 and f"{elsewhere.resolve()}\n" in completed.stdout, + "--cwd: the program runs in that directory") + # A relative -o is relative to the analyzer's own directory, not to --cwd. + relative = Path(os.path.relpath(work / "cwd_relative", TEST_DIR)) + completed = run(analyzer, "-o", str(relative), "--show-output", "--cwd", str(elsewhere), + "--", *MEMORY_ARGS, str(work / "cwd.c")) + expect(completed.returncode == 0 and f"{elsewhere.resolve()}\n" in completed.stdout, + "--cwd: a relative -o still names the binary to run") + + completed = program(analyzer, work, "chatty.c", "--show-output") + expect(completed.returncode == 0 + and "\nruntime-analyzer: stdout\nto-stdout\n" in completed.stdout + and "\nruntime-analyzer: stderr\nto-stderr\n" in completed.stdout, + "--show-output: the program's stdout and stderr follow the summary") + completed = program(analyzer, work, "chatty.c") + expect(completed.returncode == 0 and "to-stdout" not in completed.stdout + and "to-stderr" not in completed.stdout + completed.stderr, + "without --show-output: the program's output is not printed") + + completed = analyze(analyzer, work, "no_findings.c", "--show-events") + expect(completed.returncode == 0 and "\nruntime-analyzer: events\n" in completed.stdout + and "tracing-malloc" in completed.stdout, + "--show-events: the CoreTrace event lines follow the summary") + completed = analyze(analyzer, work, "no_findings.c") + expect("runtime-analyzer: events" not in completed.stdout + and "tracing-malloc" not in completed.stdout, + "without --show-events: the event lines are not printed") + + completed = program(analyzer, work, "normal.c", "--format", "text") + expect(completed.returncode == 0 and "runtime-analyzer: findings=0" in completed.stdout, + "--format text: the text summary, as a separate argument") + completed = program(analyzer, work, "normal.c", "--format", "sarif") + expect(completed.returncode == 0 and sarif_rules(completed) == [], + "--format sarif: the SARIF log, as a separate argument") + completed = run(analyzer, f"--output={work / 'normal_eq'}", "--", *MEMORY_ARGS, + str(work / "normal.c")) + expect(completed.returncode == 0 and (work / "normal_eq").exists(), + "--output=: the binary path, with an equals sign") + + completed = run(analyzer, "--help") + expect(completed.returncode == 0 and completed.stdout.startswith("Usage: runtime-analyzer"), + "--help: usage on stdout, exit 0") + + +def check_batch_options(analyzer: str, work: Path) -> None: + batch = work / "batch-options" + batch.mkdir(exist_ok=True) + for name in ("exit_one.c", "normal.c"): + (batch / name).write_text(PROGRAMS[name]) + artifacts = work / "batch-options-artifacts" + completed = run(analyzer, "--test-dir", str(batch), "--output-dir", str(artifacts), "--", + *MEMORY_ARGS, timeout=300) + expect(completed.returncode == 0, "batch: a program exiting non-zero is not a finding") + expect(f"runtime-analyzer: [RUNTIME] {batch / 'exit_one.c'} exit_code=1" in completed.stdout + and f"runtime-analyzer: [PASS] {batch / 'normal.c'} exit_code=0" in completed.stdout, + "batch: the status of each program is on its line") + expect(" runtime_failures=1\n" in completed.stdout, "batch: the summary counts it") + expect(f"binary={artifacts}/" in completed.stdout, + "--output-dir: the binaries are placed under it") + completed = run(analyzer, "--test-dir", str(batch), "--output-dir", str(artifacts), + "--strict-test-exit", "--", *MEMORY_ARGS, timeout=300) + expect(completed.returncode == 1, "--strict-test-exit: a non-zero program exit gives exit 1") + + second = work / "batch-options-second" + second.mkdir(exist_ok=True) + (second / "normal.c").write_text(PROGRAMS["normal.c"]) + completed = run(analyzer, "--test-dir", str(batch), "--test-dir", str(second), + "--output-dir", str(artifacts), "--", *MEMORY_ARGS, timeout=300) + expect(completed.returncode == 0 and " tests=3\n" in completed.stdout, + "--test-dir: repeatable, every directory is run") + + completed = run(analyzer, "--test-dir", str(work / "absent"), "--output-dir", str(artifacts), + "--", *MEMORY_ARGS) + expect(completed.returncode == 2 and "test directory does not exist" in completed.stderr, + "--test-dir: exit 2 and a message for a directory that does not exist") + + +def check_invalid_arguments(analyzer: str, work: Path) -> None: + source = str(work / "normal.c") + cases = { + "an unknown option": (["--bogus", "--", source], "unknown analyzer option: --bogus"), + "--run-arg without a value": (["--run-arg"], "--run-arg requires a value"), + "--env without a value": (["--env"], "--env requires NAME=VALUE"), + "--env without NAME=": (["--env", "NOVALUE", "--", source], "--env requires NAME=VALUE"), + "--env with an empty name": (["--env", "=x", "--", source], "--env requires NAME=VALUE"), + "--cwd without a value": (["--cwd"], "--cwd requires a path"), + "--timeout without a value": (["--timeout"], "--timeout requires"), + "--timeout negative": (["--timeout", "-1", "--", source], "non-negative number"), + "--test-dir without a value": (["--test-dir"], "--test-dir requires a path"), + "--output-dir without a value": (["--output-dir"], "--output-dir requires a path"), + "--format without a value": (["--format"], "--format requires text or sarif"), + "no compiler arguments": ([], "missing compiler arguments"), + "-- without compiler arguments": (["--"], "missing compiler arguments"), + } + for what, (args, message) in cases.items(): + completed = run(analyzer, *args) + expect(completed.returncode == 2 and message in completed.stderr + and "Usage: runtime-analyzer" in completed.stderr, + f"{what}: exit 2, the message and the usage on stderr") + + # "-o" as the last argument after -- is a compiler argument: the compiler rejects it. + completed = run(analyzer, "-o") + expect(completed.returncode == 2 and "-o requires a value" in completed.stderr, + "-o as the last argument: exit 2 and the message") + completed = run(analyzer, "-o", str(work / "twice"), "--", *MEMORY_ARGS, "-o", + str(work / "twice2"), source) + expect(completed.returncode == 2 and "output path specified twice" in completed.stderr, + "-o with a compiler -o: exit 2 and the message") + + def main() -> int: analyzer, work = str(Path(sys.argv[1]).resolve()), Path(sys.argv[2]).resolve() work.mkdir(parents=True, exist_ok=True) + for name, text in PROGRAMS.items(): + (work / name).write_text(text) + check_options(analyzer, work) + check_batch_options(analyzer, work) + check_invalid_arguments(analyzer, work) # The program's exit status is reported, but only runtime errors decide the verdict. completed = analyze(analyzer, work, "no_findings.c")