From eb91fe3730f0667c75f988c6bca568eabe96d1f4 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 27 Sep 2026 15:36:53 +0200 Subject: [PATCH 1/3] refactor: move the CLI's pure logic into cli_args and events runtime_analyzer.cpp held 1050 lines, most of them in an anonymous namespace that no test could reach: argument parsing, the compiler's -o, the compile-only and -x flags, the C-as-C++ and DEBUG heuristics, the compiler arguments of a run, the artifact names of a batch, the CoreTrace event lines and their summary, and the verdict. They move verbatim into src/cli_args.{hpp,cpp} and src/events.{hpp,cpp}, declared in the library's namespace; runtime_analyzer.cpp keeps Run, RunBatch, Main and the printers. No behaviour changes. Part of #21. --- CMakeLists.txt | 2 + src/cli_args.cpp | 415 +++++++++++++++++++++++++++++ src/cli_args.hpp | 67 +++++ src/events.cpp | 132 ++++++++++ src/events.hpp | 32 +++ src/runtime_analyzer.cpp | 549 +-------------------------------------- 6 files changed, 650 insertions(+), 547 deletions(-) create mode 100644 src/cli_args.cpp create mode 100644 src/cli_args.hpp create mode 100644 src/events.cpp create mode 100644 src/events.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f35f1b7..45298e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,8 @@ set(TOOL_NAMESPACE "coretrace" CACHE STRING "CMake namespace for ALIAS targets") # Library sources (add/remove files here) set(TOOL_SOURCES + src/cli_args.cpp + src/events.cpp src/findings.cpp src/process.cpp src/runtime_analyzer.cpp diff --git a/src/cli_args.cpp b/src/cli_args.cpp new file mode 100644 index 0000000..05d4940 --- /dev/null +++ b/src/cli_args.cpp @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "cli_args.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace coretrace::runtime_analyzer +{ + namespace + { + constexpr std::string_view kDefaultOutputPath = "runtime-analyzer.out"; + + [[nodiscard]] bool IsCompileOnlyFlag(std::string_view arg) + { + return arg == "-c" || arg == "-S" || arg == "-E" || arg == "-emit-llvm" || + arg == "--precompile" || arg == "-fsyntax-only"; + } + + [[nodiscard]] bool Contains(std::string_view haystack, std::string_view needle) + { + return haystack.find(needle) != std::string_view::npos; + } + + [[nodiscard]] std::string ReadSmallTextFile(const std::filesystem::path& path) + { + std::ifstream input(path); + if (!input) + { + return {}; + } + + std::ostringstream output; + output << input.rdbuf(); + return output.str(); + } + + [[nodiscard]] bool IsValidEnvironmentAssignment(std::string_view assignment) + { + const std::size_t eq = assignment.find('='); + return eq != std::string_view::npos && eq != 0; + } + + [[nodiscard]] std::uint64_t StablePathHash(std::string_view text) + { + constexpr std::uint64_t kFnvOffsetBasis = 14695981039346656037ULL; + constexpr std::uint64_t kFnvPrime = 1099511628211ULL; + + std::uint64_t hash = kFnvOffsetBasis; + for (const char ch : text) + { + hash ^= static_cast(ch); + hash *= kFnvPrime; + } + return hash; + } + + [[nodiscard]] std::string HexDigest(std::uint64_t value) + { + std::ostringstream output; + output << std::hex << std::nouppercase << std::setfill('0') << std::setw(16) << value; + return output.str(); + } + } // namespace + + [[nodiscard]] bool HasCompileOnlyAction(const std::vector& args) + { + return std::ranges::any_of(args, IsCompileOnlyFlag); + } + + [[nodiscard]] bool HasLanguageOverride(const std::vector& args) + { + for (std::size_t i = 0; i < args.size(); ++i) + { + if (args[i] == "-x" && i + 1 < args.size()) + { + return true; + } + if (args[i].rfind("-x=", 0) == 0 || args[i].rfind("-x", 0) == 0) + { + return true; + } + } + return false; + } + + [[nodiscard]] bool CSourceLooksLikeCxx(const std::filesystem::path& source, + std::string_view text) + { + if (source.extension() != ".c") + { + return false; + } + return Contains(text, "#include ") || Contains(text, "#include ") || + Contains(text, "std::") || Contains(text, "static_cast<") || + Contains(text, "reinterpret_cast<") || Contains(text, "class ") || + Contains(text, "namespace ") || Contains(text, "template <") || + Contains(text, "virtual "); + } + + [[nodiscard]] bool SourceRequiresDebugDefine(std::string_view text) + { + return Contains(text, "#ifndef DEBUG") || Contains(text, "#if !defined(DEBUG)"); + } + + void AddPerSourceCompatibilityArgs(const std::filesystem::path& source, + std::vector& args) + { + const std::string text = ReadSmallTextFile(source); + if (!HasLanguageOverride(args) && CSourceLooksLikeCxx(source, text)) + { + args.emplace_back("-x"); + args.emplace_back("c++"); + } + if (SourceRequiresDebugDefine(text) && + std::ranges::none_of(args, [](const std::string& arg) + { return arg == "-DDEBUG" || arg == "-DDEBUG=1"; })) + { + args.emplace_back("-DDEBUG"); + } + } + + [[nodiscard]] OutputArgument FindOutputArgument(const std::vector& args) + { + OutputArgument result; + for (std::size_t i = 0; i < args.size(); ++i) + { + const std::string& arg = args[i]; + if (arg == "-o" || arg == "--output") + { + result.present = true; + if (i + 1 < args.size()) + { + result.path = args[i + 1]; + } + return result; + } + if (arg.rfind("-o=", 0) == 0) + { + result.present = true; + result.path = arg.substr(3); + return result; + } + if (arg.rfind("--output=", 0) == 0) + { + result.present = true; + result.path = arg.substr(9); + return result; + } + } + return result; + } + + [[nodiscard]] std::vector BuildCompilerArguments(const AnalyzerOptions& options, + std::string& output_path) + { + std::vector args; + args.reserve(options.compiler_args.size() + 2); + for (const std::string& arg : options.compiler_args) + { + if (arg == "--instrument") + { + continue; + } + args.push_back(arg); + } + + const OutputArgument output = FindOutputArgument(args); + if (output.present) + { + output_path = output.path; + return args; + } + + output_path = + options.output_path.empty() ? std::string(kDefaultOutputPath) : options.output_path; + args.emplace_back("-o"); + args.push_back(output_path); + return args; + } + + [[nodiscard]] std::string SanitizeArtifactName(const std::filesystem::path& path) + { + std::string name = path.generic_string(); + for (char& ch : name) + { + const bool keep = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') || ch == '-' || ch == '_'; + if (!keep) + { + ch = '_'; + } + } + if (name.empty()) + { + return "test"; + } + return name; + } + + [[nodiscard]] std::filesystem::path MakeTestOutputPath(const AnalyzerOptions& options, + const std::filesystem::path& source) + { + std::error_code error; + std::filesystem::path relative = std::filesystem::relative(source, error); + if (error) + { + relative = source.filename(); + } + + const std::string hash_input = relative.generic_string(); + std::filesystem::path artifact_name = relative; + artifact_name.replace_extension(); + return std::filesystem::path(options.output_directory) / + (SanitizeArtifactName(artifact_name) + "_" + HexDigest(StablePathHash(hash_input))); + } + + [[nodiscard]] ParseResult ParseArgs(int argc, char** argv) + { + ParseResult result; + bool compiler_args = false; + + for (int i = 1; i < argc; ++i) + { + std::string arg = argv[i]; + if (compiler_args) + { + result.options.compiler_args.push_back(std::move(arg)); + continue; + } + + if (arg == "--") + { + compiler_args = true; + continue; + } + if (arg == "-h" || arg == "--help") + { + result.help = true; + return result; + } + if (arg == "--version") + { + result.version = true; + return result; + } + if (arg == "-o" || arg == "--output") + { + if (i + 1 >= argc) + { + result.ok = false; + result.error = arg + " requires a value"; + return result; + } + result.options.output_path = argv[++i]; + result.options.explicit_output_path = true; + continue; + } + if (arg.rfind("--output=", 0) == 0) + { + result.options.output_path = arg.substr(9); + result.options.explicit_output_path = true; + continue; + } + if (arg == "--run-arg") + { + if (i + 1 >= argc) + { + result.ok = false; + result.error = "--run-arg requires a value"; + return result; + } + result.options.program_args.emplace_back(argv[++i]); + continue; + } + if (arg == "--env") + { + if (i + 1 >= argc) + { + result.ok = false; + result.error = "--env requires NAME=VALUE"; + return result; + } + std::string assignment = argv[++i]; + if (!IsValidEnvironmentAssignment(assignment)) + { + result.ok = false; + result.error = "--env requires NAME=VALUE"; + return result; + } + result.options.environment.push_back(std::move(assignment)); + continue; + } + if (arg == "--cwd") + { + if (i + 1 >= argc) + { + result.ok = false; + result.error = "--cwd requires a path"; + return result; + } + result.options.working_directory = argv[++i]; + continue; + } + if (arg == "--timeout") + { + if (i + 1 >= argc) + { + result.ok = false; + result.error = "--timeout requires a number of seconds"; + return result; + } + const std::string value = argv[++i]; + std::size_t consumed = 0; + long long seconds = -1; + try + { + seconds = std::stoll(value, &consumed); + } + catch (const std::exception&) + { + } + if (consumed != value.size() || seconds < 0) + { + result.ok = false; + result.error = "--timeout requires a non-negative number of seconds"; + return result; + } + result.options.timeout = std::chrono::seconds(seconds); + continue; + } + if (arg == "--test-dir") + { + if (i + 1 >= argc) + { + result.ok = false; + result.error = "--test-dir requires a path"; + return result; + } + result.options.test_directories.emplace_back(argv[++i]); + continue; + } + if (arg == "--output-dir") + { + if (i + 1 >= argc) + { + result.ok = false; + result.error = "--output-dir requires a path"; + return result; + } + result.options.output_directory = argv[++i]; + continue; + } + if (arg == "--strict-test-exit") + { + result.options.strict_test_exit = true; + continue; + } + if (arg == "--no-run") + { + result.options.run_program = false; + continue; + } + if (arg == "--show-output") + { + result.options.show_program_output = true; + continue; + } + if (arg == "--show-events") + { + result.options.show_events = true; + continue; + } + if (arg == "--format" || arg.rfind("--format=", 0) == 0) + { + std::string value; + if (arg != "--format") + { + value = arg.substr(9); + } + else if (i + 1 < argc) + { + value = argv[++i]; + } + if (value == "text" || value == "sarif") + { + result.format = value == "sarif" ? OutputFormat::Sarif : OutputFormat::Text; + continue; + } + result.ok = false; + result.error = "--format requires text or sarif"; + return result; + } + + result.ok = false; + result.error = "unknown analyzer option: " + arg + " (compiler arguments go after --)"; + return result; + } + + if ((!compiler_args || result.options.compiler_args.empty()) && + result.options.test_directories.empty()) + { + result.ok = false; + result.error = "missing compiler arguments; use -- "; + } + + return result; + } +} // namespace coretrace::runtime_analyzer diff --git a/src/cli_args.hpp b/src/cli_args.hpp new file mode 100644 index 0000000..b43a600 --- /dev/null +++ b/src/cli_args.hpp @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: Apache-2.0 +// The CLI's arguments: their parsing, the compiler arguments derived from them and from each +// source, and where batch mode puts each binary. +#ifndef CORETRACE_RUNTIME_ANALYZER_CLI_ARGS_HPP +#define CORETRACE_RUNTIME_ANALYZER_CLI_ARGS_HPP + +#include "runtime_analyzer.hpp" + +#include +#include +#include +#include + +namespace coretrace::runtime_analyzer +{ + enum class OutputFormat + { + Text, + Sarif, + }; + + struct ParseResult + { + bool ok = true; + bool help = false; + bool version = false; + AnalyzerOptions options; + OutputFormat format = OutputFormat::Text; + std::string error; // why `ok` is false + }; + + [[nodiscard]] ParseResult ParseArgs(int argc, char** argv); + + struct OutputArgument + { + bool present = false; + std::string path; + }; + + // The compiler's own -o, --output, -o= or --output=, if any. + [[nodiscard]] OutputArgument FindOutputArgument(const std::vector& args); + + // -c, -S, -E and the like: no executable comes out of the compiler. + [[nodiscard]] bool HasCompileOnlyAction(const std::vector& args); + [[nodiscard]] bool HasLanguageOverride(const std::vector& args); + + // A .c source written in C++, which clang has to be told about. + [[nodiscard]] bool CSourceLooksLikeCxx(const std::filesystem::path& source, + std::string_view text); + [[nodiscard]] bool SourceRequiresDebugDefine(std::string_view text); + // Appends -x c++ and -DDEBUG when the source needs them and the user gave neither. + void AddPerSourceCompatibilityArgs(const std::filesystem::path& source, + std::vector& args); + + // The compiler arguments of a run: --instrument dropped, -o appended unless the compiler + // has one. `output_path` receives the binary's path either way. + [[nodiscard]] std::vector BuildCompilerArguments(const AnalyzerOptions& options, + std::string& output_path); + + // Letters, digits, - and _ only; "test" for an empty path. + [[nodiscard]] std::string SanitizeArtifactName(const std::filesystem::path& path); + // /_. + [[nodiscard]] std::filesystem::path MakeTestOutputPath(const AnalyzerOptions& options, + const std::filesystem::path& source); +} // namespace coretrace::runtime_analyzer + +#endif // CORETRACE_RUNTIME_ANALYZER_CLI_ARGS_HPP diff --git a/src/events.cpp b/src/events.cpp new file mode 100644 index 0000000..29055eb --- /dev/null +++ b/src/events.cpp @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "events.hpp" + +#include +#include + +namespace coretrace::runtime_analyzer +{ + namespace + { + [[nodiscard]] bool Contains(std::string_view haystack, std::string_view needle) + { + return haystack.find(needle) != std::string_view::npos; + } + + [[nodiscard]] bool IsCoreTraceLine(std::string_view line) + { + return Contains(line, "==ct==") || Contains(line, "ct:") || + Contains(line, "[ENTRY-FUNCTION]") || Contains(line, "[EXIT-FUNCTION]") || + Contains(line, "[VTABLE-DIAG]") || Contains(line, "tracing-") || + Contains(line, "auto-free") || Contains(line, "heap-buffer-overflow") || + Contains(line, "heap-use-after-free"); + } + + void UpdateSummary(std::string_view line, CollectionSummary& summary) + { + if (Contains(line, "[ENTRY-FUNCTION]") || Contains(line, "ct: enter ")) + { + ++summary.entry_events; + } + if (Contains(line, "[EXIT-FUNCTION]")) + { + ++summary.exit_events; + } + if (Contains(line, "tracing-") || Contains(line, "auto-free") || + Contains(line, "leaks detected") || Contains(line, "ct: leak ")) + { + ++summary.allocation_events; + } + if (Contains(line, "heap-buffer-overflow") || Contains(line, "heap-use-after-free")) + { + ++summary.bounds_errors; + } + if (Contains(line, "leaks detected") || Contains(line, "ct: leak ")) + { + ++summary.leak_reports; + } + if (Contains(line, "[VTABLE-DIAG]")) + { + ++summary.vtable_events; + } + if (Contains(line, "[WARN]") || Contains(line, " WARN ")) + { + ++summary.warnings; + } + if (Contains(line, "[ERROR]") || Contains(line, " ERROR ")) + { + ++summary.errors; + } + } + } // namespace + + [[nodiscard]] std::string StripAnsi(std::string_view input) + { + std::string output; + output.reserve(input.size()); + for (std::size_t i = 0; i < input.size(); ++i) + { + if (input[i] == '\x1b' && i + 1 < input.size() && input[i + 1] == '[') + { + i += 2; + while (i < input.size() && !((input[i] >= 'A' && input[i] <= 'Z') || + (input[i] >= 'a' && input[i] <= 'z'))) + { + ++i; + } + continue; + } + output.push_back(input[i]); + } + return output; + } + + [[nodiscard]] CollectionSummary CollectEvents(std::string_view text, + std::vector& events) + { + CollectionSummary summary; + std::istringstream stream(std::string(StripAnsi(text))); + std::string line; + while (std::getline(stream, line)) + { + if (!IsCoreTraceLine(line)) + { + continue; + } + + ++summary.coretrace_lines; + UpdateSummary(line, summary); + events.push_back(std::move(line)); + } + return summary; + } + + [[nodiscard]] CollectionSummary MergeSummaries(CollectionSummary lhs, + const CollectionSummary& rhs) + { + lhs.coretrace_lines += rhs.coretrace_lines; + lhs.entry_events += rhs.entry_events; + lhs.exit_events += rhs.exit_events; + lhs.allocation_events += rhs.allocation_events; + lhs.bounds_errors += rhs.bounds_errors; + lhs.leak_reports += rhs.leak_reports; + lhs.vtable_events += rhs.vtable_events; + lhs.warnings += rhs.warnings; + lhs.errors += rhs.errors; + return lhs; + } + + [[nodiscard]] int Verdict(const AnalyzerResult& result, bool run_program) + { + if (!result.compile_success || (run_program && !result.executed)) + { + return kExitNotAnalyzed; + } + if (!result.findings.empty()) + { + return kExitFindings; + } + // A program killed at the timeout was not analyzed to completion. + return result.timed_out ? kExitNotAnalyzed : kExitNoFindings; + } +} // namespace coretrace::runtime_analyzer diff --git a/src/events.hpp b/src/events.hpp new file mode 100644 index 0000000..e197bd8 --- /dev/null +++ b/src/events.hpp @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 +// What the analyzer reads from a run: the CoreTrace event lines, their summary, and the +// verdict it draws from a result. +#ifndef CORETRACE_RUNTIME_ANALYZER_EVENTS_HPP +#define CORETRACE_RUNTIME_ANALYZER_EVENTS_HPP + +#include "runtime_analyzer.hpp" + +#include +#include +#include + +namespace coretrace::runtime_analyzer +{ + // The analyzer's verdict; the program's own status stays in AnalyzerResult::exit_code. + constexpr int kExitNoFindings = 0; + constexpr int kExitFindings = 1; + constexpr int kExitNotAnalyzed = 2; + + [[nodiscard]] std::string StripAnsi(std::string_view input); + + // Appends the CoreTrace lines of `text`, without their colours, to `events`, and counts + // them. + [[nodiscard]] CollectionSummary CollectEvents(std::string_view text, + std::vector& events); + [[nodiscard]] CollectionSummary MergeSummaries(CollectionSummary lhs, + const CollectionSummary& rhs); + + [[nodiscard]] int Verdict(const AnalyzerResult& result, bool run_program); +} // namespace coretrace::runtime_analyzer + +#endif // CORETRACE_RUNTIME_ANALYZER_EVENTS_HPP diff --git a/src/runtime_analyzer.cpp b/src/runtime_analyzer.cpp index 4d5a558..c81d0ac 100644 --- a/src/runtime_analyzer.cpp +++ b/src/runtime_analyzer.cpp @@ -2,7 +2,9 @@ #include "runtime_analyzer.hpp" #include "runtime_analyzer_version.hpp" +#include "cli_args.hpp" #include "compilerlib/compiler.h" +#include "events.hpp" #include "findings.hpp" #include "process.hpp" #include "sarif.hpp" @@ -12,13 +14,8 @@ #include #include -#include #include -#include -#include #include -#include -#include #include #include #include @@ -27,187 +24,12 @@ namespace coretrace::runtime_analyzer { namespace { - constexpr std::string_view kDefaultOutputPath = "runtime-analyzer.out"; - - // The analyzer's verdict; the program's own status stays in AnalyzerResult::exit_code. - constexpr int kExitNoFindings = 0; - constexpr int kExitFindings = 1; - constexpr int kExitNotAnalyzed = 2; - - enum class OutputFormat - { - Text, - Sarif, - }; - - struct ParseResult - { - bool ok = true; - bool help = false; - bool version = false; - AnalyzerOptions options; - OutputFormat format = OutputFormat::Text; - std::string error; - }; - - struct OutputArgument - { - bool present = false; - std::string path; - }; - - [[nodiscard]] bool IsCompileOnlyFlag(std::string_view arg) - { - return arg == "-c" || arg == "-S" || arg == "-E" || arg == "-emit-llvm" || - arg == "--precompile" || arg == "-fsyntax-only"; - } - [[nodiscard]] bool IsSourceFile(const std::filesystem::path& path) { const std::string ext = path.extension().string(); return ext == ".c" || ext == ".cc" || ext == ".cpp" || ext == ".cxx"; } - [[nodiscard]] bool Contains(std::string_view haystack, std::string_view needle) - { - return haystack.find(needle) != std::string_view::npos; - } - - [[nodiscard]] bool HasCompileOnlyAction(const std::vector& args) - { - return std::ranges::any_of(args, IsCompileOnlyFlag); - } - - [[nodiscard]] bool HasLanguageOverride(const std::vector& args) - { - for (std::size_t i = 0; i < args.size(); ++i) - { - if (args[i] == "-x" && i + 1 < args.size()) - { - return true; - } - if (args[i].rfind("-x=", 0) == 0 || args[i].rfind("-x", 0) == 0) - { - return true; - } - } - return false; - } - - [[nodiscard]] std::string ReadSmallTextFile(const std::filesystem::path& path) - { - std::ifstream input(path); - if (!input) - { - return {}; - } - - std::ostringstream output; - output << input.rdbuf(); - return output.str(); - } - - [[nodiscard]] bool CSourceLooksLikeCxx(const std::filesystem::path& source, - std::string_view text) - { - if (source.extension() != ".c") - { - return false; - } - return Contains(text, "#include ") || Contains(text, "#include ") || - Contains(text, "std::") || Contains(text, "static_cast<") || - Contains(text, "reinterpret_cast<") || Contains(text, "class ") || - Contains(text, "namespace ") || Contains(text, "template <") || - Contains(text, "virtual "); - } - - [[nodiscard]] bool SourceRequiresDebugDefine(std::string_view text) - { - return Contains(text, "#ifndef DEBUG") || Contains(text, "#if !defined(DEBUG)"); - } - - void AddPerSourceCompatibilityArgs(const std::filesystem::path& source, - std::vector& args) - { - const std::string text = ReadSmallTextFile(source); - if (!HasLanguageOverride(args) && CSourceLooksLikeCxx(source, text)) - { - args.emplace_back("-x"); - args.emplace_back("c++"); - } - if (SourceRequiresDebugDefine(text) && - std::ranges::none_of(args, [](const std::string& arg) - { return arg == "-DDEBUG" || arg == "-DDEBUG=1"; })) - { - args.emplace_back("-DDEBUG"); - } - } - - [[nodiscard]] OutputArgument FindOutputArgument(const std::vector& args) - { - OutputArgument result; - for (std::size_t i = 0; i < args.size(); ++i) - { - const std::string& arg = args[i]; - if (arg == "-o" || arg == "--output") - { - result.present = true; - if (i + 1 < args.size()) - { - result.path = args[i + 1]; - } - return result; - } - if (arg.rfind("-o=", 0) == 0) - { - result.present = true; - result.path = arg.substr(3); - return result; - } - if (arg.rfind("--output=", 0) == 0) - { - result.present = true; - result.path = arg.substr(9); - return result; - } - } - return result; - } - - [[nodiscard]] std::vector - BuildCompilerArguments(const AnalyzerOptions& options, std::string& output_path) - { - std::vector args; - args.reserve(options.compiler_args.size() + 2); - for (const std::string& arg : options.compiler_args) - { - if (arg == "--instrument") - { - continue; - } - args.push_back(arg); - } - - const OutputArgument output = FindOutputArgument(args); - if (output.present) - { - output_path = output.path; - return args; - } - - output_path = - options.output_path.empty() ? std::string(kDefaultOutputPath) : options.output_path; - args.emplace_back("-o"); - args.push_back(output_path); - return args; - } - - [[nodiscard]] bool IsValidEnvironmentAssignment(std::string_view assignment) - { - const std::size_t eq = assignment.find('='); - return eq != std::string_view::npos && eq != 0; - } - [[nodiscard]] std::string AbsolutePathForExecution(std::string_view output_path) { std::error_code error; @@ -220,162 +42,6 @@ namespace coretrace::runtime_analyzer return absolute.string(); } - [[nodiscard]] std::string StripAnsi(std::string_view input) - { - std::string output; - output.reserve(input.size()); - for (std::size_t i = 0; i < input.size(); ++i) - { - if (input[i] == '\x1b' && i + 1 < input.size() && input[i + 1] == '[') - { - i += 2; - while (i < input.size() && !((input[i] >= 'A' && input[i] <= 'Z') || - (input[i] >= 'a' && input[i] <= 'z'))) - { - ++i; - } - continue; - } - output.push_back(input[i]); - } - return output; - } - - [[nodiscard]] bool IsCoreTraceLine(std::string_view line) - { - return Contains(line, "==ct==") || Contains(line, "ct:") || - Contains(line, "[ENTRY-FUNCTION]") || Contains(line, "[EXIT-FUNCTION]") || - Contains(line, "[VTABLE-DIAG]") || Contains(line, "tracing-") || - Contains(line, "auto-free") || Contains(line, "heap-buffer-overflow") || - Contains(line, "heap-use-after-free"); - } - - void UpdateSummary(std::string_view line, CollectionSummary& summary) - { - if (Contains(line, "[ENTRY-FUNCTION]") || Contains(line, "ct: enter ")) - { - ++summary.entry_events; - } - if (Contains(line, "[EXIT-FUNCTION]")) - { - ++summary.exit_events; - } - if (Contains(line, "tracing-") || Contains(line, "auto-free") || - Contains(line, "leaks detected") || Contains(line, "ct: leak ")) - { - ++summary.allocation_events; - } - if (Contains(line, "heap-buffer-overflow") || Contains(line, "heap-use-after-free")) - { - ++summary.bounds_errors; - } - if (Contains(line, "leaks detected") || Contains(line, "ct: leak ")) - { - ++summary.leak_reports; - } - if (Contains(line, "[VTABLE-DIAG]")) - { - ++summary.vtable_events; - } - if (Contains(line, "[WARN]") || Contains(line, " WARN ")) - { - ++summary.warnings; - } - if (Contains(line, "[ERROR]") || Contains(line, " ERROR ")) - { - ++summary.errors; - } - } - - [[nodiscard]] CollectionSummary CollectEvents(std::string_view text, - std::vector& events) - { - CollectionSummary summary; - std::istringstream stream(std::string(StripAnsi(text))); - std::string line; - while (std::getline(stream, line)) - { - if (!IsCoreTraceLine(line)) - { - continue; - } - - ++summary.coretrace_lines; - UpdateSummary(line, summary); - events.push_back(std::move(line)); - } - return summary; - } - - [[nodiscard]] CollectionSummary MergeSummaries(CollectionSummary lhs, - const CollectionSummary& rhs) - { - lhs.coretrace_lines += rhs.coretrace_lines; - lhs.entry_events += rhs.entry_events; - lhs.exit_events += rhs.exit_events; - lhs.allocation_events += rhs.allocation_events; - lhs.bounds_errors += rhs.bounds_errors; - lhs.leak_reports += rhs.leak_reports; - lhs.vtable_events += rhs.vtable_events; - lhs.warnings += rhs.warnings; - lhs.errors += rhs.errors; - return lhs; - } - - [[nodiscard]] int Verdict(const AnalyzerResult& result, bool run_program) - { - if (!result.compile_success || (run_program && !result.executed)) - { - return kExitNotAnalyzed; - } - if (!result.findings.empty()) - { - return kExitFindings; - } - // A program killed at the timeout was not analyzed to completion. - return result.timed_out ? kExitNotAnalyzed : kExitNoFindings; - } - - [[nodiscard]] std::string SanitizeArtifactName(const std::filesystem::path& path) - { - std::string name = path.generic_string(); - for (char& ch : name) - { - const bool keep = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || - (ch >= '0' && ch <= '9') || ch == '-' || ch == '_'; - if (!keep) - { - ch = '_'; - } - } - if (name.empty()) - { - return "test"; - } - return name; - } - - [[nodiscard]] std::uint64_t StablePathHash(std::string_view text) - { - constexpr std::uint64_t kFnvOffsetBasis = 14695981039346656037ULL; - constexpr std::uint64_t kFnvPrime = 1099511628211ULL; - - std::uint64_t hash = kFnvOffsetBasis; - for (const char ch : text) - { - hash ^= static_cast(ch); - hash *= kFnvPrime; - } - return hash; - } - - [[nodiscard]] std::string HexDigest(std::uint64_t value) - { - std::ostringstream output; - output << std::hex << std::nouppercase << std::setfill('0') << std::setw(16) << value; - return output.str(); - } - [[nodiscard]] std::vector DiscoverSourceFiles(const std::vector& directories, std::string& diagnostics) { @@ -419,24 +85,6 @@ namespace coretrace::runtime_analyzer return sources; } - [[nodiscard]] std::filesystem::path MakeTestOutputPath(const AnalyzerOptions& options, - const std::filesystem::path& source) - { - std::error_code error; - std::filesystem::path relative = std::filesystem::relative(source, error); - if (error) - { - relative = source.filename(); - } - - const std::string hash_input = relative.generic_string(); - std::filesystem::path artifact_name = relative; - artifact_name.replace_extension(); - return std::filesystem::path(options.output_directory) / - (SanitizeArtifactName(artifact_name) + "_" + - HexDigest(StablePathHash(hash_input))); - } - void PrintHelp(std::ostream& out) { out << "Usage: runtime-analyzer [options] -- \n\n" @@ -469,199 +117,6 @@ namespace coretrace::runtime_analyzer << " runtime-analyzer -o ./app -- --ct-modules=trace,alloc,bounds main.c\n"; } - [[nodiscard]] ParseResult ParseArgs(int argc, char** argv) - { - ParseResult result; - bool compiler_args = false; - - for (int i = 1; i < argc; ++i) - { - std::string arg = argv[i]; - if (compiler_args) - { - result.options.compiler_args.push_back(std::move(arg)); - continue; - } - - if (arg == "--") - { - compiler_args = true; - continue; - } - if (arg == "-h" || arg == "--help") - { - result.help = true; - return result; - } - if (arg == "--version") - { - result.version = true; - return result; - } - if (arg == "-o" || arg == "--output") - { - if (i + 1 >= argc) - { - result.ok = false; - result.error = arg + " requires a value"; - return result; - } - result.options.output_path = argv[++i]; - result.options.explicit_output_path = true; - continue; - } - if (arg.rfind("--output=", 0) == 0) - { - result.options.output_path = arg.substr(9); - result.options.explicit_output_path = true; - continue; - } - if (arg == "--run-arg") - { - if (i + 1 >= argc) - { - result.ok = false; - result.error = "--run-arg requires a value"; - return result; - } - result.options.program_args.emplace_back(argv[++i]); - continue; - } - if (arg == "--env") - { - if (i + 1 >= argc) - { - result.ok = false; - result.error = "--env requires NAME=VALUE"; - return result; - } - std::string assignment = argv[++i]; - if (!IsValidEnvironmentAssignment(assignment)) - { - result.ok = false; - result.error = "--env requires NAME=VALUE"; - return result; - } - result.options.environment.push_back(std::move(assignment)); - continue; - } - if (arg == "--cwd") - { - if (i + 1 >= argc) - { - result.ok = false; - result.error = "--cwd requires a path"; - return result; - } - result.options.working_directory = argv[++i]; - continue; - } - if (arg == "--timeout") - { - if (i + 1 >= argc) - { - result.ok = false; - result.error = "--timeout requires a number of seconds"; - return result; - } - const std::string value = argv[++i]; - std::size_t consumed = 0; - long long seconds = -1; - try - { - seconds = std::stoll(value, &consumed); - } - catch (const std::exception&) - { - } - if (consumed != value.size() || seconds < 0) - { - result.ok = false; - result.error = "--timeout requires a non-negative number of seconds"; - return result; - } - result.options.timeout = std::chrono::seconds(seconds); - continue; - } - if (arg == "--test-dir") - { - if (i + 1 >= argc) - { - result.ok = false; - result.error = "--test-dir requires a path"; - return result; - } - result.options.test_directories.emplace_back(argv[++i]); - continue; - } - if (arg == "--output-dir") - { - if (i + 1 >= argc) - { - result.ok = false; - result.error = "--output-dir requires a path"; - return result; - } - result.options.output_directory = argv[++i]; - continue; - } - if (arg == "--strict-test-exit") - { - result.options.strict_test_exit = true; - continue; - } - if (arg == "--no-run") - { - result.options.run_program = false; - continue; - } - if (arg == "--show-output") - { - result.options.show_program_output = true; - continue; - } - if (arg == "--show-events") - { - result.options.show_events = true; - continue; - } - if (arg == "--format" || arg.rfind("--format=", 0) == 0) - { - std::string value; - if (arg != "--format") - { - value = arg.substr(9); - } - else if (i + 1 < argc) - { - value = argv[++i]; - } - if (value == "text" || value == "sarif") - { - result.format = value == "sarif" ? OutputFormat::Sarif : OutputFormat::Text; - continue; - } - result.ok = false; - result.error = "--format requires text or sarif"; - return result; - } - - result.ok = false; - result.error = - "unknown analyzer option: " + arg + " (compiler arguments go after --)"; - return result; - } - - if ((!compiler_args || result.options.compiler_args.empty()) && - result.options.test_directories.empty()) - { - result.ok = false; - result.error = "missing compiler arguments; use -- "; - } - - return result; - } - void PrintLocation(const SourceLocation& location) { std::cout << location.file << ':' << location.line; From 18909d526501a01291c85370fadb6b08df3b6aa5 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 27 Sep 2026 15:36:53 +0200 Subject: [PATCH 2/3] test: unit-test the CLI's arguments, the events and the SARIF writer cli_args_unit parses every option and every invalid spelling, and checks the compiler's -o in its four forms, the compile-only and -x flags, the C-as-C++ and DEBUG heuristics on real files, the compiler arguments of a run and the artifact names of a batch. events_unit feeds a captured run through CollectEvents and checks each counter, the colours stripped, and every branch of the verdict. sarif_unit compares WriteSarif with the exact log a consumer reads: empty, an error with its location and allocation site, a warning without a location or column. The hand-written Expect and the exit logic, copied in four tests, now live in unittests/expect.hpp. With the verdict, a counter, the artifact name, the --env message, a compile-only flag and the SARIF level broken in turn, eight checks fail. Part of #21. --- CMakeLists.txt | 7 + README.md | 3 + unittests/api_test.cpp | 21 +-- unittests/cli_args_test.cpp | 261 +++++++++++++++++++++++++++ unittests/events_test.cpp | 94 ++++++++++ unittests/expect.hpp | 35 ++++ unittests/findings_test.cpp | 22 +-- unittests/process_test.cpp | 22 +-- unittests/sarif_test.cpp | 152 ++++++++++++++++ unittests/shipped_toolchain_test.cpp | 22 +-- 10 files changed, 568 insertions(+), 71 deletions(-) create mode 100644 unittests/cli_args_test.cpp create mode 100644 unittests/events_test.cpp create mode 100644 unittests/expect.hpp create mode 100644 unittests/sarif_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 45298e9..3425dc5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -272,6 +272,13 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) target_link_libraries(shipped_toolchain_test PRIVATE ${TOOL_NAME}_lib) add_test(NAME shipped_toolchain_unit COMMAND shipped_toolchain_test) + foreach(_unit cli_args events sarif) + add_executable(${_unit}_test unittests/${_unit}_test.cpp) + target_include_directories(${_unit}_test PRIVATE src) + target_link_libraries(${_unit}_test PRIVATE ${TOOL_NAME}_lib) + add_test(NAME ${_unit}_unit COMMAND ${_unit}_test) + endforeach() + # The library API as a consumer links it: Run and RunBatch on real programs. add_executable(api_test unittests/api_test.cpp) target_link_libraries(api_test PRIVATE ${TOOL_NAME}_lib) diff --git a/README.md b/README.md index d2bb7b1..16f5928 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,9 @@ ctest --test-dir build --output-on-failure | Test | Checks | |---|---| | `findings_unit`, `process_unit` | the report parser and the process runner (`unittests/`) | +| `cli_args_unit` | argument parsing, every option and every invalid spelling, the compiler's `-o`, the compile-only and `-x` flags, the C-as-C++ and `DEBUG` heuristics, the compiler arguments of a run, the artifact names of a batch (`src/cli_args.hpp`) | +| `events_unit` | the CoreTrace event lines of a run and their summary, ANSI colours stripped, and the verdict drawn from a result (`src/events.hpp`) | +| `sarif_unit` | the SARIF writer against the exact log a consumer reads: empty, an error with its locations, a warning without one | | `api_unit` | the library API: `Run` and `RunBatch` on real programs, a heap overflow, a compile-only run, a hang against the timeout, a batch, the refused and empty cases | | `consumer` | a project fetching this repository the way ctrace does (`unittests/consumer/`) configures, builds, links `coretrace::runtime-analyzer_lib` and analyzes a program | | `fixtures` | every source under `test/` states what it proves in its header (`// args:`, `// expect:`), and the analyzer proves it: findings with rule, CWE, level, faulting line and allocation site, or none; the program's exit status and output when stated. A fixture without an expectation fails (`test/check_fixtures.py`) | diff --git a/unittests/api_test.cpp b/unittests/api_test.cpp index f282aed..a5e3b63 100644 --- a/unittests/api_test.cpp +++ b/unittests/api_test.cpp @@ -3,6 +3,8 @@ // compiled with coretrace-compiler, executed, and reported through the public structures. #include "runtime_analyzer.hpp" +#include "expect.hpp" + #include #include #include @@ -20,24 +22,15 @@ namespace using coretrace::runtime_analyzer::Run; using coretrace::runtime_analyzer::RunBatch; using coretrace::runtime_analyzer::Severity; + using unittests::Expect; constexpr std::string_view kHang = "#include \nint main(void)\n{\n for (;;)\n" " pause();\n}\n"; constexpr std::string_view kNormal = "int main(void)\n{\n return 0;\n}\n"; - int failures = 0; std::filesystem::path fixtures; // the repository's test/ directory std::filesystem::path work; // scratch directory, outside test/ (the sweeps run it) - void Expect(bool condition, std::string_view what) - { - std::cout << (condition ? "[PASS] " : "[FAIL] ") << what << '\n'; - if (!condition) - { - ++failures; - } - } - [[nodiscard]] bool Contains(std::string_view text, std::string_view needle) { return text.find(needle) != std::string_view::npos; @@ -194,11 +187,5 @@ int main(int argc, char** argv) RunBatchReportsEachSource(); RefusesCompileOnlyFlagsWithARun(); EmptyBatchIsNotAnalyzed(); - if (failures != 0) - { - std::cerr << failures << " check(s) failed\n"; - return 1; - } - std::cout << "api_test: all checks passed\n"; - return 0; + return unittests::Finish("api_test"); } diff --git a/unittests/cli_args_test.cpp b/unittests/cli_args_test.cpp new file mode 100644 index 0000000..09a9e88 --- /dev/null +++ b/unittests/cli_args_test.cpp @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 +// Checks the CLI's argument parsing and the per-source compiler arguments it derives. +#include "cli_args.hpp" + +#include "expect.hpp" + +#include +#include +#include +#include +#include + +namespace +{ + using coretrace::runtime_analyzer::AddPerSourceCompatibilityArgs; + using coretrace::runtime_analyzer::AnalyzerOptions; + using coretrace::runtime_analyzer::BuildCompilerArguments; + using coretrace::runtime_analyzer::CSourceLooksLikeCxx; + using coretrace::runtime_analyzer::FindOutputArgument; + using coretrace::runtime_analyzer::HasCompileOnlyAction; + using coretrace::runtime_analyzer::HasLanguageOverride; + using coretrace::runtime_analyzer::MakeTestOutputPath; + using coretrace::runtime_analyzer::OutputFormat; + using coretrace::runtime_analyzer::ParseArgs; + using coretrace::runtime_analyzer::ParseResult; + using coretrace::runtime_analyzer::SanitizeArtifactName; + using coretrace::runtime_analyzer::SourceRequiresDebugDefine; + using unittests::Expect; + + using Args = std::vector; + + [[nodiscard]] ParseResult Parse(Args args) + { + args.insert(args.begin(), "runtime-analyzer"); + std::vector argv; + for (std::string& arg : args) + { + argv.push_back(arg.data()); + } + return ParseArgs(static_cast(argv.size()), argv.data()); + } + + [[nodiscard]] std::filesystem::path Source(std::string_view name, std::string_view text) + { + const std::filesystem::path path = std::filesystem::temp_directory_path() / name; + std::ofstream(path) << text; + return path; + } + + void EveryOptionLandsInTheOptions() + { + const ParseResult parsed = Parse({"-o", + "bin", + "--run-arg", + "one", + "--run-arg", + "two", + "--env", + "A=1", + "--cwd", + "dir", + "--timeout", + "5", + "--test-dir", + "t1", + "--test-dir", + "t2", + "--output-dir", + "out", + "--strict-test-exit", + "--no-run", + "--show-output", + "--show-events", + "--format=sarif", + "--", + "--ct-modules=alloc", + "main.c"}); + Expect(parsed.ok && !parsed.help && !parsed.version, "all options: parsed"); + const AnalyzerOptions& o = parsed.options; + Expect(o.output_path == "bin" && o.explicit_output_path, "all options: -o"); + Expect(o.program_args == Args{"one", "two"}, "all options: --run-arg, repeatable"); + Expect(o.environment == Args{"A=1"}, "all options: --env"); + Expect(o.working_directory == "dir", "all options: --cwd"); + Expect(o.timeout == std::chrono::seconds(5), "all options: --timeout in seconds"); + Expect(o.test_directories == Args{"t1", "t2"}, "all options: --test-dir, repeatable"); + Expect(o.output_directory == "out", "all options: --output-dir"); + Expect(o.strict_test_exit && !o.run_program && o.show_program_output && o.show_events, + "all options: the flags"); + Expect(parsed.format == OutputFormat::Sarif, "all options: --format=sarif"); + Expect(o.compiler_args == Args{"--ct-modules=alloc", "main.c"}, + "all options: everything after -- is a compiler argument"); + } + + void DefaultsWithoutOptions() + { + const ParseResult parsed = Parse({"--", "main.c"}); + const AnalyzerOptions& o = parsed.options; + Expect(parsed.ok && parsed.format == OutputFormat::Text, "defaults: text output"); + Expect(o.output_path == "runtime-analyzer.out" && !o.explicit_output_path, + "defaults: the default binary path, not explicit"); + Expect(o.timeout == std::chrono::seconds(60) && o.run_program && !o.strict_test_exit && + !o.show_program_output && !o.show_events, + "defaults: 60 s, run, no flag"); + Expect(o.output_directory == "runtime-analyzer-artifacts", "defaults: artifact directory"); + } + + void OptionSpellings() + { + Expect(Parse({"--output=bin", "--", "a.c"}).options.output_path == "bin", + "--output="); + Expect(Parse({"--format", "text", "--", "a.c"}).format == OutputFormat::Text && + Parse({"--format", "sarif", "--", "a.c"}).format == OutputFormat::Sarif, + "--format as a separate argument"); + Expect(Parse({"--timeout", "0", "--", "a.c"}).options.timeout == + std::chrono::milliseconds(0), + "--timeout 0 disables the limit"); + Expect(Parse({"-h"}).help && Parse({"--help"}).help, "-h and --help"); + Expect(Parse({"--version"}).version, "--version"); + Expect(Parse({"--test-dir", "t"}).ok, "--test-dir needs no compiler argument"); + Expect(Parse({"--", "-o", "x", "a.c"}).options.compiler_args == Args{"-o", "x", "a.c"}, + "-o after -- belongs to the compiler"); + } + + void InvalidArguments() + { + struct Case + { + Args args; + std::string_view message; + }; + const Case cases[] = { + {{"--bogus", "--", "a.c"}, "unknown analyzer option: --bogus"}, + {{"-o"}, "-o requires a value"}, + {{"--run-arg"}, "--run-arg requires a value"}, + {{"--env"}, "--env requires NAME=VALUE"}, + {{"--env", "NOVALUE", "--", "a.c"}, "--env requires NAME=VALUE"}, + {{"--env", "=x", "--", "a.c"}, "--env requires NAME=VALUE"}, + {{"--cwd"}, "--cwd requires a path"}, + {{"--timeout"}, "--timeout requires a number of seconds"}, + {{"--timeout", "-1", "--", "a.c"}, "non-negative number of seconds"}, + {{"--timeout", "5x", "--", "a.c"}, "non-negative number of seconds"}, + {{"--test-dir"}, "--test-dir requires a path"}, + {{"--output-dir"}, "--output-dir requires a path"}, + {{"--format"}, "--format requires text or sarif"}, + {{"--format=xml", "--", "a.c"}, "--format requires text or sarif"}, + {{}, "missing compiler arguments"}, + {{"--"}, "missing compiler arguments"}, + }; + for (const Case& c : cases) + { + const ParseResult parsed = Parse(c.args); + Expect(!parsed.ok && parsed.error.find(c.message) != std::string::npos, + "invalid: " + std::string(c.message)); + } + } + + void OutputArgumentOfTheCompiler() + { + Expect(!FindOutputArgument({"a.c"}).present, "no -o: absent"); + Expect(FindOutputArgument({"-o", "x", "a.c"}).path == "x", "-o x"); + Expect(FindOutputArgument({"--output", "x"}).path == "x", "--output x"); + Expect(FindOutputArgument({"-o=x"}).path == "x", "-o=x"); + Expect(FindOutputArgument({"--output=x"}).path == "x", "--output=x"); + const auto last = FindOutputArgument({"a.c", "-o"}); + Expect(last.present && last.path.empty(), "-o as the last argument: present, no path"); + } + + void CompileOnlyAndLanguageFlags() + { + for (const char* flag : {"-c", "-S", "-E", "-emit-llvm", "--precompile", "-fsyntax-only"}) + { + Expect(HasCompileOnlyAction({"a.c", flag}), std::string("compile only: ") + flag); + } + Expect(!HasCompileOnlyAction({"-O2", "a.c"}), "compile only: not with -O2"); + Expect(HasLanguageOverride({"-x", "c++", "a.c"}) && HasLanguageOverride({"-xc++"}) && + HasLanguageOverride({"-x=c++"}), + "language override: -x c++, -xc++, -x=c++"); + Expect(!HasLanguageOverride({"-O2", "a.c"}), "language override: none"); + } + + void SourceHeuristics() + { + Expect(CSourceLooksLikeCxx("a.c", "#include \nint main() { std::string s; }"), + "a .c using std:: is C++"); + Expect(!CSourceLooksLikeCxx("a.c", "int main(void) { return 0; }"), "plain C is C"); + Expect(!CSourceLooksLikeCxx("a.cpp", "std::string s;"), "a .cpp needs no override"); + Expect(SourceRequiresDebugDefine("#ifndef DEBUG\n#error x\n#endif") && + SourceRequiresDebugDefine("#if !defined(DEBUG)"), + "DEBUG guard: #ifndef and #if !defined"); + Expect(!SourceRequiresDebugDefine("int main(void) { return 0; }"), "DEBUG guard: none"); + + Args args{"-O1"}; + AddPerSourceCompatibilityArgs(Source("cli_args_cxx.c", "std::string s;"), args); + Expect(args == Args{"-O1", "-x", "c++"}, "compat: -x c++ added for C that is C++"); + args = {"-x", "c"}; + AddPerSourceCompatibilityArgs(Source("cli_args_cxx.c", "std::string s;"), args); + Expect(args == Args{"-x", "c"}, "compat: the user's -x is kept"); + args = {}; + AddPerSourceCompatibilityArgs(Source("cli_args_debug.c", "#ifndef DEBUG\n#endif"), args); + Expect(args == Args{"-DDEBUG"}, "compat: -DDEBUG added for a DEBUG guard"); + args = {"-DDEBUG=1"}; + AddPerSourceCompatibilityArgs(Source("cli_args_debug.c", "#ifndef DEBUG\n#endif"), args); + Expect(args == Args{"-DDEBUG=1"}, "compat: the user's -DDEBUG is kept"); + args = {}; + AddPerSourceCompatibilityArgs(Source("cli_args_plain.c", "int main(void){}"), args); + Expect(args.empty(), "compat: nothing for plain C"); + } + + void CompilerArguments() + { + AnalyzerOptions options; + options.compiler_args = {"--instrument", "--ct-modules=alloc", "a.c"}; + options.output_path = "bin"; + std::string output; + Expect(BuildCompilerArguments(options, output) == + Args{"--ct-modules=alloc", "a.c", "-o", "bin"} && + output == "bin", + "compiler arguments: --instrument dropped, -o appended"); + options.compiler_args = {"a.c", "-o", "theirs"}; + Expect(BuildCompilerArguments(options, output) == Args{"a.c", "-o", "theirs"} && + output == "theirs", + "compiler arguments: the compiler's -o is kept and reported"); + options.compiler_args = {"a.c"}; + options.output_path.clear(); + (void)BuildCompilerArguments(options, output); + Expect(output == "runtime-analyzer.out", "compiler arguments: the default binary path"); + } + + void ArtifactNames() + { + Expect(SanitizeArtifactName("dir/sub/a b.c") == "dir_sub_a_b_c", + "artifact name: only letters, digits, - and _ survive"); + Expect(SanitizeArtifactName("") == "test", "artifact name: 'test' for an empty path"); + + AnalyzerOptions options; + options.output_directory = "out"; + const std::filesystem::path first = MakeTestOutputPath(options, "dir/a.c"); + const std::string name = first.filename().string(); + Expect(first.parent_path() == "out" && name.rfind("dir_a_", 0) == 0 && + name.size() == std::string("dir_a_").size() + 16, + "artifact path: /_<16 hex digits>"); + Expect(MakeTestOutputPath(options, "dir/a.c") == first, "artifact path: stable"); + Expect(MakeTestOutputPath(options, "dir/b.c") != first && + MakeTestOutputPath(options, "other/a.c") != first, + "artifact path: distinct sources get distinct paths"); + } +} // namespace + +int main() +{ + EveryOptionLandsInTheOptions(); + DefaultsWithoutOptions(); + OptionSpellings(); + InvalidArguments(); + OutputArgumentOfTheCompiler(); + CompileOnlyAndLanguageFlags(); + SourceHeuristics(); + CompilerArguments(); + ArtifactNames(); + return unittests::Finish("cli_args_test"); +} diff --git a/unittests/events_test.cpp b/unittests/events_test.cpp new file mode 100644 index 0000000..60f6032 --- /dev/null +++ b/unittests/events_test.cpp @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +// Checks what the analyzer reads from a run: the CoreTrace event lines, their summary, and the +// verdict it draws from a result. +#include "events.hpp" + +#include "expect.hpp" + +#include +#include + +namespace +{ + using coretrace::runtime_analyzer::AnalyzerResult; + using coretrace::runtime_analyzer::CollectEvents; + using coretrace::runtime_analyzer::CollectionSummary; + using coretrace::runtime_analyzer::Finding; + using coretrace::runtime_analyzer::kExitFindings; + using coretrace::runtime_analyzer::kExitNoFindings; + using coretrace::runtime_analyzer::kExitNotAnalyzed; + using coretrace::runtime_analyzer::MergeSummaries; + using coretrace::runtime_analyzer::StripAnsi; + using coretrace::runtime_analyzer::Verdict; + using unittests::Expect; + + void AnsiSequencesAreStripped() + { + Expect(StripAnsi("\x1b[31mred\x1b[0m plain") == "red plain", "ANSI: colours removed"); + Expect(StripAnsi("no escape") == "no escape", "ANSI: untouched otherwise"); + } + + void OnlyCoreTraceLinesAreEvents() + { + const std::string output = + "hello from the program\n" + "|1| ==ct== [INFO] tracing-malloc :: site=a.c:6:19\n" + "[ENTRY-FUNCTION] main\n" + "[EXIT-FUNCTION] main\n" + "\x1b[31m|1| ==ct== [ERROR] ct: heap-buffer-overflow WRITE\x1b[0m\n" + "|1| ==ct== [WARN] ct: leak ptr=0x1 size=16\n" + "[VTABLE-DIAG] box\n" + "bye\n"; + std::vector events; + const CollectionSummary summary = CollectEvents(output, events); + Expect(events.size() == 6 && summary.coretrace_lines == 6, + "events: the six CoreTrace lines, not the program's own"); + Expect(events[3] == "|1| ==ct== [ERROR] ct: heap-buffer-overflow WRITE", + "events: kept without their colours"); + Expect(summary.entry_events == 1 && summary.exit_events == 1, "summary: entry and exit"); + Expect(summary.allocation_events == 2, "summary: the malloc trace and the leak"); + Expect(summary.bounds_errors == 1 && summary.leak_reports == 1, + "summary: bounds error and leak report"); + Expect(summary.vtable_events == 1, "summary: vtable diagnostic"); + Expect(summary.warnings == 1 && summary.errors == 1, "summary: one WARN, one ERROR"); + } + + void SummariesAdd() + { + CollectionSummary a{1, 2, 3, 4, 5, 6, 7, 8, 9}; + const CollectionSummary b{10, 20, 30, 40, 50, 60, 70, 80, 90}; + a = MergeSummaries(a, b); + Expect(a.coretrace_lines == 11 && a.entry_events == 22 && a.exit_events == 33 && + a.allocation_events == 44 && a.bounds_errors == 55 && a.leak_reports == 66 && + a.vtable_events == 77 && a.warnings == 88 && a.errors == 99, + "merge: every counter adds up"); + } + + void TheVerdict() + { + AnalyzerResult result; + Expect(Verdict(result, true) == kExitNotAnalyzed, "verdict: not built is 2"); + result.compile_success = true; + Expect(Verdict(result, true) == kExitNotAnalyzed, "verdict: built but not run is 2"); + Expect(Verdict(result, false) == kExitNoFindings, "verdict: compile only is 0"); + result.executed = true; + Expect(Verdict(result, true) == kExitNoFindings, "verdict: clean run is 0"); + result.exit_code = 3; + Expect(Verdict(result, true) == kExitNoFindings, + "verdict: the program's own status is not a finding"); + result.timed_out = true; + Expect(Verdict(result, true) == kExitNotAnalyzed, + "verdict: timed out without findings is 2"); + result.findings.push_back(Finding{"memory-leak", "CWE-401"}); + Expect(Verdict(result, true) == kExitFindings, "verdict: findings are 1, even timed out"); + } +} // namespace + +int main() +{ + AnsiSequencesAreStripped(); + OnlyCoreTraceLinesAreEvents(); + SummariesAdd(); + TheVerdict(); + return unittests::Finish("events_test"); +} diff --git a/unittests/expect.hpp b/unittests/expect.hpp new file mode 100644 index 0000000..4a929d7 --- /dev/null +++ b/unittests/expect.hpp @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// The one check the unit tests share: print [PASS] or [FAIL] and count the failures. +#ifndef CORETRACE_RUNTIME_ANALYZER_UNITTESTS_EXPECT_HPP +#define CORETRACE_RUNTIME_ANALYZER_UNITTESTS_EXPECT_HPP + +#include +#include + +namespace unittests +{ + inline int failures = 0; + + inline void Expect(bool condition, std::string_view what) + { + std::cout << (condition ? "[PASS] " : "[FAIL] ") << what << '\n'; + if (!condition) + { + ++failures; + } + } + + // The exit status of a test executable: 1 with a failed check, 0 otherwise. + inline int Finish(std::string_view name) + { + if (failures != 0) + { + std::cerr << failures << " check(s) failed\n"; + return 1; + } + std::cout << name << ": all checks passed\n"; + return 0; + } +} // namespace unittests + +#endif // CORETRACE_RUNTIME_ANALYZER_UNITTESTS_EXPECT_HPP diff --git a/unittests/findings_test.cpp b/unittests/findings_test.cpp index e7e60d6..a59e9db 100644 --- a/unittests/findings_test.cpp +++ b/unittests/findings_test.cpp @@ -2,6 +2,8 @@ // Checks the runtime report parser against output captured from CoreTrace's runtime. #include "findings.hpp" +#include "expect.hpp" + #include #include @@ -11,17 +13,7 @@ namespace using coretrace::runtime_analyzer::ParseFindings; using coretrace::runtime_analyzer::Severity; using coretrace::runtime_analyzer::SourceLocation; - - int failures = 0; - - void Expect(bool condition, std::string_view what) - { - std::cout << (condition ? "[PASS] " : "[FAIL] ") << what << '\n'; - if (!condition) - { - ++failures; - } - } + using unittests::Expect; [[nodiscard]] bool IsAt(const std::optional& location, std::string_view file, unsigned line, unsigned column) @@ -317,11 +309,5 @@ int main() VtableInfoBoxIsTracing(); VcallBoxJoinsWrappedValues(); VtableRulesByWarning(); - if (failures != 0) - { - std::cerr << failures << " check(s) failed\n"; - return 1; - } - std::cout << "findings_test: all checks passed\n"; - return 0; + return unittests::Finish("findings_test"); } diff --git a/unittests/process_test.cpp b/unittests/process_test.cpp index 681bc4a..8c152af 100644 --- a/unittests/process_test.cpp +++ b/unittests/process_test.cpp @@ -2,6 +2,8 @@ // Checks the process runner on small shell commands. #include "process.hpp" +#include "expect.hpp" + #include #include #include @@ -11,17 +13,7 @@ namespace using coretrace::runtime_analyzer::ProcessResult; using coretrace::runtime_analyzer::ProcessSpec; using coretrace::runtime_analyzer::RunProcess; - - int failures = 0; - - void Expect(bool condition, std::string_view what) - { - std::cout << (condition ? "[PASS] " : "[FAIL] ") << what << '\n'; - if (!condition) - { - ++failures; - } - } + using unittests::Expect; [[nodiscard]] ProcessSpec Shell(std::string script) { @@ -94,11 +86,5 @@ int main() TimeoutKeepsTheOutputSoFar(); TimeoutKillsTheWholeProcessGroup(); NoTimeoutWaitsForTheProgram(); - if (failures != 0) - { - std::cerr << failures << " check(s) failed\n"; - return 1; - } - std::cout << "process_test: all checks passed\n"; - return 0; + return unittests::Finish("process_test"); } diff --git a/unittests/sarif_test.cpp b/unittests/sarif_test.cpp new file mode 100644 index 0000000..4a27b40 --- /dev/null +++ b/unittests/sarif_test.cpp @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 +// Checks the SARIF writer against the exact log a consumer reads. +#include "sarif.hpp" + +#include "expect.hpp" + +#include +#include +#include + +namespace +{ + using coretrace::runtime_analyzer::Finding; + using coretrace::runtime_analyzer::Severity; + using coretrace::runtime_analyzer::SourceLocation; + using coretrace::runtime_analyzer::WriteSarif; + using unittests::Expect; + + [[nodiscard]] std::string Sarif(const std::vector& findings) + { + std::ostringstream out; + WriteSarif(out, findings); + return out.str(); + } + + constexpr std::string_view kHead = R"({ + "$schema": "https://json.schemastore.org/sarif-2.1.0.json", + "runs": [ + { + "results": )"; + + constexpr std::string_view kTail = R"(, + "tool": { + "driver": { + "informationUri": "https://github.com/CoreTrace/coretrace-runtime-analyzer", + "name": "coretrace-runtime-analyzer" + } + } + } + ], + "version": "2.1.0" +} +)"; + + [[nodiscard]] std::string Log(std::string_view results) + { + return std::string(kHead) + std::string(results) + std::string(kTail); + } + + void EmptyLog() + { + Expect(Sarif({}) == Log("[]"), "no finding: an empty results array"); + } + + void FindingWithLocations() + { + Finding finding{"heap-buffer-overflow", + "CWE-122", + Severity::Error, + "heap-buffer-overflow WRITE of size 4", + SourceLocation{"a.c", 7, 15}, + SourceLocation{"a.c", 6, 19}}; + const std::string expected = Log(R"([ + { + "level": "error", + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "a.c" + }, + "region": { + "startColumn": 15, + "startLine": 7 + } + } + } + ], + "message": { + "text": "heap-buffer-overflow WRITE of size 4" + }, + "properties": { + "cwe": "CWE-122" + }, + "relatedLocations": [ + { + "id": 0, + "message": { + "text": "allocated here" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "a.c" + }, + "region": { + "startColumn": 19, + "startLine": 6 + } + } + } + ], + "ruleId": "heap-buffer-overflow" + } + ])"); + Expect(Sarif({finding}) == expected, + "one error: location, allocation site as related location 0, rule, CWE"); + } + + void WarningWithoutLocation() + { + Finding finding{"memory-leak", "CWE-401", Severity::Warning, + "leak of 16 bytes", std::nullopt, SourceLocation{"a.c", 6, 0}}; + const std::string expected = Log(R"([ + { + "level": "warning", + "message": { + "text": "leak of 16 bytes" + }, + "properties": { + "cwe": "CWE-401" + }, + "relatedLocations": [ + { + "id": 0, + "message": { + "text": "allocated here" + }, + "physicalLocation": { + "artifactLocation": { + "uri": "a.c" + }, + "region": { + "startLine": 6 + } + } + } + ], + "ruleId": "memory-leak" + } + ])"); + Expect(Sarif({finding}) == expected, + "one warning: no locations, no column when the runtime gave none"); + } +} // namespace + +int main() +{ + EmptyLog(); + FindingWithLocations(); + WarningWithoutLocation(); + return unittests::Finish("sarif_test"); +} diff --git a/unittests/shipped_toolchain_test.cpp b/unittests/shipped_toolchain_test.cpp index f925a8b..acd3caf 100644 --- a/unittests/shipped_toolchain_test.cpp +++ b/unittests/shipped_toolchain_test.cpp @@ -2,6 +2,8 @@ // Checks the rule that points coretrace-compiler at the Clang headers shipped next to the CLI. #include "shipped_toolchain.hpp" +#include "expect.hpp" + #include #include #include @@ -14,17 +16,7 @@ namespace { using coretrace::runtime_analyzer::ShippedLinkArguments; using coretrace::runtime_analyzer::UseShippedClangHeaders; - - int failures = 0; - - void Expect(bool condition, std::string_view what) - { - std::cout << (condition ? "[PASS] " : "[FAIL] ") << what << '\n'; - if (!condition) - { - ++failures; - } - } + using unittests::Expect; [[nodiscard]] std::string Env(const char* name) { @@ -96,11 +88,5 @@ int main() LeavesTheUsersClangAlone(); LinksAgainstTheShippedCxxRuntime(); DoesNothingWithoutHeaders(); - if (failures != 0) - { - std::cerr << failures << " check(s) failed\n"; - return 1; - } - std::cout << "shipped_toolchain_test: all checks passed\n"; - return 0; + return unittests::Finish("shipped_toolchain_test"); } From a653df85d7fb4db7a300e0cd8c57ce0663a706fe Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 27 Sep 2026 15:42:20 +0200 Subject: [PATCH 3/3] test: give the artifact-path checks real sources MakeTestOutputPath takes the source's path relative to the working directory. The checks passed it paths that did not exist: libc++ still returned them, libstdc++ did not, so on Linux every source collapsed to the same artifact name and two checks failed. Batch sources exist, so the checks now create dir/a.c, dir/b.c and other/a.c under a temporary directory and run from there. --- unittests/cli_args_test.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/unittests/cli_args_test.cpp b/unittests/cli_args_test.cpp index 09a9e88..bae96d7 100644 --- a/unittests/cli_args_test.cpp +++ b/unittests/cli_args_test.cpp @@ -232,6 +232,17 @@ namespace "artifact name: only letters, digits, - and _ survive"); Expect(SanitizeArtifactName("") == "test", "artifact name: 'test' for an empty path"); + // Batch sources exist, and their path is taken relative to the working directory. + const std::filesystem::path root = + std::filesystem::temp_directory_path() / "cli_args_artifacts"; + std::filesystem::remove_all(root); + for (const char* source : {"dir/a.c", "dir/b.c", "other/a.c"}) + { + std::filesystem::create_directories((root / source).parent_path()); + std::ofstream(root / source) << "int main(void) { return 0; }\n"; + } + std::filesystem::current_path(root); + AnalyzerOptions options; options.output_directory = "out"; const std::filesystem::path first = MakeTestOutputPath(options, "dir/a.c");