From 1e0b545dc00a472449a6c95dd5efa2d5cc61b74f Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 27 Sep 2026 15:45:39 +0200 Subject: [PATCH 1/3] build: fail configuration without Python 3 The fixtures, cli, timeout and compile_failure tests are Python scripts. Without an interpreter, configuration used to succeed and the suite ran without them, silently. Python 3 is now required when the tests are configured. The README also said CMake 3.16 where 3.21 has been required since the release archives. Part of #21. --- CMakeLists.txt | 38 +++++++++++++++++++------------------- README.md | 6 +++--- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3425dc5..ca0d980 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -293,25 +293,25 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) COMMAND sh -c "rm -rf \"${_consumer}\" && ${CMAKE_COMMAND} -S \"${CMAKE_CURRENT_SOURCE_DIR}/unittests/consumer\" -B \"${_consumer}\" -G \"${CMAKE_GENERATOR}\" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_COMPILER=\"${CMAKE_C_COMPILER}\" -DCMAKE_CXX_COMPILER=\"${CMAKE_CXX_COMPILER}\" -DLLVM_DIR=\"${LLVM_DIR}\" -DClang_DIR=\"${Clang_DIR}\" -DFETCHCONTENT_SOURCE_DIR_RUNTIME_ANALYZER=\"${CMAKE_CURRENT_SOURCE_DIR}\" -DFETCHCONTENT_SOURCE_DIR_CC=\"${cc_SOURCE_DIR}\" -DFETCHCONTENT_SOURCE_DIR_CORETRACE_LOGGER=\"${coretrace_logger_SOURCE_DIR}\" && ${CMAKE_COMMAND} --build \"${_consumer}\" --parallel && \"${_consumer}/consumer\" \"${CMAKE_CURRENT_SOURCE_DIR}/test/findings/heap_overflow_write.c\" \"${_consumer}/heap_overflow_write\" | grep -x 'heap-buffer-overflow CWE-122'") if(BUILD_CLI) - find_package(Python3 COMPONENTS Interpreter) - if(Python3_Interpreter_FOUND) - # Every fixture under test/ states what it proves in its header; a fixture without an - # expectation fails. - add_test(NAME fixtures - COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/check_fixtures.py" - $ "${CMAKE_CURRENT_SOURCE_DIR}/test" - "${CMAKE_CURRENT_BINARY_DIR}/fixtures") - add_test(NAME cli - COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/cli/check_cli.py" - $ "${CMAKE_CURRENT_BINARY_DIR}/cli") - add_test(NAME timeout - COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/timeout/check_timeout.py" - $ "${CMAKE_CURRENT_BINARY_DIR}/timeout") - add_test(NAME compile_failure - COMMAND ${Python3_EXECUTABLE} - "${CMAKE_CURRENT_SOURCE_DIR}/test/compile_failure/check_compile_failure.py" - $ "${CMAKE_CURRENT_BINARY_DIR}/compile_failure") - endif() + # The Python tests are part of the suite: without an interpreter, configuration fails + # rather than silently dropping them. + find_package(Python3 REQUIRED COMPONENTS Interpreter) + # Every fixture under test/ states what it proves in its header; a fixture without an + # expectation fails. + add_test(NAME fixtures + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/check_fixtures.py" + $ "${CMAKE_CURRENT_SOURCE_DIR}/test" + "${CMAKE_CURRENT_BINARY_DIR}/fixtures") + add_test(NAME cli + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/cli/check_cli.py" + $ "${CMAKE_CURRENT_BINARY_DIR}/cli") + add_test(NAME timeout + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/timeout/check_timeout.py" + $ "${CMAKE_CURRENT_BINARY_DIR}/timeout") + add_test(NAME compile_failure + COMMAND ${Python3_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/test/compile_failure/check_compile_failure.py" + $ "${CMAKE_CURRENT_BINARY_DIR}/compile_failure") # A program runs and its entry/exit events are collected. A shell checks the exit status # and the summary together: PASS_REGULAR_EXPRESSION alone would ignore the status. diff --git a/README.md b/README.md index 16f5928..ac6fd11 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,9 @@ link each instrumented program against the runtime, which is C++: `g++` on Debia ### From source -Needs CMake 3.16 or later, a C++20 compiler, LLVM and Clang 19 or later with their CMake -packages, and Python 3 for the tests. coretrace-compiler is fetched at configure time, at the -release tag pinned in `CMakeLists.txt`. +Needs CMake 3.21 or later, a C++20 compiler, LLVM and Clang 19 or later with their CMake +packages, and Python 3, which the test suite runs (configuration fails without it). +coretrace-compiler is fetched at configure time, at the release tag pinned in `CMakeLists.txt`. ```zsh # macOS: brew install llvm@20. Debian/Ubuntu: /usr/lib/llvm-20/lib/cmake/{llvm,clang}. From 397a44f2711a7c369a064882b83c571216032c64 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 27 Sep 2026 15:45:39 +0200 Subject: [PATCH 2/3] ci: measure the suite's coverage with llvm-cov and fail below a floor -DCORETRACE_COVERAGE=ON instruments this project's own targets (the library, the CLI and the unit tests) at -O0 for source-based coverage; fetched dependencies are not. scripts/ci/coverage.sh runs the suite with LLVM_PROFILE_FILE set, merges the profiles, reports src/, include/ and main.cpp per file, writes an HTML report, adds the totals to the GitHub Actions job summary, and fails when the line coverage is below the floor it is given. The coverage job of the Build workflow runs it on Ubuntu 24.04 with clang-20 and uploads the report. Its floor starts at 100 so that the first run shows the value reached; the next commit sets it there. Locally on macOS (AppleClang, xcrun llvm-cov): 89.20% of lines, 100% of functions; process.cpp is the low file at 58%, the other platform's paths. Part of #21. --- .github/workflows/build.yml | 48 ++++++++++++++++++++++ .gitignore | 4 +- CMakeLists.txt | 24 +++++++++++ README.md | 7 +++- scripts/ci/coverage.sh | 80 +++++++++++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 2 deletions(-) create mode 100755 scripts/ci/coverage.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cdc0acf..779d91c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,3 +50,51 @@ jobs: - name: Test run: ctest --test-dir build --output-on-failure + + # How much of this project's own code the suite runs, from a Clang build with + # -DCORETRACE_COVERAGE=ON (scripts/ci/coverage.sh): the totals in the job summary, the report + # as an artifact, and a failure below the line-coverage floor. + coverage: + name: Coverage (llvm-cov) + runs-on: ubuntu-24.04 + env: + COVERAGE_FLOOR: 100 + + steps: + - uses: actions/checkout@v4 + + - name: Install LLVM ${{ env.LLVM_VERSION }} with its profile runtime + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ninja-build + wget -qO /tmp/llvm.sh https://apt.llvm.org/llvm.sh + sudo bash /tmp/llvm.sh "$LLVM_VERSION" + sudo apt-get install -y "llvm-$LLVM_VERSION-dev" "libclang-$LLVM_VERSION-dev" \ + "clang-$LLVM_VERSION" "libclang-rt-$LLVM_VERSION-dev" + + - name: Configure + run: | + cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER="clang-$LLVM_VERSION" -DCMAKE_CXX_COMPILER="clang++-$LLVM_VERSION" \ + -DCORETRACE_COVERAGE=ON \ + -DLLVM_DIR="/usr/lib/llvm-$LLVM_VERSION/lib/cmake/llvm" \ + -DClang_DIR="/usr/lib/llvm-$LLVM_VERSION/lib/cmake/clang" + + - name: Build + run: cmake --build build --parallel + + - name: Test and measure + env: + LLVM_PROFDATA: /usr/lib/llvm-${{ env.LLVM_VERSION }}/bin/llvm-profdata + LLVM_COV: /usr/lib/llvm-${{ env.LLVM_VERSION }}/bin/llvm-cov + run: sh scripts/ci/coverage.sh build coverage "$COVERAGE_FLOOR" + + - name: Upload the coverage report + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage/report.txt + coverage/summary.md + coverage/html diff --git a/.gitignore b/.gitignore index 01c7c3a..2e66ea3 100644 --- a/.gitignore +++ b/.gitignore @@ -46,5 +46,7 @@ runtime-analyzer-artifacts/ # Python cache __pycache__/ -# Build tree +# Build trees and coverage reports build/ +build-*/ +coverage/ diff --git a/CMakeLists.txt b/CMakeLists.txt index ca0d980..2e0d1f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,11 @@ set(LLVM_COMPONENTS support ) +# Source-based coverage of this project's own code (scripts/ci/coverage.sh reads it): the +# library, the CLI and the unit tests are built at -O0 and instrumented; fetched dependencies +# are not. +option(CORETRACE_COVERAGE "Instrument this project's own targets for llvm-cov (Clang only)" OFF) + # CoreTrace compiler dependency option(CORETRACE_FETCH_COMPILERLIB "Fetch coretrace-compiler via FetchContent if needed" ON) set(CORETRACE_COMPILER_GIT_REPOSITORY "https://github.com/CoreTrace/coretrace-compiler.git" CACHE STRING "coretrace-compiler repository") @@ -74,6 +79,17 @@ set(TOOL_GENERATED_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include") configure_file(cmake/runtime_analyzer_version.hpp.in "${TOOL_GENERATED_INCLUDE_DIR}/runtime_analyzer_version.hpp" @ONLY) +if(CORETRACE_COVERAGE AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + message(FATAL_ERROR "CORETRACE_COVERAGE needs Clang: it uses source-based coverage") +endif() + +function(coretrace_enable_coverage target) + if(CORETRACE_COVERAGE) + target_compile_options(${target} PRIVATE -O0 -fprofile-instr-generate -fcoverage-mapping) + target_link_options(${target} PRIVATE -fprofile-instr-generate) + endif() +endfunction() + include(FetchContent) # ---------------------------------------------------------------------------- @@ -163,6 +179,8 @@ function(coretrace_add_tool) target_link_libraries(${_lib_target} PUBLIC ${_llvm_libs}) endif() + coretrace_enable_coverage(${_lib_target}) + # ALIAS for FetchContent consumers add_library(${CTOOL_NAMESPACE}::${_lib_target} ALIAS ${_lib_target}) @@ -176,6 +194,7 @@ function(coretrace_add_tool) ${CTOOL_CLI_SOURCE} ) target_link_libraries(${CTOOL_NAME} PRIVATE ${_lib_target}) + coretrace_enable_coverage(${CTOOL_NAME}) endif() endfunction() @@ -260,28 +279,33 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) add_executable(findings_test unittests/findings_test.cpp) target_include_directories(findings_test PRIVATE src) target_link_libraries(findings_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(findings_test) add_test(NAME findings_unit COMMAND findings_test) add_executable(process_test unittests/process_test.cpp) target_include_directories(process_test PRIVATE src) target_link_libraries(process_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(process_test) add_test(NAME process_unit COMMAND process_test) add_executable(shipped_toolchain_test unittests/shipped_toolchain_test.cpp) target_include_directories(shipped_toolchain_test PRIVATE src) target_link_libraries(shipped_toolchain_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(shipped_toolchain_test) add_test(NAME shipped_toolchain_unit COMMAND shipped_toolchain_test) foreach(_unit cli_args events sarif) add_executable(${_unit}_test unittests/${_unit}_test.cpp) target_include_directories(${_unit}_test PRIVATE src) target_link_libraries(${_unit}_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(${_unit}_test) add_test(NAME ${_unit}_unit COMMAND ${_unit}_test) endforeach() # The library API as a consumer links it: Run and RunBatch on real programs. add_executable(api_test unittests/api_test.cpp) target_link_libraries(api_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(api_test) add_test(NAME api_unit COMMAND api_test "${CMAKE_CURRENT_SOURCE_DIR}/test" "${CMAKE_CURRENT_BINARY_DIR}/api") diff --git a/README.md b/README.md index ac6fd11..d994c60 100644 --- a/README.md +++ b/README.md @@ -248,7 +248,12 @@ analyzer must report, for example: line to one platform. `test/check_fixtures.py` documents the keys. The `Build` workflow runs this suite on Ubuntu 24.04 and macOS against LLVM 20 for every push and -pull request. `clang-format` (version 17) is checked by the `clang-format` workflow. +pull request. Its `coverage` job measures how much of `src/`, `include/` and `main.cpp` the +suite runs, with `llvm-cov` on a Clang build configured with `-DCORETRACE_COVERAGE=ON`: the +totals go to the job summary, the report to an artifact, and the job fails when the line +coverage drops below the floor set in the workflow. The same measure runs locally with +`scripts/ci/coverage.sh []`, through `xcrun` on macOS. +`clang-format` (version 17) is checked by the `clang-format` workflow. ## Releasing diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh new file mode 100755 index 0000000..4d5c1a6 --- /dev/null +++ b/scripts/ci/coverage.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# SPDX-License-Identifier: Apache-2.0 +# +# coverage.sh [] +# +# Measures how much of this project's own code (src/, include/, main.cpp) the test suite runs, +# from a build configured with -DCORETRACE_COVERAGE=ON. Writes the llvm-cov text report and an +# HTML report, prints the totals, adds them to the GitHub Actions job summary when there is +# one, and fails when the line coverage is below the floor. A failing test fails the script. +# +# LLVM_PROFDATA and LLVM_COV name the LLVM tools matching the compiler (llvm-profdata-20 and +# llvm-cov-20 for clang-20); by default llvm-profdata and llvm-cov, through xcrun on macOS. +set -eu + +if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then + echo "usage: $0 []" >&2 + exit 2 +fi +build=$(cd "$1" && pwd) +mkdir -p "$2" +out=$(cd "$2" && pwd) +floor=${3:-0} +source_dir=$(cd "$(dirname "$0")/../.." && pwd) + +tool_prefix= +if [ "$(uname -s)" = Darwin ]; then + tool_prefix="xcrun " +fi +profdata=${LLVM_PROFDATA:-${tool_prefix}llvm-profdata} +cov=${LLVM_COV:-${tool_prefix}llvm-cov} + +# The CLI and every unit test executable: each holds part of the code the tests ran. +objects= +for binary in "$build"/*_test; do + if [ -x "$binary" ]; then + objects="$objects -object $binary" + fi +done + +rm -rf "$out/profiles" "$out/html" +mkdir -p "$out/profiles" +# %4m: one profile per binary, merged by the processes that share it; the Python tests spawn +# the CLI many times. +LLVM_PROFILE_FILE="$out/profiles/%4m.profraw" ctest --test-dir "$build" --output-on-failure + +$profdata merge -sparse "$out"/profiles/*.profraw -o "$out/all.profdata" +sources="$source_dir/src $source_dir/include $source_dir/main.cpp" +# llvm-cov warns that functions "have mismatched data": an inline function that one binary +# uses and another does not emit has two records. Each binary's counts are still read from its +# own record, so the totals are right. +# shellcheck disable=SC2086 # $objects and $sources are lists of words. +$cov report "$build/runtime-analyzer" $objects -instr-profile "$out/all.profdata" $sources \ + > "$out/report.txt" +# shellcheck disable=SC2086 +$cov show "$build/runtime-analyzer" $objects -instr-profile "$out/all.profdata" -format=html \ + -output-dir "$out/html" $sources + +# The TOTAL row of `llvm-cov report`: regions, missed, cover, functions, missed, executed, +# lines, missed, cover, branches, missed, cover. +lines=$(awk '$1 == "TOTAL" { print $10 }' "$out/report.txt") +{ + echo "| Lines | Functions | Branches |" + echo "|---|---|---|" + awk '$1 == "TOTAL" { printf "| %s | %s | %s |\n", $10, $7, $13 }' "$out/report.txt" +} > "$out/summary.md" +grep -v '^-' "$out/report.txt" +cat "$out/summary.md" +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Coverage of src/, include/ and main.cpp (floor: ${floor}% of lines)" + echo + cat "$out/summary.md" + } >> "$GITHUB_STEP_SUMMARY" +fi + +if awk -v lines="${lines%\%}" -v floor="$floor" 'BEGIN { exit !(lines + 0 < floor + 0) }'; then + echo "coverage: ${lines} of lines is below the floor of ${floor}%" >&2 + exit 1 +fi +echo "coverage: ${lines} of lines, floor ${floor}%" From 3a7ea9b4fa4be43b95bf06a2434dfa7021324d8e Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 27 Sep 2026 15:50:15 +0200 Subject: [PATCH 3/3] ci: set the coverage floor at 89% of lines The first run of the coverage job on Ubuntu 24.04 with clang-20 measured 89.20% of lines and 100% of functions, the same as the local macOS run. process.cpp is the low file at 58%: its child-side code between fork and exec never reaches a profile, and its pipe, fork and waitpid failure paths are not provoked. The floor is set just under the value reached, to be raised as tests land. Part of #21. --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 779d91c..7a0f405 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,7 +58,9 @@ jobs: name: Coverage (llvm-cov) runs-on: ubuntu-24.04 env: - COVERAGE_FLOOR: 100 + # Lines of src/, include/ and main.cpp the suite must run: 89.20% reached on + # 2026-09-27 (#21). Raise it as tests land. + COVERAGE_FLOOR: 89 steps: - uses: actions/checkout@v4