ci: require Python for the tests and measure their coverage with a floor

.github/workflows/build.yml

@@ -50,3 +50,53 @@ jobs:
 
       - name: Test
         run: ctest --test-dir build --output-on-failure
+
+  # How much of this project's own code the suite runs, from a Clang build with
+  # -DCORETRACE_COVERAGE=ON (scripts/ci/coverage.sh): the totals in the job summary, the report
+  # as an artifact, and a failure below the line-coverage floor.
+  coverage:
+    name: Coverage (llvm-cov)
+    runs-on: ubuntu-24.04
+    env:
+      # Lines of src/, include/ and main.cpp the suite must run: 89.20% reached on
+      # 2026-09-27 (#21). Raise it as tests land.
+      COVERAGE_FLOOR: 89
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install LLVM ${{ env.LLVM_VERSION }} with its profile runtime
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake ninja-build
+          wget -qO /tmp/llvm.sh https://apt.llvm.org/llvm.sh
+          sudo bash /tmp/llvm.sh "$LLVM_VERSION"
+          sudo apt-get install -y "llvm-$LLVM_VERSION-dev" "libclang-$LLVM_VERSION-dev" \
+            "clang-$LLVM_VERSION" "libclang-rt-$LLVM_VERSION-dev"
+
+      - name: Configure
+        run: |
+          cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_C_COMPILER="clang-$LLVM_VERSION" -DCMAKE_CXX_COMPILER="clang++-$LLVM_VERSION" \
+            -DCORETRACE_COVERAGE=ON \
+            -DLLVM_DIR="/usr/lib/llvm-$LLVM_VERSION/lib/cmake/llvm" \
+            -DClang_DIR="/usr/lib/llvm-$LLVM_VERSION/lib/cmake/clang"
+
+      - name: Build
+        run: cmake --build build --parallel
+
+      - name: Test and measure
+        env:
+          LLVM_PROFDATA: /usr/lib/llvm-${{ env.LLVM_VERSION }}/bin/llvm-profdata
+          LLVM_COV: /usr/lib/llvm-${{ env.LLVM_VERSION }}/bin/llvm-cov
+        run: sh scripts/ci/coverage.sh build coverage "$COVERAGE_FLOOR"
+
+      - name: Upload the coverage report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: |
+            coverage/report.txt
+            coverage/summary.md
+            coverage/html

.gitignore

@@ -46,5 +46,7 @@ runtime-analyzer-artifacts/
 # Python cache
 __pycache__/
 
-# Build tree
+# Build trees and coverage reports
 build/
+build-*/
+coverage/

CMakeLists.txt

@@ -42,6 +42,11 @@ set(LLVM_COMPONENTS
     support
 )
 
+# Source-based coverage of this project's own code (scripts/ci/coverage.sh reads it): the
+# library, the CLI and the unit tests are built at -O0 and instrumented; fetched dependencies
+# are not.
+option(CORETRACE_COVERAGE "Instrument this project's own targets for llvm-cov (Clang only)" OFF)
+
 # CoreTrace compiler dependency
 option(CORETRACE_FETCH_COMPILERLIB "Fetch coretrace-compiler via FetchContent if needed" ON)
 set(CORETRACE_COMPILER_GIT_REPOSITORY "https://github.com/CoreTrace/coretrace-compiler.git" CACHE STRING "coretrace-compiler repository")
@@ -74,6 +79,17 @@ set(TOOL_GENERATED_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include")
 configure_file(cmake/runtime_analyzer_version.hpp.in
                "${TOOL_GENERATED_INCLUDE_DIR}/runtime_analyzer_version.hpp" @ONLY)
 
+if(CORETRACE_COVERAGE AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    message(FATAL_ERROR "CORETRACE_COVERAGE needs Clang: it uses source-based coverage")
+endif()
+
+function(coretrace_enable_coverage target)
+    if(CORETRACE_COVERAGE)
+        target_compile_options(${target} PRIVATE -O0 -fprofile-instr-generate -fcoverage-mapping)
+        target_link_options(${target} PRIVATE -fprofile-instr-generate)
+    endif()
+endfunction()
+
 include(FetchContent)
 
 # ----------------------------------------------------------------------------
@@ -163,6 +179,8 @@ function(coretrace_add_tool)
         target_link_libraries(${_lib_target} PUBLIC ${_llvm_libs})
     endif()
 
+    coretrace_enable_coverage(${_lib_target})
+
     # ALIAS for FetchContent consumers
     add_library(${CTOOL_NAMESPACE}::${_lib_target} ALIAS ${_lib_target})
 
@@ -176,6 +194,7 @@ function(coretrace_add_tool)
             ${CTOOL_CLI_SOURCE}
         )
         target_link_libraries(${CTOOL_NAME} PRIVATE ${_lib_target})
+        coretrace_enable_coverage(${CTOOL_NAME})
     endif()
 endfunction()
 
@@ -260,28 +279,33 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
   add_executable(findings_test unittests/findings_test.cpp)
   target_include_directories(findings_test PRIVATE src)
   target_link_libraries(findings_test PRIVATE ${TOOL_NAME}_lib)
+  coretrace_enable_coverage(findings_test)
   add_test(NAME findings_unit COMMAND findings_test)
 
   add_executable(process_test unittests/process_test.cpp)
   target_include_directories(process_test PRIVATE src)
   target_link_libraries(process_test PRIVATE ${TOOL_NAME}_lib)
+  coretrace_enable_coverage(process_test)
   add_test(NAME process_unit COMMAND process_test)
 
   add_executable(shipped_toolchain_test unittests/shipped_toolchain_test.cpp)
   target_include_directories(shipped_toolchain_test PRIVATE src)
   target_link_libraries(shipped_toolchain_test PRIVATE ${TOOL_NAME}_lib)
+  coretrace_enable_coverage(shipped_toolchain_test)
   add_test(NAME shipped_toolchain_unit COMMAND shipped_toolchain_test)
 
   foreach(_unit cli_args events sarif)
     add_executable(${_unit}_test unittests/${_unit}_test.cpp)
     target_include_directories(${_unit}_test PRIVATE src)
     target_link_libraries(${_unit}_test PRIVATE ${TOOL_NAME}_lib)
+    coretrace_enable_coverage(${_unit}_test)
     add_test(NAME ${_unit}_unit COMMAND ${_unit}_test)
   endforeach()
 
   # The library API as a consumer links it: Run and RunBatch on real programs.
   add_executable(api_test unittests/api_test.cpp)
   target_link_libraries(api_test PRIVATE ${TOOL_NAME}_lib)
+  coretrace_enable_coverage(api_test)
   add_test(NAME api_unit
     COMMAND api_test "${CMAKE_CURRENT_SOURCE_DIR}/test" "${CMAKE_CURRENT_BINARY_DIR}/api")
 
@@ -293,25 +317,25 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     COMMAND sh -c "rm -rf \"${_consumer}\" && ${CMAKE_COMMAND} -S \"${CMAKE_CURRENT_SOURCE_DIR}/unittests/consumer\" -B \"${_consumer}\" -G \"${CMAKE_GENERATOR}\" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_COMPILER=\"${CMAKE_C_COMPILER}\" -DCMAKE_CXX_COMPILER=\"${CMAKE_CXX_COMPILER}\" -DLLVM_DIR=\"${LLVM_DIR}\" -DClang_DIR=\"${Clang_DIR}\" -DFETCHCONTENT_SOURCE_DIR_RUNTIME_ANALYZER=\"${CMAKE_CURRENT_SOURCE_DIR}\" -DFETCHCONTENT_SOURCE_DIR_CC=\"${cc_SOURCE_DIR}\" -DFETCHCONTENT_SOURCE_DIR_CORETRACE_LOGGER=\"${coretrace_logger_SOURCE_DIR}\" && ${CMAKE_COMMAND} --build \"${_consumer}\" --parallel && \"${_consumer}/consumer\" \"${CMAKE_CURRENT_SOURCE_DIR}/test/findings/heap_overflow_write.c\" \"${_consumer}/heap_overflow_write\" | grep -x 'heap-buffer-overflow CWE-122'")
 
   if(BUILD_CLI)
-    find_package(Python3 COMPONENTS Interpreter)
-    if(Python3_Interpreter_FOUND)
-      # Every fixture under test/ states what it proves in its header; a fixture without an
-      # expectation fails.
-      add_test(NAME fixtures
-        COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/check_fixtures.py"
-                $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_SOURCE_DIR}/test"
-                "${CMAKE_CURRENT_BINARY_DIR}/fixtures")
-      add_test(NAME cli
-        COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/cli/check_cli.py"
-                $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_BINARY_DIR}/cli")
-      add_test(NAME timeout
-        COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/timeout/check_timeout.py"
-                $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_BINARY_DIR}/timeout")
-      add_test(NAME compile_failure
-        COMMAND ${Python3_EXECUTABLE}
-                "${CMAKE_CURRENT_SOURCE_DIR}/test/compile_failure/check_compile_failure.py"
-                $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_BINARY_DIR}/compile_failure")
-    endif()
+    # The Python tests are part of the suite: without an interpreter, configuration fails
+    # rather than silently dropping them.
+    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+    # Every fixture under test/ states what it proves in its header; a fixture without an
+    # expectation fails.
+    add_test(NAME fixtures
+      COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/check_fixtures.py"
+              $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_SOURCE_DIR}/test"
+              "${CMAKE_CURRENT_BINARY_DIR}/fixtures")
+    add_test(NAME cli
+      COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/cli/check_cli.py"
+              $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_BINARY_DIR}/cli")
+    add_test(NAME timeout
+      COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/timeout/check_timeout.py"
+              $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_BINARY_DIR}/timeout")
+    add_test(NAME compile_failure
+      COMMAND ${Python3_EXECUTABLE}
+              "${CMAKE_CURRENT_SOURCE_DIR}/test/compile_failure/check_compile_failure.py"
+              $<TARGET_FILE:${TOOL_NAME}> "${CMAKE_CURRENT_BINARY_DIR}/compile_failure")
 
     # A program runs and its entry/exit events are collected. A shell checks the exit status
     # and the summary together: PASS_REGULAR_EXPRESSION alone would ignore the status.

README.md

@@ -59,9 +59,9 @@ link each instrumented program against the runtime, which is C++: `g++` on Debia
 
 ### From source
 
-Needs CMake 3.16 or later, a C++20 compiler, LLVM and Clang 19 or later with their CMake
-packages, and Python 3 for the tests. coretrace-compiler is fetched at configure time, at the
-release tag pinned in `CMakeLists.txt`.
+Needs CMake 3.21 or later, a C++20 compiler, LLVM and Clang 19 or later with their CMake
+packages, and Python 3, which the test suite runs (configuration fails without it).
+coretrace-compiler is fetched at configure time, at the release tag pinned in `CMakeLists.txt`.
 
 ```zsh
 # macOS: brew install llvm@20. Debian/Ubuntu: /usr/lib/llvm-20/lib/cmake/{llvm,clang}.
@@ -248,7 +248,12 @@ analyzer must report, for example:
 line to one platform. `test/check_fixtures.py` documents the keys.
 
 The `Build` workflow runs this suite on Ubuntu 24.04 and macOS against LLVM 20 for every push and
-pull request. `clang-format` (version 17) is checked by the `clang-format` workflow.
+pull request. Its `coverage` job measures how much of `src/`, `include/` and `main.cpp` the
+suite runs, with `llvm-cov` on a Clang build configured with `-DCORETRACE_COVERAGE=ON`: the
+totals go to the job summary, the report to an artifact, and the job fails when the line
+coverage drops below the floor set in the workflow. The same measure runs locally with
+`scripts/ci/coverage.sh <build dir> <output dir> [<floor>]`, through `xcrun` on macOS.
+`clang-format` (version 17) is checked by the `clang-format` workflow.
 
 ## Releasing
 

scripts/ci/coverage.sh

@@ -0,0 +1,80 @@
+#!/bin/sh
+# SPDX-License-Identifier: Apache-2.0
+#
+# coverage.sh <build directory> <output directory> [<line coverage floor, in percent>]
+#
+# Measures how much of this project's own code (src/, include/, main.cpp) the test suite runs,
+# from a build configured with -DCORETRACE_COVERAGE=ON. Writes the llvm-cov text report and an
+# HTML report, prints the totals, adds them to the GitHub Actions job summary when there is
+# one, and fails when the line coverage is below the floor. A failing test fails the script.
+#
+# LLVM_PROFDATA and LLVM_COV name the LLVM tools matching the compiler (llvm-profdata-20 and
+# llvm-cov-20 for clang-20); by default llvm-profdata and llvm-cov, through xcrun on macOS.
+set -eu
+
+if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
+    echo "usage: $0 <build directory> <output directory> [<line coverage floor>]" >&2
+    exit 2
+fi
+build=$(cd "$1" && pwd)
+mkdir -p "$2"
+out=$(cd "$2" && pwd)
+floor=${3:-0}
+source_dir=$(cd "$(dirname "$0")/../.." && pwd)
+
+tool_prefix=
+if [ "$(uname -s)" = Darwin ]; then
+    tool_prefix="xcrun "
+fi
+profdata=${LLVM_PROFDATA:-${tool_prefix}llvm-profdata}
+cov=${LLVM_COV:-${tool_prefix}llvm-cov}
+
+# The CLI and every unit test executable: each holds part of the code the tests ran.
+objects=
+for binary in "$build"/*_test; do
+    if [ -x "$binary" ]; then
+        objects="$objects -object $binary"
+    fi
+done
+
+rm -rf "$out/profiles" "$out/html"
+mkdir -p "$out/profiles"
+# %4m: one profile per binary, merged by the processes that share it; the Python tests spawn
+# the CLI many times.
+LLVM_PROFILE_FILE="$out/profiles/%4m.profraw" ctest --test-dir "$build" --output-on-failure
+
+$profdata merge -sparse "$out"/profiles/*.profraw -o "$out/all.profdata"
+sources="$source_dir/src $source_dir/include $source_dir/main.cpp"
+# llvm-cov warns that functions "have mismatched data": an inline function that one binary
+# uses and another does not emit has two records. Each binary's counts are still read from its
+# own record, so the totals are right.
+# shellcheck disable=SC2086 # $objects and $sources are lists of words.
+$cov report "$build/runtime-analyzer" $objects -instr-profile "$out/all.profdata" $sources \
+    > "$out/report.txt"
+# shellcheck disable=SC2086
+$cov show "$build/runtime-analyzer" $objects -instr-profile "$out/all.profdata" -format=html \
+    -output-dir "$out/html" $sources
+
+# The TOTAL row of `llvm-cov report`: regions, missed, cover, functions, missed, executed,
+# lines, missed, cover, branches, missed, cover.
+lines=$(awk '$1 == "TOTAL" { print $10 }' "$out/report.txt")
+{
+    echo "| Lines | Functions | Branches |"
+    echo "|---|---|---|"
+    awk '$1 == "TOTAL" { printf "| %s | %s | %s |\n", $10, $7, $13 }' "$out/report.txt"
+} > "$out/summary.md"
+grep -v '^-' "$out/report.txt"
+cat "$out/summary.md"
+if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+    {
+        echo "### Coverage of src/, include/ and main.cpp (floor: ${floor}% of lines)"
+        echo
+        cat "$out/summary.md"
+    } >> "$GITHUB_STEP_SUMMARY"
+fi
+
+if awk -v lines="${lines%\%}" -v floor="$floor" 'BEGIN { exit !(lines + 0 < floor + 0) }'; then
+    echo "coverage: ${lines} of lines is below the floor of ${floor}%" >&2
+    exit 1
+fi
+echo "coverage: ${lines} of lines, floor ${floor}%"