diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cdc0acf..7a0f405 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,3 +50,53 @@ jobs: - name: Test run: ctest --test-dir build --output-on-failure + + # How much of this project's own code the suite runs, from a Clang build with + # -DCORETRACE_COVERAGE=ON (scripts/ci/coverage.sh): the totals in the job summary, the report + # as an artifact, and a failure below the line-coverage floor. + coverage: + name: Coverage (llvm-cov) + runs-on: ubuntu-24.04 + env: + # Lines of src/, include/ and main.cpp the suite must run: 89.20% reached on + # 2026-09-27 (#21). Raise it as tests land. + COVERAGE_FLOOR: 89 + + steps: + - uses: actions/checkout@v4 + + - name: Install LLVM ${{ env.LLVM_VERSION }} with its profile runtime + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ninja-build + wget -qO /tmp/llvm.sh https://apt.llvm.org/llvm.sh + sudo bash /tmp/llvm.sh "$LLVM_VERSION" + sudo apt-get install -y "llvm-$LLVM_VERSION-dev" "libclang-$LLVM_VERSION-dev" \ + "clang-$LLVM_VERSION" "libclang-rt-$LLVM_VERSION-dev" + + - name: Configure + run: | + cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER="clang-$LLVM_VERSION" -DCMAKE_CXX_COMPILER="clang++-$LLVM_VERSION" \ + -DCORETRACE_COVERAGE=ON \ + -DLLVM_DIR="/usr/lib/llvm-$LLVM_VERSION/lib/cmake/llvm" \ + -DClang_DIR="/usr/lib/llvm-$LLVM_VERSION/lib/cmake/clang" + + - name: Build + run: cmake --build build --parallel + + - name: Test and measure + env: + LLVM_PROFDATA: /usr/lib/llvm-${{ env.LLVM_VERSION }}/bin/llvm-profdata + LLVM_COV: /usr/lib/llvm-${{ env.LLVM_VERSION }}/bin/llvm-cov + run: sh scripts/ci/coverage.sh build coverage "$COVERAGE_FLOOR" + + - name: Upload the coverage report + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage/report.txt + coverage/summary.md + coverage/html diff --git a/.gitignore b/.gitignore index 01c7c3a..2e66ea3 100644 --- a/.gitignore +++ b/.gitignore @@ -46,5 +46,7 @@ runtime-analyzer-artifacts/ # Python cache __pycache__/ -# Build tree +# Build trees and coverage reports build/ +build-*/ +coverage/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 3425dc5..2e0d1f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,11 @@ set(LLVM_COMPONENTS support ) +# Source-based coverage of this project's own code (scripts/ci/coverage.sh reads it): the +# library, the CLI and the unit tests are built at -O0 and instrumented; fetched dependencies +# are not. +option(CORETRACE_COVERAGE "Instrument this project's own targets for llvm-cov (Clang only)" OFF) + # CoreTrace compiler dependency option(CORETRACE_FETCH_COMPILERLIB "Fetch coretrace-compiler via FetchContent if needed" ON) set(CORETRACE_COMPILER_GIT_REPOSITORY "https://github.com/CoreTrace/coretrace-compiler.git" CACHE STRING "coretrace-compiler repository") @@ -74,6 +79,17 @@ set(TOOL_GENERATED_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include") configure_file(cmake/runtime_analyzer_version.hpp.in "${TOOL_GENERATED_INCLUDE_DIR}/runtime_analyzer_version.hpp" @ONLY) +if(CORETRACE_COVERAGE AND NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + message(FATAL_ERROR "CORETRACE_COVERAGE needs Clang: it uses source-based coverage") +endif() + +function(coretrace_enable_coverage target) + if(CORETRACE_COVERAGE) + target_compile_options(${target} PRIVATE -O0 -fprofile-instr-generate -fcoverage-mapping) + target_link_options(${target} PRIVATE -fprofile-instr-generate) + endif() +endfunction() + include(FetchContent) # ---------------------------------------------------------------------------- @@ -163,6 +179,8 @@ function(coretrace_add_tool) target_link_libraries(${_lib_target} PUBLIC ${_llvm_libs}) endif() + coretrace_enable_coverage(${_lib_target}) + # ALIAS for FetchContent consumers add_library(${CTOOL_NAMESPACE}::${_lib_target} ALIAS ${_lib_target}) @@ -176,6 +194,7 @@ function(coretrace_add_tool) ${CTOOL_CLI_SOURCE} ) target_link_libraries(${CTOOL_NAME} PRIVATE ${_lib_target}) + coretrace_enable_coverage(${CTOOL_NAME}) endif() endfunction() @@ -260,28 +279,33 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) add_executable(findings_test unittests/findings_test.cpp) target_include_directories(findings_test PRIVATE src) target_link_libraries(findings_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(findings_test) add_test(NAME findings_unit COMMAND findings_test) add_executable(process_test unittests/process_test.cpp) target_include_directories(process_test PRIVATE src) target_link_libraries(process_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(process_test) add_test(NAME process_unit COMMAND process_test) add_executable(shipped_toolchain_test unittests/shipped_toolchain_test.cpp) target_include_directories(shipped_toolchain_test PRIVATE src) target_link_libraries(shipped_toolchain_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(shipped_toolchain_test) add_test(NAME shipped_toolchain_unit COMMAND shipped_toolchain_test) foreach(_unit cli_args events sarif) add_executable(${_unit}_test unittests/${_unit}_test.cpp) target_include_directories(${_unit}_test PRIVATE src) target_link_libraries(${_unit}_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(${_unit}_test) add_test(NAME ${_unit}_unit COMMAND ${_unit}_test) endforeach() # The library API as a consumer links it: Run and RunBatch on real programs. add_executable(api_test unittests/api_test.cpp) target_link_libraries(api_test PRIVATE ${TOOL_NAME}_lib) + coretrace_enable_coverage(api_test) add_test(NAME api_unit COMMAND api_test "${CMAKE_CURRENT_SOURCE_DIR}/test" "${CMAKE_CURRENT_BINARY_DIR}/api") @@ -293,25 +317,25 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) COMMAND sh -c "rm -rf \"${_consumer}\" && ${CMAKE_COMMAND} -S \"${CMAKE_CURRENT_SOURCE_DIR}/unittests/consumer\" -B \"${_consumer}\" -G \"${CMAKE_GENERATOR}\" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_COMPILER=\"${CMAKE_C_COMPILER}\" -DCMAKE_CXX_COMPILER=\"${CMAKE_CXX_COMPILER}\" -DLLVM_DIR=\"${LLVM_DIR}\" -DClang_DIR=\"${Clang_DIR}\" -DFETCHCONTENT_SOURCE_DIR_RUNTIME_ANALYZER=\"${CMAKE_CURRENT_SOURCE_DIR}\" -DFETCHCONTENT_SOURCE_DIR_CC=\"${cc_SOURCE_DIR}\" -DFETCHCONTENT_SOURCE_DIR_CORETRACE_LOGGER=\"${coretrace_logger_SOURCE_DIR}\" && ${CMAKE_COMMAND} --build \"${_consumer}\" --parallel && \"${_consumer}/consumer\" \"${CMAKE_CURRENT_SOURCE_DIR}/test/findings/heap_overflow_write.c\" \"${_consumer}/heap_overflow_write\" | grep -x 'heap-buffer-overflow CWE-122'") if(BUILD_CLI) - find_package(Python3 COMPONENTS Interpreter) - if(Python3_Interpreter_FOUND) - # Every fixture under test/ states what it proves in its header; a fixture without an - # expectation fails. - add_test(NAME fixtures - COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/check_fixtures.py" - $ "${CMAKE_CURRENT_SOURCE_DIR}/test" - "${CMAKE_CURRENT_BINARY_DIR}/fixtures") - add_test(NAME cli - COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/cli/check_cli.py" - $ "${CMAKE_CURRENT_BINARY_DIR}/cli") - add_test(NAME timeout - COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/timeout/check_timeout.py" - $ "${CMAKE_CURRENT_BINARY_DIR}/timeout") - add_test(NAME compile_failure - COMMAND ${Python3_EXECUTABLE} - "${CMAKE_CURRENT_SOURCE_DIR}/test/compile_failure/check_compile_failure.py" - $ "${CMAKE_CURRENT_BINARY_DIR}/compile_failure") - endif() + # The Python tests are part of the suite: without an interpreter, configuration fails + # rather than silently dropping them. + find_package(Python3 REQUIRED COMPONENTS Interpreter) + # Every fixture under test/ states what it proves in its header; a fixture without an + # expectation fails. + add_test(NAME fixtures + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/check_fixtures.py" + $ "${CMAKE_CURRENT_SOURCE_DIR}/test" + "${CMAKE_CURRENT_BINARY_DIR}/fixtures") + add_test(NAME cli + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/cli/check_cli.py" + $ "${CMAKE_CURRENT_BINARY_DIR}/cli") + add_test(NAME timeout + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/timeout/check_timeout.py" + $ "${CMAKE_CURRENT_BINARY_DIR}/timeout") + add_test(NAME compile_failure + COMMAND ${Python3_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/test/compile_failure/check_compile_failure.py" + $ "${CMAKE_CURRENT_BINARY_DIR}/compile_failure") # A program runs and its entry/exit events are collected. A shell checks the exit status # and the summary together: PASS_REGULAR_EXPRESSION alone would ignore the status. diff --git a/README.md b/README.md index 16f5928..d994c60 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,9 @@ link each instrumented program against the runtime, which is C++: `g++` on Debia ### From source -Needs CMake 3.16 or later, a C++20 compiler, LLVM and Clang 19 or later with their CMake -packages, and Python 3 for the tests. coretrace-compiler is fetched at configure time, at the -release tag pinned in `CMakeLists.txt`. +Needs CMake 3.21 or later, a C++20 compiler, LLVM and Clang 19 or later with their CMake +packages, and Python 3, which the test suite runs (configuration fails without it). +coretrace-compiler is fetched at configure time, at the release tag pinned in `CMakeLists.txt`. ```zsh # macOS: brew install llvm@20. Debian/Ubuntu: /usr/lib/llvm-20/lib/cmake/{llvm,clang}. @@ -248,7 +248,12 @@ analyzer must report, for example: line to one platform. `test/check_fixtures.py` documents the keys. The `Build` workflow runs this suite on Ubuntu 24.04 and macOS against LLVM 20 for every push and -pull request. `clang-format` (version 17) is checked by the `clang-format` workflow. +pull request. Its `coverage` job measures how much of `src/`, `include/` and `main.cpp` the +suite runs, with `llvm-cov` on a Clang build configured with `-DCORETRACE_COVERAGE=ON`: the +totals go to the job summary, the report to an artifact, and the job fails when the line +coverage drops below the floor set in the workflow. The same measure runs locally with +`scripts/ci/coverage.sh []`, through `xcrun` on macOS. +`clang-format` (version 17) is checked by the `clang-format` workflow. ## Releasing diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh new file mode 100755 index 0000000..4d5c1a6 --- /dev/null +++ b/scripts/ci/coverage.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# SPDX-License-Identifier: Apache-2.0 +# +# coverage.sh [] +# +# Measures how much of this project's own code (src/, include/, main.cpp) the test suite runs, +# from a build configured with -DCORETRACE_COVERAGE=ON. Writes the llvm-cov text report and an +# HTML report, prints the totals, adds them to the GitHub Actions job summary when there is +# one, and fails when the line coverage is below the floor. A failing test fails the script. +# +# LLVM_PROFDATA and LLVM_COV name the LLVM tools matching the compiler (llvm-profdata-20 and +# llvm-cov-20 for clang-20); by default llvm-profdata and llvm-cov, through xcrun on macOS. +set -eu + +if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then + echo "usage: $0 []" >&2 + exit 2 +fi +build=$(cd "$1" && pwd) +mkdir -p "$2" +out=$(cd "$2" && pwd) +floor=${3:-0} +source_dir=$(cd "$(dirname "$0")/../.." && pwd) + +tool_prefix= +if [ "$(uname -s)" = Darwin ]; then + tool_prefix="xcrun " +fi +profdata=${LLVM_PROFDATA:-${tool_prefix}llvm-profdata} +cov=${LLVM_COV:-${tool_prefix}llvm-cov} + +# The CLI and every unit test executable: each holds part of the code the tests ran. +objects= +for binary in "$build"/*_test; do + if [ -x "$binary" ]; then + objects="$objects -object $binary" + fi +done + +rm -rf "$out/profiles" "$out/html" +mkdir -p "$out/profiles" +# %4m: one profile per binary, merged by the processes that share it; the Python tests spawn +# the CLI many times. +LLVM_PROFILE_FILE="$out/profiles/%4m.profraw" ctest --test-dir "$build" --output-on-failure + +$profdata merge -sparse "$out"/profiles/*.profraw -o "$out/all.profdata" +sources="$source_dir/src $source_dir/include $source_dir/main.cpp" +# llvm-cov warns that functions "have mismatched data": an inline function that one binary +# uses and another does not emit has two records. Each binary's counts are still read from its +# own record, so the totals are right. +# shellcheck disable=SC2086 # $objects and $sources are lists of words. +$cov report "$build/runtime-analyzer" $objects -instr-profile "$out/all.profdata" $sources \ + > "$out/report.txt" +# shellcheck disable=SC2086 +$cov show "$build/runtime-analyzer" $objects -instr-profile "$out/all.profdata" -format=html \ + -output-dir "$out/html" $sources + +# The TOTAL row of `llvm-cov report`: regions, missed, cover, functions, missed, executed, +# lines, missed, cover, branches, missed, cover. +lines=$(awk '$1 == "TOTAL" { print $10 }' "$out/report.txt") +{ + echo "| Lines | Functions | Branches |" + echo "|---|---|---|" + awk '$1 == "TOTAL" { printf "| %s | %s | %s |\n", $10, $7, $13 }' "$out/report.txt" +} > "$out/summary.md" +grep -v '^-' "$out/report.txt" +cat "$out/summary.md" +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Coverage of src/, include/ and main.cpp (floor: ${floor}% of lines)" + echo + cat "$out/summary.md" + } >> "$GITHUB_STEP_SUMMARY" +fi + +if awk -v lines="${lines%\%}" -v floor="$floor" 'BEGIN { exit !(lines + 0 < floor + 0) }'; then + echo "coverage: ${lines} of lines is below the floor of ${floor}%" >&2 + exit 1 +fi +echo "coverage: ${lines} of lines, floor ${floor}%"