Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 28 additions & 195 deletions .github/workflows/bench.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@ name: Benchmarks
# Both land in `core.bench.json` (the Observatory's bench format 2), written
# by .github/scripts/bench-report.py; on a pull request it's an artifact.
#
# The jobs are the shared reusable workflow's (ComlineProject/.github,
# bench.yml), run in the bench CI image (valgrind and iai-callgrind-runner in
# it, nothing installed here; the container's default options lift the seccomp
# profile valgrind needs). What is core's is the report script, the ledger and
# the inputs below.
#
# Accepting a regression: a benchmark that grows past the limit fails, unless
# the pull request appends an `[[accepted]]` entry for it to the ledger,
# core/benches/accepted-regressions.toml, naming the benchmark, the most it
Expand All @@ -26,7 +32,7 @@ name: Benchmarks
# Plain `pull_request`, not `pull_request_target`: a fork's pull request runs
# with a read-only token and no secrets.
#
# Publishing: on every push to `master` (and a manual run), the `master` job
# Publishing: on every push to `master` (and a manual run), the publish job
# benches the head alone (nothing to gate) and commits `core.bench.json` to
# the `reports` branch with the Observatory's publish-report action. Only
# that job can write, and only from `master`: a manual run from another
Expand All @@ -40,202 +46,29 @@ on:
permissions:
contents: read

# A newer push to the same pull request or branch supersedes this run.
# A newer push to the same pull request supersedes this run; a branch's runs
# (the publish) queue instead.
concurrency:
group: bench-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

env:
CARGO_TERM_COLOR: always
# Must match the `iai-callgrind` dev-dependency (core/Cargo.toml).
IAI_CALLGRIND_RUNNER_VERSION: 0.16.1
# A benchmark fails when its instruction count grows by more than this many
# percent, unless the ledger accepts it (see the top). With one codegen
# unit (`[profile.bench]` in Cargo.toml) a change to one function leaves
# the others' counts alone.
BENCH_LIMIT_PERCENT: 1
LEDGER: core/benches/accepted-regressions.toml
# Base and head build into separate target directories (in a shared one,
# the head's older source mtimes can make Cargo reuse the base's build), on
# paths of the same length: a bench binary's path is in its arguments, and
# its length moves the heap, which moves some counts. They share
# iai-callgrind's output directory, where it keeps the base's counts for
# the head to compare with.
IAI_CALLGRIND_HOME: ${{ github.workspace }}/target-iai
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
pull-request:
name: instruction-count gate
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Install Valgrind
run: sudo apt-get update && sudo apt-get install -y valgrind
- name: Toolchain
id: toolchain
# `rust-toolchain.toml` picks it; base and head build with the same one.
run: echo "version=$(rustc --version)" >> "$GITHUB_OUTPUT"
- name: Cache iai-callgrind-runner
id: runner-cache
uses: actions/cache@v6
with:
path: ~/.cargo/bin/iai-callgrind-runner
key: iai-callgrind-runner-${{ env.IAI_CALLGRIND_RUNNER_VERSION }}-${{ runner.os }}
- name: Install iai-callgrind-runner
if: steps.runner-cache.outputs.cache-hit != 'true'
run: cargo install iai-callgrind-runner --version "$IAI_CALLGRIND_RUNNER_VERSION" --locked
- name: Bench the base
id: base
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
git worktree add "$RUNNER_TEMP/base" "$BASE_SHA"
cd "$RUNNER_TEMP/base"
# The base's own targets; a base from before this workflow has none.
if ! python3 .github/scripts/bench-report.py --targets > "$RUNNER_TEMP/base-targets" 2>/dev/null; then
: > "$RUNNER_TEMP/base-targets"
fi
if [ ! -s "$RUNNER_TEMP/base-targets" ]; then
echo "The base ($BASE_SHA) has no instruction-count benches; recording head only."
echo "compare=false" >> "$GITHUB_OUTPUT"
exit 0
fi
mapfile -t targets < "$RUNNER_TEMP/base-targets"
for target in "${targets[@]}"; do
read -r package bench <<< "$target"
echo "::group::Base: $package --bench $bench"
CARGO_TARGET_DIR="$RUNNER_TEMP/base-target" cargo bench -p "$package" --bench "$bench" -- \
--save-baseline=base --save-summary=json
echo "::endgroup::"
done
# The base's own summaries, kept apart from the head's: they say
# which benchmarks the base had, so one the head drops or renames
# is noticed, and give a renamed one its old count.
mkdir -p "$RUNNER_TEMP/base-iai"
(cd "$IAI_CALLGRIND_HOME" && find . -name summary.json -exec cp --parents {} "$RUNNER_TEMP/base-iai" \;)
echo "compare=true" >> "$GITHUB_OUTPUT"
- name: Bench the head
env:
COMPARE: ${{ steps.base.outputs.compare }}
CARGO_TARGET_DIR: ${{ runner.temp }}/head-target
run: |
find "$IAI_CALLGRIND_HOME" -name summary.json -delete 2>/dev/null || true
# No --callgrind-limits: the gate is the Report step's, which knows
# the ledger. So this step fails only when a benchmark can't run.
listed=$(python3 .github/scripts/bench-report.py --targets)
mapfile -t targets <<< "$listed"
for target in "${targets[@]}"; do
read -r package bench <<< "$target"
echo "::group::Head: $package --bench $bench"
if [ "$COMPARE" = "true" ] && grep -qxF "$target" "$RUNNER_TEMP/base-targets"; then
cargo bench -p "$package" --bench "$bench" -- --baseline=base --save-summary=json
else
echo "The base doesn't have this target; recording head only."
cargo bench -p "$package" --bench "$bench" -- --save-summary=json
fi
echo "::endgroup::"
done
- name: Wall time (not gated)
env:
CARGO_TARGET_DIR: ${{ runner.temp }}/head-target
run: cargo bench -p comline-core --bench parser_benchmark
- name: Report and gate
env:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR: ${{ github.event.pull_request.number }}
TOOLCHAIN: ${{ steps.toolchain.outputs.version }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
# The base's ledger tells this pull request's entries from history;
# a base without one has none. The script's exit status is the
# verdict, given after the summary is written.
set +e
python3 .github/scripts/bench-report.py \
--home "$IAI_CALLGRIND_HOME" \
--criterion "$RUNNER_TEMP/head-target/criterion" \
--commit "$HEAD_SHA" \
--toolchain "$TOOLCHAIN" \
--run "$RUN_URL" \
--limit "$BENCH_LIMIT_PERCENT" \
--pr "$PR" \
--base-home "$RUNNER_TEMP/base-iai" \
--ledger "$LEDGER" \
--base-ledger "$RUNNER_TEMP/base/$LEDGER" \
--report reports/core.bench.json \
--markdown "$RUNNER_TEMP/bench.md"
verdict=$?
cat "$RUNNER_TEMP/bench.md" >> "$GITHUB_STEP_SUMMARY"
if [ "$verdict" != 0 ]; then
echo "::error::The instruction-count gate failed; see the job summary."
fi
exit "$verdict"
- uses: actions/upload-artifact@v7
if: always() && hashFiles('reports/core.bench.json') != ''
with:
name: core.bench.json
path: reports/core.bench.json

master:
name: publish the bench report
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
bench:
name: bench
# `contents: write` is the ceiling for the reusable workflow's publish job,
# the only one that asks for it, and it never runs on a pull request.
permissions:
contents: write
steps:
- uses: actions/checkout@v7
with:
# The publish checks the report is newer than the published one.
fetch-depth: 0
- name: Install Valgrind
run: sudo apt-get update && sudo apt-get install -y valgrind
- name: Toolchain
id: toolchain
run: echo "version=$(rustc --version)" >> "$GITHUB_OUTPUT"
- name: Cache iai-callgrind-runner
id: runner-cache
uses: actions/cache@v6
with:
path: ~/.cargo/bin/iai-callgrind-runner
key: iai-callgrind-runner-${{ env.IAI_CALLGRIND_RUNNER_VERSION }}-${{ runner.os }}
- name: Install iai-callgrind-runner
if: steps.runner-cache.outputs.cache-hit != 'true'
run: cargo install iai-callgrind-runner --version "$IAI_CALLGRIND_RUNNER_VERSION" --locked
- name: Instruction counts
run: |
listed=$(python3 .github/scripts/bench-report.py --targets)
mapfile -t targets <<< "$listed"
for target in "${targets[@]}"; do
read -r package bench <<< "$target"
echo "::group::$package --bench $bench"
cargo bench -p "$package" --bench "$bench" -- --save-summary=json
echo "::endgroup::"
done
- name: Wall time
run: cargo bench -p comline-core --bench parser_benchmark
- name: Report
env:
TOOLCHAIN: ${{ steps.toolchain.outputs.version }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
python3 .github/scripts/bench-report.py \
--home "$IAI_CALLGRIND_HOME" \
--criterion target/criterion \
--commit "$GITHUB_SHA" \
--toolchain "$TOOLCHAIN" \
--run "$RUN_URL" \
--report reports/core.bench.json \
--markdown "$RUNNER_TEMP/bench.md"
cat "$RUNNER_TEMP/bench.md" >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@v7
with:
name: core.bench.json
path: reports/core.bench.json
- name: Publish to the reports branch
uses: ComlineProject/observatory/.github/actions/publish-report@v1
with:
reports: reports/core.bench.json
readme: .github/reports-branch.md
dry-run: ${{ github.ref != 'refs/heads/master' }}
uses: ComlineProject/.github/.github/workflows/bench.yml@797d473553d7f2fc50895bb2573acb83ac63c7cb # main, no release tag yet
with:
image: ghcr.io/comlineproject/ci-bench@sha256:235555448c3c6027960cd95c0b2974014484e0c4f2808f064eb4cb3df91cebc0
ledger: core/benches/accepted-regressions.toml
report-path: reports/core.bench.json
# A benchmark fails when its instruction count grows by more than this
# many percent, unless the ledger accepts it (see the top). With one
# codegen unit (`[profile.bench]` in Cargo.toml) a change to one
# function leaves the others' counts alone.
limit-percent: '1'
wall-time-command: cargo bench -p comline-core --bench parser_benchmark
publish: true
default-branch: master
78 changes: 35 additions & 43 deletions .github/workflows/build_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,71 +7,63 @@ on:
permissions:
contents: read

# A newer push to the same pull request supersedes this run; a branch's runs
# queue instead.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
CARGO_TERM_COLOR: always

jobs:
build_and_test:
name: Rust project - latest
runs-on: ubuntu-latest
strategy:
matrix:
toolchain:
- stable
- beta
- nightly
steps:
- uses: actions/checkout@v3
- run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
- run: cargo build --verbose
- run: cargo test --verbose

# The toolchain matrix (stable, beta, nightly), the same build and tests on
# the other platforms (core#9: paths and line endings are where they're
# expected to differ), and a stable canary on ubuntu-26.04 that may fail
# without failing the run. The reusable workflow's own jobs set the runner,
# the timeouts and the pinned actions.
rust:
name: rust
permissions:
contents: read
uses: ComlineProject/.github/.github/workflows/rust-ci.yml@797d473553d7f2fc50895bb2573acb83ac63c7cb # main, no release tag yet
with:
# Dependency resolution is behind `deps` (the CLI turns it on), and so
# are its tests.
- run: cargo test --verbose -p comline-core --features deps
features: deps
features-package: comline-core

# The size vectors the sizer is tested against and the compiler cases the
# build is (core/tests/spec) are the specification's own, at the rev
# `comline-catalog` is pinned to.
spec_vectors:
name: Vendored specification files
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
timeout-minutes: 10
steps:
- uses: actions/checkout@v3
- uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- run: bash core/tests/spec/vendor.sh --check

# The IR alone (`default-features = false`): what a generator depends on.
# It has no parser, so unlike the default build it compiles for WASI and
# the browser; this keeps tree-sitter (and anything else that can't) out.
# In the wasm CI image (stable 1.99.1, which `rust-toolchain.toml` pins, and
# both wasm targets).
ir_only:
name: IR only (no default features)
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
timeout-minutes: 20
container: ghcr.io/comlineproject/ci-wasm@sha256:badd6af13cee6c459c9284de8ef0ef4a1667ed7e35582f87d1afb08f3c30fd8f
steps:
- uses: actions/checkout@v3
- run: rustup update stable && rustup default stable
- run: rustup target add wasm32-wasip1 wasm32-unknown-unknown
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- run: cargo build --verbose -p comline-core --no-default-features
- run: cargo test --verbose -p comline-core --no-default-features --lib
- run: cargo build --verbose -p comline-core --no-default-features --target wasm32-wasip1
- run: cargo build --verbose -p comline-core --no-default-features --target wasm32-unknown-unknown

# The same build and tests on the other platforms (core#9): paths and line
# endings are where they're expected to differ. Stable only; the toolchain
# matrix above stays Linux x64.
cross_platform:
name: test (${{ matrix.os }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- windows-latest
- macos-latest
- ubuntu-24.04-arm
steps:
- uses: actions/checkout@v7
- run: rustup update stable && rustup default stable
- uses: Swatinem/rust-cache@v2
- run: cargo build --verbose
- run: cargo test --verbose
- run: cargo test --verbose -p comline-core --features deps
Loading
Loading