Repository navigation
207 lines (197 loc) · 8.57 KB
/
Copy pathfuzz.yml
File metadata and controls
207 lines (197 loc) · 8.57 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
name: Fuzz
# The cargo-fuzz targets in runtime/fuzz, and the property tests (the
# conformance crate's, and the C ABI's differential test) at a high case
# count. A pull request runs each target for a minute, seeded from the
# conformance vectors. The nightly run gives each target ten minutes on every
# core of the runner (libFuzzer's fork mode, which merges what each process
# finds into the corpus) from a corpus cached between runs, runs proptest
# with 100000 cases, and opens an issue on a failure instead of blocking
# anyone's pull request.
# See runtime/fuzz/README.md.
#
# Every run writes `rust.fuzz.json` (runs, corpus, findings per target, and
# the property tests' verdicts; format in runtime/fuzz/README.md) as an
# artifact and to its summary, with the Observatory's shared `report-fuzz`
# action. A run on `main` (the nightly, or a manual one) also commits it to
# the `reports` branch, which the docs and the Observatory read, with its
# `publish-report` action. Only the `publish` job can write, and never on a
# pull request; other runs do a dry run of the publish.
#
# The fuzz jobs, the report and the publish are the org's reusable fuzz
# workflow (`fuzz` below), in the CI fuzz image. The property tests and the
# size tests are this repository's own jobs, in the same image; the property
# tests' verdicts reach the report as a `fuzz-stats-proptest` fragment, so the
# `fuzz` call waits for that job (the nightly runs it; a pull request doesn't).
on:
pull_request:
schedule:
- cron: "17 3 * * *"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# Container jobs default to `sh`; the steps are bash (arrays, mapfile, pipefail).
defaults:
run:
shell: bash
env:
CARGO_TERM_COLOR: always
jobs:
# The targets, from runtime/fuzz/Cargo.toml's [[bin]]s: the one list, which
# the reusable workflow's matrix and the report's `expect` read. Fails when
# the README's targets table differs.
targets:
name: fuzz targets
runs-on: ubuntu-24.04
timeout-minutes: 10
outputs:
list: ${{ steps.targets.outputs.list }}
steps:
- uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- id: targets
run: echo "list=$(.github/scripts/fuzz-targets.py)" >> "$GITHUB_OUTPUT"
fuzz:
name: fuzz
needs: [targets, properties]
# The property tests are skipped on a pull request, which would skip this.
if: ${{ !cancelled() && needs.targets.result == 'success' }}
# contents: write for the publish job, issues: write for the fuzz jobs;
# inside the reusable workflow each is on that job alone.
permissions:
contents: write
issues: write
uses: ComlineProject/.github/.github/workflows/fuzz.yml@807dd25bcbf848275f5f2bc5dcdd15070dcc321b # 2026-11-12
with:
image: ghcr.io/comlineproject/ci-fuzz@sha256:d18222e3007d318c73b1df500d997222ff6dca536b4b0e89e6ab7e55fc81326a
targets: ${{ needs.targets.outputs.list }}
fuzz-dir: runtime/fuzz
duration: 600
pr-duration: 60
crash-format: hex
report-path: runtime/conformance/reports/rust.fuzz.json
report-subject: rust
publish: true
properties:
name: proptest (100000 cases)
if: github.event_name != 'pull_request'
runs-on: ubuntu-24.04
timeout-minutes: 120
permissions:
contents: read
issues: write
container: ghcr.io/comlineproject/ci-fuzz@sha256:d18222e3007d318c73b1df500d997222ff6dca536b4b0e89e6ab7e55fc81326a
env:
# The image's nightly, over the root rust-toolchain.toml.
RUSTUP_TOOLCHAIN: nightly
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
mkdir -p fuzz-out/stats
- name: Property tests
id: properties
shell: bash
env:
PROPTEST_CASES: "100000"
# Both run, whichever fails, so the report has every verdict.
run: |
set -o pipefail
status=0
cargo test --release -p comline-runtime-conformance --test properties \
2>&1 | tee fuzz-out/proptest.log || status=1
cargo test --release -p comline-runtime-ffi --test differential the_abi_agrees_with_the_core \
2>&1 | tee -a fuzz-out/proptest.log || status=1
exit $status
- name: Record the run
if: always()
uses: ComlineProject/observatory/.github/actions/report-fuzz@6544a11bdfb9e4def0a64d6cff1cf8c3df464951 # v1
with:
mode: proptest
log: fuzz-out/proptest.log
cases: 100000
fragment: fuzz-out/stats/proptest.json
- if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: fuzz-stats-proptest
path: fuzz-out/stats
- name: Upload the failing seeds
if: failure()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: proptest-regressions
path: |
runtime/conformance/tests/properties.proptest-regressions
runtime/ffi/tests/differential.proptest-regressions
if-no-files-found: ignore
- name: Open an issue
if: failure()
env:
GH_TOKEN: ${{ github.token }}
TITLE: "Nightly property tests failed"
RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
open=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --limit 200 \
--json title --jq "map(select(.title == env.TITLE)) | length")
if [ "$open" != "0" ]; then
echo "An issue titled '$TITLE' is already open."
exit 0
fi
gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body \
"The nightly proptest run (PROPTEST_CASES=100000) failed: $RUN. The minimal failing input is in the job log; the seed is the run's \`proptest-regressions\` artifact."
# `sizes_vs_core`'s proptest twin, and the tiers too large for a vector
# (runtime/fuzz/tests/sizes.rs): core's byte ranges held to the real
# encoders. In release: the large values are megabytes. A pull request runs
# proptest's default count; the nightly run 20000 cases, and opens an issue
# on a failure.
sizes:
name: sizes (proptest)
runs-on: ubuntu-24.04
timeout-minutes: 120
permissions:
contents: read
issues: write
container: ghcr.io/comlineproject/ci-fuzz@sha256:d18222e3007d318c73b1df500d997222ff6dca536b4b0e89e6ab7e55fc81326a
env:
RUSTUP_TOOLCHAIN: nightly
NIGHTLY: ${{ github.event_name != 'pull_request' }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Size tests
env:
PROPTEST_CASES: ${{ github.event_name == 'pull_request' && '256' || '20000' }}
run: cargo test --release --manifest-path runtime/fuzz/Cargo.toml --test sizes
- name: Upload the failing seeds
if: failure()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: sizes-proptest-regressions
path: runtime/fuzz/tests/sizes.proptest-regressions
if-no-files-found: ignore
- name: Open an issue
if: failure() && env.NIGHTLY == 'true'
env:
GH_TOKEN: ${{ github.token }}
TITLE: "Nightly size tests failed"
RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
open=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --limit 200 \
--json title --jq "map(select(.title == env.TITLE)) | length")
if [ "$open" != "0" ]; then
echo "An issue titled '$TITLE' is already open."
exit 0
fi
gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body \
"The nightly size tests (runtime/fuzz/tests/sizes.rs, PROPTEST_CASES=20000) failed: $RUN. The minimal failing input is in the job log; the seed is the run's \`sizes-proptest-regressions\` artifact."