Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .machine_readable/hypatia-baseline.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
],
"properties": {
"severity": {
"description": "Severity of the finding as reported by Hypatia. Must match the finding's severity exactly for the entry to apply.",
"description": "Severity of the finding as reported by Hypatia. Must match the finding's severity for the entry to apply, except legacy `warn` and current `medium` are treated as equivalent by the gate.",
"type": "string",
"enum": ["critical", "high", "medium", "warn", "low", "info", "informational"]
},
Expand Down
6 changes: 3 additions & 3 deletions docs/HYPATIA-BASELINE-FORMAT.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,9 @@ Hypatia finding, plus optional metadata.
== Required fields

`severity`:: One of `critical`, `high`, `medium`, `warn`, `low`, `info`,
`informational` — the scanner's full vocabulary (`warn` is what the research
rules emit; it ranks with `medium`). Must
match the finding's reported severity exactly for the entry to apply.
`informational` — the scanner's full vocabulary. `warn` is retained for
legacy findings and baseline records; it is matched as the same tier as the
current `medium` output. Other severity labels must match exactly.

`rule_module`:: Hypatia rule module (e.g. `cicd_rules`, `code_safety`,
`migration_rules`) — **a string, or a list of strings**. A list matches when
Expand Down
78 changes: 68 additions & 10 deletions scripts/apply-baseline.sh
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,39 @@ echo "$BASELINE_JSON" | jq -e 'type == "array"' >/dev/null || {
SCHEMA_ERRORS="$(jq -r '
def known: ["severity","rule_module","type","file","file_pattern",
"severity_override","expires_at","note","tracking_issue"];
def sevs: ["critical","high","medium","low","info"];
# Baseline severity vocabulary mirrors both the scanner validator and the
# authoritative schema. `warn` is retained for old records; matching treats
# it as the same tier as current scanner `medium` output.
def sevs: ["critical","high","medium","warn","low","info","informational"];
# Overrides are a distinct schema contract: aliases are valid source
# severities, not valid override targets.
def override_sevs: ["critical","high","medium","low","info","advisory"];
[ to_entries[] | .key as $i | .value as $e |
if ($e|type) != "object" then "entry[\($i)]: not an object"
else (
(["severity","rule_module","type"][]
(["severity","type"][]
| select(($e[.]|type) != "string")
| "entry[\($i)]: required key \(.) missing or not a string"),
# `rule_module` is the one key that admits a LIST as well as a string
# (standards#966: one defect, two emitting modules, one acknowledgement).
# Validated separately rather than being dropped from the required set —
# a key that is merely absent from every check is not validated, it is
# unchecked.
(if ($e.rule_module|type) as $t | $t != "string" and $t != "array"
then "entry[\($i)]: required key rule_module missing or not a string/array"
else empty end),
(if ($e.rule_module|type) == "array" and ($e.rule_module|length) == 0
then "entry[\($i)]: rule_module list is empty — an entry that names no module matches nothing"
else empty end),
(if ($e.rule_module|type) == "array"
and (($e.rule_module | unique | length) != ($e.rule_module | length))
then "entry[\($i)]: rule_module list contains duplicate members"
else empty end),
(if ($e.rule_module|type) == "array"
then ($e.rule_module[]
| select(type != "string")
| "entry[\($i)]: rule_module list member is not a string: \(tojson)")
else empty end),
(if (($e|has("file")) == ($e|has("file_pattern")))
then "entry[\($i)]: exactly one of file / file_pattern is required"
else empty end),
Expand All @@ -97,6 +123,12 @@ SCHEMA_ERRORS="$(jq -r '
| "entry[\($i)]: unknown key \(.)"),
(if ($e.severity|type) == "string" and ((sevs|index($e.severity))|not)
then "entry[\($i)]: invalid severity \($e.severity)" else empty end),
(if ($e.rule_module|type) == "array"
then ($e.rule_module[]
| select(type == "string")
| select((test("^[a-z][a-z0-9_]*$"))|not)
| "entry[\($i)]: rule_module list member fails pattern: \(.)")
else empty end),
(if ($e.rule_module|type) == "string"
and (($e.rule_module|test("^[a-z][a-z0-9_]*$"))|not)
then "entry[\($i)]: rule_module fails pattern: \($e.rule_module)"
Expand All @@ -123,14 +155,17 @@ SCHEMA_ERRORS="$(jq -r '
then "entry[\($i)]: file_pattern must be a non-empty string"
else empty end),
(if ($e|has("severity_override"))
and (((sevs + ["advisory"])|index($e.severity_override))|not)
then "entry[\($i)]: invalid severity_override \($e.severity_override)"
and ((($e.severity_override|type) != "string")
or ((override_sevs|index($e.severity_override))|not))
then "entry[\($i)]: invalid severity_override \($e.severity_override|tojson)"
else empty end),
(if ($e|has("expires_at"))
and ((($e.expires_at|type) != "string")
or (($e.expires_at|test("^[0-9]{4}-[0-9]{2}-[0-9]{2}$"))|not))
then "entry[\($i)]: expires_at must be an ISO date (YYYY-MM-DD)"
else empty end),
(if ($e|has("note")) and (($e.note|type) != "string")
then "entry[\($i)]: note must be a string" else empty end),
(if ($e|has("tracking_issue"))
and ((($e.tracking_issue|type) != "string")
or (($e.tracking_issue|test("^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+#[0-9]+$"))|not))
Expand Down Expand Up @@ -180,12 +215,34 @@ ANNOTATED="$(jq -n \
# referencing `.file_pattern` there would error with "Cannot index
# string". Capture the entry pattern first, then reference $pat
# inside the test() regex argument.
# Hypatia emitted `warn` for advisory findings until hypatia#895
# (2026-10-01), which now emits them as `medium`. Normalize only this
# historical transition; all other severity labels remain distinct.
def sev_tier: if . == "warn" then "medium" else . end;
def match_entry(f):
f as $finding
| $baseline
| map(select(
.severity == $finding.severity
and .rule_module == $finding.rule_module
(.severity | sev_tier) == ($finding.severity | sev_tier)
# `rule_module` is a string OR a list of strings.
#
# ⚠ THE LIST FORM EXISTS BECAUSE ONE DEFECT CAN BE EMITTED BY TWO
# MODULES. Hypatia raises `invalid_actions_lock` from BOTH
# `workflow_audit` and `workflow_hardening` for a single desynced
# lockfile. Under exact string equality an acknowledgement could
# only ever name one of them, so the other stayed unsuppressed and
# went on blocking — and it did, on `main`. The symptom is
# especially misleading: the entry looks correct, the file matches,
# the severity matches, and the finding is still kept.
#
# Normalising to a list here rather than duplicating the entry keeps
# ONE acknowledgement per defect, which is what `expires_at` and
# `tracking_issue` are actually about. Two entries for one defect
# means two expiry dates for one decision.
and (
(.rule_module | if type == "array" then . else [.] end)
| any(. == $finding.rule_module)
)
and .type == $finding.type
and (
(.file? // null) == $finding.file
Expand Down Expand Up @@ -228,9 +285,8 @@ ANNOTATED="$(jq -n \
KEPT="$(jq '[.[] | select(.baseline_status != "acknowledged")]' <<<"$ANNOTATED")"
SUPPRESSED="$(jq '[.[] | select(.baseline_status == "acknowledged")]' <<<"$ANNOTATED")"

# Severity rank for blocking decision. `warn` and `informational` are treated
# as aliases of `medium` and `info` respectively (Hypatia's research rules use
# `warn` at medium rank). Unknown severities are treated as critical to fail-safe.
# Severity rank for blocking decision.
# Unknown severities are treated as critical to fail-safe.
rank() {
case "$1" in
critical) echo 5 ;;
Expand All @@ -245,9 +301,11 @@ rank() {

THRESHOLD_RANK="$(rank "$BLOCKING_THRESHOLD")"
MAX_KEPT_RANK=0
BLOCKING_COUNT=0 # kept findings at or above the threshold (the gate message names these, not all kept)
while IFS= read -r sev; do
r="$(rank "$sev")"
if (( r > MAX_KEPT_RANK )); then MAX_KEPT_RANK=$r; fi
if (( r >= THRESHOLD_RANK )); then BLOCKING_COUNT=$((BLOCKING_COUNT + 1)); fi
done < <(jq -r '.[].severity' <<<"$KEPT")

KEPT_COUNT="$(jq 'length' <<<"$KEPT")"
Expand Down Expand Up @@ -286,7 +344,7 @@ if [[ "$MODE" == "advisory" ]]; then
fi

if (( MAX_KEPT_RANK >= THRESHOLD_RANK )); then
echo "::error::Gate failed: $KEPT_COUNT unfiltered finding(s) at or above '$BLOCKING_THRESHOLD'." >&2
echo "::error::Gate failed: $BLOCKING_COUNT unfiltered finding(s) at or above '$BLOCKING_THRESHOLD' ($KEPT_COUNT kept in total)." >&2
exit 1
fi

Expand Down
134 changes: 130 additions & 4 deletions scripts/tests/apply-baseline-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,13 @@ trap 'rm -rf "$WORK"' EXIT

pass=0
fail=0
COUNTS_JQ='"\(.findings_suppressed | length),\(.findings_kept | length)"'

assert_status() {
local label="$1" findings="$2" baseline="$3" expected="$4"
local got
got=$(bash "$APPLY" "$findings" "$baseline" advisory \
| jq -r '"\(.findings_suppressed | length),\(.findings_kept | length)"')
| jq -r "$COUNTS_JQ")
if [ "$got" = "$expected" ]; then
echo "PASS: $label (suppressed,kept=$got)"
pass=$((pass + 1))
Expand Down Expand Up @@ -81,6 +82,131 @@ assert_status "medium acknowledgement matches a warn finding" \
assert_status "a different tier does not match" \
"$WORK/findings2b.json" "$WORK/baseline2b-high.json" "0,1"

# Exercise baseline mode too: aliases must suppress before thresholding, but
# only `warn`/`medium` are compatibility-equivalent. `info` and
# `informational` remain distinct emitted labels.
printf '[]\n' > "$WORK/empty.json"
assert_gate() {
local label="$1" findings="$2" baseline="$3" threshold="$4" expected_status="$5" expected_counts="$6"
local status=0 got
BLOCKING_THRESHOLD="$threshold" bash "$APPLY" "$findings" "$baseline" blocking \
>"$WORK/gate.out" 2>"$WORK/gate.err" || status=$?
got=$(jq -r "$COUNTS_JQ" "$WORK/gate.out")
if [[ "$status" -eq "$expected_status" && "$got" == "$expected_counts" ]]; then
echo "PASS: $label (exit=$status, suppressed/kept=$got)"
pass=$((pass + 1))
else
echo "FAIL: $label expected exit=$expected_status counts=$expected_counts; got exit=$status counts=$got"
cat "$WORK/gate.err"
fail=$((fail + 1))
fi
}

assert_gate "legacy warn ack suppresses current medium at blocking threshold" \
"$WORK/findings2b.json" "$WORK/baseline2b-warn.json" high 0 "1,0"
assert_gate "legacy medium ack suppresses warn at blocking threshold" \
"$WORK/findings2b-warn.json" "$WORK/baseline2b-medium.json" high 0 "1,0"
cat > "$WORK/findings-informational.json" <<'EOF'
[{"severity":"informational","rule_module":"research_extensions","type":"RE001","file":"ci.yml"}]
EOF
cat > "$WORK/baseline-info.json" <<'EOF'
[{"severity":"info","rule_module":"research_extensions","type":"RE001","file":"ci.yml"}]
EOF
cat > "$WORK/baseline-informational.json" <<'EOF'
[{"severity":"informational","rule_module":"research_extensions","type":"RE001","file":"ci.yml"}]
EOF
assert_status "informational baseline severity is accepted and matches exactly" \
"$WORK/findings-informational.json" "$WORK/baseline-informational.json" "1,0"
assert_gate "info and informational are not conflated" \
"$WORK/findings-informational.json" "$WORK/baseline-info.json" info 1 "0,1"
printf '%s\n' '[{"severity":"warn"}]' > "$WORK/findings-warn.json"
printf '%s\n' '[{"severity":"informational"}]' > "$WORK/findings-informational-only.json"
assert_gate "warn blocks at medium threshold" \
"$WORK/findings-warn.json" "$WORK/empty.json" medium 1 "0,1"
assert_gate "warn does not block at high threshold" \
"$WORK/findings-warn.json" "$WORK/empty.json" high 0 "0,1"
assert_gate "informational blocks at info threshold" \
"$WORK/findings-informational-only.json" "$WORK/empty.json" info 1 "0,1"
assert_gate "informational does not block at low threshold" \
"$WORK/findings-informational-only.json" "$WORK/empty.json" low 0 "0,1"

# Source severities and override targets are separate schema enums. A valid
# downgrade is carried through the output and then ranked for the gate.
cat > "$WORK/findings-override.json" <<'EOF'
[{"severity":"high","rule_module":"cicd_rules","type":"banned_language_file","file":"src/Legacy.kt"}]
EOF
cat > "$WORK/baseline-override-low.json" <<'EOF'
[{"severity":"high","rule_module":"cicd_rules","type":"banned_language_file","file":"src/Legacy.kt","severity_override":"low"}]
EOF
assert_gate "low override downgrades a matching high finding" \
"$WORK/findings-override.json" "$WORK/baseline-override-low.json" high 0 "0,1"
if [[ "$(jq -r '.findings_kept[0] | [.severity,.baseline_status] | join(",")' "$WORK/gate.out")" == "low,downgraded" ]]; then
echo "PASS: low override is visible as a downgraded finding"
pass=$((pass + 1))
else
echo "FAIL: low override was not preserved in output"
fail=$((fail + 1))
fi
cat > "$WORK/baseline-override-info.json" <<'EOF'
[{"severity":"high","rule_module":"cicd_rules","type":"banned_language_file","file":"src/Legacy.kt","severity_override":"info"}]
EOF
assert_gate "info override blocks at info but not low threshold" \
"$WORK/findings-override.json" "$WORK/baseline-override-info.json" info 1 "0,1"
assert_gate "info override remains below low threshold" \
"$WORK/findings-override.json" "$WORK/baseline-override-info.json" low 0 "0,1"
cat > "$WORK/baseline-override-advisory.json" <<'EOF'
[{"severity":"high","rule_module":"cicd_rules","type":"banned_language_file","file":"src/Legacy.kt","severity_override":"advisory"}]
EOF
assert_gate "advisory override does not block info threshold" \
"$WORK/findings-override.json" "$WORK/baseline-override-advisory.json" info 0 "0,1"
for override in warn informational 7; do
jq -n --arg override "$override" '[{severity:"high",rule_module:"cicd_rules",type:"banned_language_file",file:"src/Legacy.kt",severity_override:$override}]' > "$WORK/baseline-invalid-override.json"
status=0
bash "$APPLY" "$WORK/findings-override.json" "$WORK/baseline-invalid-override.json" advisory >/dev/null 2>&1 || status=$?
if [[ "$status" -eq 2 ]]; then
echo "PASS: invalid severity_override rejected with schema exit 2: $override"
pass=$((pass + 1))
else
echo "FAIL: severity_override $override returned $status (expected 2)"
fail=$((fail + 1))
fi
done

assert_invalid_baseline() {
local label="$1" content="$2" status=0
printf '%s\n' "$content" > "$WORK/baseline-invalid.json"
bash "$APPLY" "$WORK/findings-override.json" "$WORK/baseline-invalid.json" advisory >/dev/null 2>&1 || status=$?
if [[ "$status" -eq 2 ]]; then
echo "PASS: invalid baseline rejected with schema exit 2: $label"
pass=$((pass + 1))
else
echo "FAIL: invalid baseline $label returned $status (expected 2)"
fail=$((fail + 1))
fi
}
assert_invalid_baseline "duplicate rule_module array member" \
'[{"severity":"high","rule_module":["cicd_rules","cicd_rules"],"type":"banned_language_file","file":"src/Legacy.kt"}]'
assert_invalid_baseline "non-string note" \
'[{"severity":"high","rule_module":"cicd_rules","type":"banned_language_file","file":"src/Legacy.kt","note":1}]'
assert_invalid_baseline "unknown source severity" \
'[{"severity":"urgent","rule_module":"cicd_rules","type":"banned_language_file","file":"src/Legacy.kt"}]'

# The gate message counts only findings at/above the threshold, while still
# reporting how many total findings remain for context.
printf '%s\n' '[{"severity":"high"},{"severity":"info"}]' > "$WORK/findings-mixed.json"
echo '[]' > "$WORK/empty-baseline.json"
if BLOCKING_THRESHOLD=high bash "$APPLY" "$WORK/findings-mixed.json" "$WORK/empty-baseline.json" blocking >"$WORK/gate.out" 2>"$WORK/gate.err"; then
echo "FAIL: high finding did not block"
fail=$((fail + 1))
elif grep -Fq "1 unfiltered finding(s) at or above 'high' (2 kept in total)" "$WORK/gate.err"; then
echo "PASS: blocking diagnostic distinguishes threshold findings from total kept"
pass=$((pass + 1))
else
echo "FAIL: blocking diagnostic count is inaccurate"
cat "$WORK/gate.err"
fail=$((fail + 1))
fi

# === Case 3: file_pattern MUST NOT match unrelated file (regression
# against the always-matches bug from `.file_pattern` inside test()) ===
cat > "$WORK/findings3.json" <<'EOF'
Expand Down Expand Up @@ -146,7 +272,7 @@ done
assert_invalid_option() {
local label="$1" mode="$2" threshold="$3" status=0
BLOCKING_THRESHOLD="$threshold" bash "$APPLY" "$WORK/findings1.json" "$WORK/empty.json" "$mode" > "$WORK/invalid.out" 2> "$WORK/invalid.err" || status=$?
if [ "$status" -eq 2 ]; then
if [[ "$status" -eq 2 ]]; then
echo "PASS: $label rejected as invalid configuration"
pass=$((pass + 1))
else
Expand Down Expand Up @@ -236,7 +362,7 @@ elif ! bash -n "$MUTANT" 2>/dev/null; then
else
# Non-first member must now be MISSED.
mutant_got=$(bash "$MUTANT" "$WORK/findings-mod2only.json" "$WORK/baseline-2mod.json" advisory \
| jq -r '"\(.findings_suppressed | length),\(.findings_kept | length)"')
| jq -r "$COUNTS_JQ")
if [ "$mutant_got" = "0,1" ]; then
echo "PASS: MUTANT (first element only) misses the non-first module — any() IS load-bearing"
pass=$((pass + 1))
Expand All @@ -247,7 +373,7 @@ else
# And the two-module case must regress to half-suppressed: exactly the
# #966 symptom, reproduced on demand.
mutant_both=$(bash "$MUTANT" "$WORK/findings-2mod.json" "$WORK/baseline-2mod.json" advisory \
| jq -r '"\(.findings_suppressed | length),\(.findings_kept | length)"')
| jq -r "$COUNTS_JQ")
if [ "$mutant_both" = "1,1" ]; then
echo "PASS: MUTANT reproduces the #966 symptom (half-suppressed, entry looks correct)"
pass=$((pass + 1))
Expand Down
Loading
Loading