diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d51d297..8a54dd0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,19 @@ ruff check . CI runs both on every push and PR, on Python 3.10 and 3.12. +The text parsers (report severities, skill frontmatter, severity filters) +also have property-based tests in `tests/test_property_parsers.py` +(Hypothesis, run as part of `pytest tests`) and a coverage-guided fuzz +harness in `fuzz/fuzz_parsers.py`: + +```sh +pip install atheris +python fuzz/fuzz_parsers.py -max_total_time=60 +``` + +If you change one of those parsers, run the harness for a minute before +opening the PR. + ## Adding or changing a skill Skills live under `src/secfoo/skills/definitions/*.md` as a prompt diff --git a/fuzz/fuzz_parsers.py b/fuzz/fuzz_parsers.py new file mode 100644 index 0000000..10c932d --- /dev/null +++ b/fuzz/fuzz_parsers.py @@ -0,0 +1,60 @@ +"""Coverage-guided fuzzing of secfoo's text parsers with Atheris. + +The parsers here consume text written by an LLM (reports) or by hand (skill +definitions), so they see every shape of input. The properties checked are +the same ones tests/test_property_parsers.py asserts with Hypothesis; this +harness drives them with libFuzzer's coverage feedback instead of random +generation, which finds deeper paths. + +Run locally (Linux or macOS with clang): + + pip install atheris + python fuzz/fuzz_parsers.py -max_total_time=60 + +Any crash is written to the current directory as crash-; reproduce +with `python fuzz/fuzz_parsers.py crash-`. +""" + +from __future__ import annotations + +import sys + +import atheris + +with atheris.instrument_imports(): + from secfoo.report.severity import count_severities, extract_overall_risk_rating + from secfoo.skills.loader import SkillLoadError, _parse_frontmatter + from secfoo.web.finding_filters import normalize_severity + + +def test_one_input(data: bytes) -> None: + provider = atheris.FuzzedDataProvider(data) + text = provider.ConsumeUnicodeNoSurrogates(4096) + + # Report parsers must accept any text and never raise. + counts = count_severities(text) + assert counts.total == counts.critical + counts.high + counts.medium + counts.low + counts.info + rating = extract_overall_risk_rating(text) + assert rating is None or rating == rating.lower() + + # Frontmatter parsing may reject input, but only with SkillLoadError. + try: + fields, body = _parse_frontmatter(text, source="") + except SkillLoadError: + pass + else: + assert isinstance(fields, dict) + assert isinstance(body, str) + + # Severity normalisation is total and idempotent. + normalised = normalize_severity(text[:64]) + assert normalize_severity(normalised) == normalised + + +def main() -> None: + atheris.Setup(sys.argv, test_one_input) + atheris.Fuzz() + + +if __name__ == "__main__": + main()