From fed2f5ca272fad9022126cd37b1b751c0ab589c4 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Wed, 15 Jul 2026 14:22:38 +0200 Subject: [PATCH 01/15] Add pinned HTML API fuzz oracles --- .gitignore | 3 + run.sh | 11 + tools/html-api-fuzz/README.md | 467 ++++++ .../codex-triage-orchestrator.php | 935 +++++++++++ tools/html-api-fuzz/launcher.php | 362 +++++ tools/html-api-fuzz/lib/Corpus.php | 78 + tools/html-api-fuzz/lib/Generator.php | 1409 +++++++++++++++++ tools/html-api-fuzz/lib/HtmlApiBootstrap.php | 52 + tools/html-api-fuzz/lib/Mutator.php | 111 ++ tools/html-api-fuzz/lib/OracleFinding.php | 135 ++ tools/html-api-fuzz/lib/OracleRenderer.php | 917 +++++++++++ tools/html-api-fuzz/lib/Prng.php | 56 + tools/html-api-fuzz/lib/ResultStore.php | 406 +++++ tools/html-api-fuzz/lib/Signature.php | 142 ++ tools/html-api-fuzz/lib/Support.php | 493 ++++++ tools/html-api-fuzz/lib/TagInvariants.php | 437 +++++ tools/html-api-fuzz/lib/TreeRenderer.php | 715 +++++++++ tools/html-api-fuzz/lib/Worker.php | 601 +++++++ tools/html-api-fuzz/lib/autoload.php | 14 + tools/html-api-fuzz/lib/wp-stubs.php | 46 + tools/html-api-fuzz/minimize.php | 513 ++++++ tools/html-api-fuzz/oracles/chrome/README.md | 50 + tools/html-api-fuzz/oracles/chrome/VERSION | 1 + .../oracles/chrome/chrome-tree-oracle.js | 1015 ++++++++++++ tools/html-api-fuzz/oracles/chrome/install.sh | 80 + .../oracles/chrome/smoke-test.js | 187 +++ .../oracles/html5ever/Cargo.lock | 310 ++++ .../oracles/html5ever/Cargo.toml | 16 + .../html-api-fuzz/oracles/html5ever/README.md | 33 + .../html-api-fuzz/oracles/html5ever/build.sh | 45 + .../oracles/html5ever/install-rust.sh | 62 + .../oracles/html5ever/rust-toolchain.toml | 3 + .../html-api-fuzz/oracles/html5ever/smoke.sh | 135 ++ .../oracles/html5ever/src/main.rs | 502 ++++++ tools/html-api-fuzz/oracles/lexbor/COMMIT | 1 + tools/html-api-fuzz/oracles/lexbor/README.md | 66 + tools/html-api-fuzz/oracles/lexbor/build.sh | 66 + .../oracles/lexbor/lexbor-tree-oracle.c | 1243 +++++++++++++++ tools/html-api-fuzz/preflight.sh | 60 + tools/html-api-fuzz/replay.php | 156 ++ tools/html-api-fuzz/runner.php | 519 ++++++ tools/html-api-fuzz/setup-oracles.sh | 17 + .../start-continuous-run-tmux.sh | 193 +++ tools/html-api-fuzz/stop.php | 390 +++++ .../tests/codex-triage-orchestrator-smoke.php | 179 +++ .../tests/generator-policy-smoke.php | 1119 +++++++++++++ .../tests/lexbor-oracle-smoke.php | 141 ++ .../tests/result-store-smoke.php | 235 +++ .../tests/runner-retention-smoke.php | 232 +++ tools/html-api-fuzz/tests/stop-smoke.php | 531 +++++++ .../tree-renderer-normalization-smoke.php | 396 +++++ .../tests/watcher-sqlite-smoke.php | 206 +++ tools/html-api-fuzz/verify-preflight.php | 126 ++ tools/html-api-fuzz/watcher.php | 620 ++++++++ tools/html-api-fuzz/worker.php | 103 ++ 55 files changed, 16941 insertions(+) create mode 100755 run.sh create mode 100644 tools/html-api-fuzz/README.md create mode 100755 tools/html-api-fuzz/codex-triage-orchestrator.php create mode 100755 tools/html-api-fuzz/launcher.php create mode 100644 tools/html-api-fuzz/lib/Corpus.php create mode 100644 tools/html-api-fuzz/lib/Generator.php create mode 100644 tools/html-api-fuzz/lib/HtmlApiBootstrap.php create mode 100644 tools/html-api-fuzz/lib/Mutator.php create mode 100644 tools/html-api-fuzz/lib/OracleFinding.php create mode 100644 tools/html-api-fuzz/lib/OracleRenderer.php create mode 100644 tools/html-api-fuzz/lib/Prng.php create mode 100644 tools/html-api-fuzz/lib/ResultStore.php create mode 100644 tools/html-api-fuzz/lib/Signature.php create mode 100644 tools/html-api-fuzz/lib/Support.php create mode 100644 tools/html-api-fuzz/lib/TagInvariants.php create mode 100644 tools/html-api-fuzz/lib/TreeRenderer.php create mode 100644 tools/html-api-fuzz/lib/Worker.php create mode 100644 tools/html-api-fuzz/lib/autoload.php create mode 100644 tools/html-api-fuzz/lib/wp-stubs.php create mode 100755 tools/html-api-fuzz/minimize.php create mode 100644 tools/html-api-fuzz/oracles/chrome/README.md create mode 100644 tools/html-api-fuzz/oracles/chrome/VERSION create mode 100755 tools/html-api-fuzz/oracles/chrome/chrome-tree-oracle.js create mode 100755 tools/html-api-fuzz/oracles/chrome/install.sh create mode 100755 tools/html-api-fuzz/oracles/chrome/smoke-test.js create mode 100644 tools/html-api-fuzz/oracles/html5ever/Cargo.lock create mode 100644 tools/html-api-fuzz/oracles/html5ever/Cargo.toml create mode 100644 tools/html-api-fuzz/oracles/html5ever/README.md create mode 100755 tools/html-api-fuzz/oracles/html5ever/build.sh create mode 100755 tools/html-api-fuzz/oracles/html5ever/install-rust.sh create mode 100644 tools/html-api-fuzz/oracles/html5ever/rust-toolchain.toml create mode 100755 tools/html-api-fuzz/oracles/html5ever/smoke.sh create mode 100644 tools/html-api-fuzz/oracles/html5ever/src/main.rs create mode 100644 tools/html-api-fuzz/oracles/lexbor/COMMIT create mode 100644 tools/html-api-fuzz/oracles/lexbor/README.md create mode 100755 tools/html-api-fuzz/oracles/lexbor/build.sh create mode 100644 tools/html-api-fuzz/oracles/lexbor/lexbor-tree-oracle.c create mode 100755 tools/html-api-fuzz/preflight.sh create mode 100755 tools/html-api-fuzz/replay.php create mode 100755 tools/html-api-fuzz/runner.php create mode 100755 tools/html-api-fuzz/setup-oracles.sh create mode 100755 tools/html-api-fuzz/start-continuous-run-tmux.sh create mode 100644 tools/html-api-fuzz/stop.php create mode 100755 tools/html-api-fuzz/tests/codex-triage-orchestrator-smoke.php create mode 100644 tools/html-api-fuzz/tests/generator-policy-smoke.php create mode 100755 tools/html-api-fuzz/tests/lexbor-oracle-smoke.php create mode 100644 tools/html-api-fuzz/tests/result-store-smoke.php create mode 100644 tools/html-api-fuzz/tests/runner-retention-smoke.php create mode 100644 tools/html-api-fuzz/tests/stop-smoke.php create mode 100644 tools/html-api-fuzz/tests/tree-renderer-normalization-smoke.php create mode 100644 tools/html-api-fuzz/tests/watcher-sqlite-smoke.php create mode 100755 tools/html-api-fuzz/verify-preflight.php create mode 100755 tools/html-api-fuzz/watcher.php create mode 100755 tools/html-api-fuzz/worker.php diff --git a/.gitignore b/.gitignore index 15876fa47fee8..b3180b88db126 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,9 @@ wp-tests-config.php /packagehash.txt /.gutenberg-hash /artifacts +/tools/html-api-fuzz/oracles/lexbor/build +/tools/html-api-fuzz/oracles/html5ever/build +/tools/html-api-fuzz/oracles/chrome/.chrome-for-testing /setup.log /coverage diff --git a/run.sh b/run.sh new file mode 100755 index 0000000000000..06846c870214f --- /dev/null +++ b/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +cd "$REPO_ROOT" + +if [[ "${SETUP_ORACLES:-0}" == "1" ]]; then + tools/html-api-fuzz/setup-oracles.sh +fi + +exec tools/html-api-fuzz/preflight.sh diff --git a/tools/html-api-fuzz/README.md b/tools/html-api-fuzz/README.md new file mode 100644 index 0000000000000..730c7a847be13 --- /dev/null +++ b/tools/html-api-fuzz/README.md @@ -0,0 +1,467 @@ +# HTML API Fuzzer + +Fuzzer for the WordPress HTML API under `src/wp-includes/html-api`. It compares +`WP_HTML_Processor` against one of three pinned, independent tree-construction +oracles using an html5lib-style textual tree, and separately checks the API +invariants described below: + +- source-built Lexbor at the commit in `oracles/lexbor/COMMIT`; +- source-built html5ever 0.39.0 with a locked crate graph and Rust 1.88.0; +- Chrome for Testing 150.0.7871.114, driven directly through CDP with no + Playwright or npm dependency. + +`lexbor-source` is the default oracle. + +## Requirements + +- PHP 8.4+ with SQLite3 and the extensions required by WordPress trunk. +- Run from the repository root. +- Lexbor build: `git`, `cmake`, and a C compiler. +- html5ever build: `curl`; the setup script installs its pinned Rust toolchain + in the ignored repository cache. +- Chrome oracle: Node 20+ and `curl`/`unzip`. The pinned browser is installed + under the oracle directory's ignored cache. + +## Common Commands + +Build/install all three pinned oracles: + +```sh +tools/html-api-fuzz/setup-oracles.sh +``` + +Run one generated seed with the default pinned Lexbor oracle: + +```sh +php tools/html-api-fuzz/worker.php --seed 1 --output-dir artifacts/html-api-fuzz/seed-1 +``` + +Run one generated seed with a specific terminal payload policy: + +```sh +php tools/html-api-fuzz/worker.php --seed 1 --payload-policy valid-utf8 --output-dir artifacts/html-api-fuzz/seed-1 +``` + +Run a batch in worker subprocesses (seeds are batched into shared worker +processes, 25 per process by default; see `--batch-size`): + +```sh +php tools/html-api-fuzz/runner.php --max-seeds 100 --duration-seconds 60 +``` + +Run a structural UTF-8-biased batch with a post-generation byte cap: + +```sh +php tools/html-api-fuzz/runner.php --max-seeds 100 --payload-policy valid-utf8 --max-input-bytes 4096 +``` + +Build and run against the source-built Lexbor oracle: + +```sh +tools/html-api-fuzz/oracles/lexbor/build.sh +php tools/html-api-fuzz/worker.php --seed 1 --dom-oracle lexbor-source --output-dir artifacts/html-api-fuzz/seed-1-lexbor +php tools/html-api-fuzz/runner.php --max-seeds 100 --dom-oracle lexbor-source --duration-seconds 60 +``` + +Use `--lexbor-oracle-bin PATH` or `HTML_API_FUZZ_LEXBOR_ORACLE` when the +oracle binary is not at +`tools/html-api-fuzz/oracles/lexbor/build/lexbor-tree-oracle`. + +Build and run against html5ever: + +```sh +tools/html-api-fuzz/setup-oracles.sh +php tools/html-api-fuzz/worker.php --seed 1 --dom-oracle html5ever-source --output-dir artifacts/html-api-fuzz/seed-1-html5ever +``` + +Install and run against direct-CDP Chrome for Testing: + +```sh +tools/html-api-fuzz/oracles/chrome/install.sh +php tools/html-api-fuzz/worker.php --seed 1 --dom-oracle chrome-cdp --oracle-timeout-ms 15000 --output-dir artifacts/html-api-fuzz/seed-1-chrome +``` + +The Chrome runner starts one daemon per lane and shares it across isolated +worker batches through a local Unix socket. Full documents use base64 data +URLs with author scripts disabled and external network schemes blocked; +fragments use contextual `Range.createContextualFragment()` parsing. Use +`--chrome-executable PATH`, `--chrome-oracle-script PATH`, and `--node-bin +PATH` only for explicit compatible paths; the Chrome executable must still +report the tracked pinned version. See `oracles/chrome/README.md`. + +Run the bounded shared-seed preflight against all three oracles (100 seeds by +default): + +```sh +tools/html-api-fuzz/preflight.sh +# or: MAX_SEEDS=250 ./run.sh +``` + +The verifier requires identical `(seed,input_sha1)` rows, exact recorded pins, +and no oracle/worker infrastructure failures across the three result stores. + +Run indefinitely: + +```sh +php tools/html-api-fuzz/runner.php --duration-seconds 0 --max-seeds 0 +``` + +Run parallel lanes and triage failures after completion: + +```sh +php tools/html-api-fuzz/launcher.php --lanes 4 --max-seeds 1000 --watcher +``` + +For continuous fuzzing, run the launcher with `--duration-seconds 0 --max-seeds 0` +and run `watcher.php` in a second shell against the same output directory. + +Stop an indefinite run gracefully (each lane finishes its current batch, the +watcher performs a final scan, and the codex orchestrator drains its running +jobs): + +```sh +php tools/html-api-fuzz/stop.php --run-dir artifacts/html-api-fuzz/run-... +``` + +Without `--run-dir` the most recently active *unfinished* run under +`artifacts/html-api-fuzz` is targeted. Finished and stale runs are not +preferred; if nothing live is found, the most recent stopped-looking run is +targeted with a warning. The script creates the stop file advertised by the +run state and also `RUN_DIR/STOP` when a run directory is known, so watchers +and orchestrators see the stop request. For a standalone runner with custom +`--stop-file PATH`, both files are written. Relative custom stop files are +resolved with the runner cwd recorded in new runner state; for older state, +pass only `--stop-file PATH` to write a known stop file directly if needed. +With `--run-dir --stop-file PATH`, the explicit path is added to the run-state +and `RUN_DIR/STOP` targets. `touch` works just as well. The launcher and runner refuse to start while a stop file already +exists — remove `STOP` (or the custom stop file) before reusing a run +directory. (A stop requested in the sub-second window between the launcher's +startup check and a lane's own makes that lane refuse rather than stop +gracefully; the run still ends.) +If state cannot be read or old state lacks enough context to locate a relative +custom stop file, the tool still writes `RUN_DIR/STOP` but exits `2` with +`ok: false` and warnings because a standalone custom stop file may be unknown. + +The watcher exits after a final scan once every runner under the run +directory reports a stop reason. A runner whose state has gone silent is +presumed dead after `--stop-stale-seconds` (default 120); per lane that +threshold is floored at twice the lane's advertised batch budget +(`timeout-ms × batch-size`), so long batches are not mistaken for crashes. +The stop tool uses the same default stale threshold when auto-selecting the +latest unfinished run. + +Replay a failure from a retained seed directory, or from the lane's SQLite +store when the seed directory was pruned (see "Artifact Retention"): + +```sh +php tools/html-api-fuzz/replay.php --replay artifacts/html-api-fuzz/run-.../seed-.../primary/replay.json +php tools/html-api-fuzz/replay.php --store artifacts/html-api-fuzz/run-.../lane-00/results.sqlite --seed 12345 +``` + +Minimize a failure while preserving the same signature: + +```sh +php tools/html-api-fuzz/minimize.php --replay artifacts/html-api-fuzz/run-.../seed-.../primary/replay.json +``` + +By default (`--probe-mode auto`) the minimizer evaluates candidates in worker +subprocesses so `--timeout-ms` can kill pathological candidates and each probe +starts with fresh PHP state. Use `--probe-mode in-process` for faster +exact-signature minimization when that isolation is not needed; in-process +probes write only the final minimized artifacts unless +`--keep-candidate-artifacts` is also passed. Use `--probe-mode process` to +force subprocess probes explicitly. + +Watch an existing run directory and minimize new distinct signatures: + +```sh +php tools/html-api-fuzz/watcher.php --run-dir artifacts/html-api-fuzz/run-... --once +``` + +Configured ceilings are reported as `failureClass: "resource-limit"` and remain +in the watcher/minimizer triage path. This bucket includes tag/tree token +ceilings (`tag-token-limit-exceeded`, `mutation-token-limit-exceeded`, +`wordpress-token-limit-exceeded`) and oracle node ceilings +(`node-limit-exceeded`, recorded as `dom-node-limit-exceeded` in historical +signature facts). Process timeouts, PHP fatal errors, and memory failures are +separate failures and are also in scope for triage. + +## Execution Model + +The runner batches consecutive seeds into one worker process +(`worker.php --batch-count N`, default `--batch-size 25` on the runner) so the +WordPress bootstrap and process spawn are paid once per batch rather than once +per seed. Each seed still writes its own `seed-N/primary` artifacts. If a batch +process dies or times out mid-way, seeds left without a `result.json` are +re-run individually in isolation, so a crash on one input cannot take +neighboring seeds' results with it. + +## Input Stages + +Seeds are deterministically split between two input stages: + +- **Generated** (default ~80%): the structural grammar described under + “Generator Profiles”. +- **Corpus-mutated** (default ~20%, `--corpus-mutate-percent N` on + `worker.php`/`runner.php`): a `#data` section from the html5lib-tests + tree-construction corpus (`tests/phpunit/data/html5lib-tests`), passed + through 1–4 deterministic mutations (byte insert/replace, chunk + delete/duplicate, tag-name swap, case toggle, corpus splice). The stage, + corpus file, entry index, and operations are recorded in result metadata, + and the mutated input itself is in the replay manifest, so replays are + standalone. Inputs report `inputSource: "corpus-mutated"` and + `profile: "corpus-mutated"`. + +Both stages derive entirely from the seed, so seed N always produces the same +input for the same fuzzer version and corpus. + +## Artifact Layout + +The runner writes: + +- `results.sqlite`: one row per attempted seed (table `attempts`, WAL mode). + Passing attempts store summary columns only — every attempt is regenerable + from its seed. Failure rows additionally store the summary, result, and + replay JSON documents; the replay embeds the input as base64, so a pruned + failure can be reproduced with `replay.php --store results.sqlite --seed N`. + `signature_hash` and `family_key` are indexed columns for grouping + failures without `json_extract`. `oracle_kind`, `oracle_version`, + `oracle_commit`, and `oracle_binary` record which oracle generated the + summary, including for passing rows whose JSON payloads are pruned. The + watcher tails these stores + incrementally by row id. (`summary.ndjson` files from older runs are still + scanned.) Durability is `synchronous=NORMAL`: an OS crash (not a process + crash) can lose the last moments of a run. +- `events.ndjson`: runner lifecycle events, including batch boundaries. +- `logs/batch-N.log`: output of a batch worker process, kept only when the + batch contained a retained failure or a seed that needed an isolated + re-run — over-cap repeats of a known signature do not accumulate batch + logs. +- `state.json`: aggregate counters, stop reason, and compact Git metadata. + Oracle losses are counted per class: `oracleParseErrors` (inputs the + selected oracle rejects receive no differential coverage) and + `oracleUnsupported` (tree shapes the oracle cannot represent). + `oracleTolerated` remains in the state schema for compatibility with older + runs; current trunk comparisons do not apply the old scalar tolerance. +- `seed-N/primary/input.bin`: raw generated bytes. +- `seed-N/primary/replay.json`: base64 replay manifest, including the commit + hash, tracked-file dirty state, selected oracle, and fragment context needed + to interpret a standalone replay. +- `seed-N/primary/result.json`: full worker result. +- `seed-N/primary/wordpress-tree.txt` and `dom-tree.txt`: rendered trees when available. + +### Artifact Retention + +Seed directories are working space, not the archive. After each seed is +recorded in `results.sqlite`, its `seed-N` directory is deleted unless the +attempt failed *and* the failure's signature has fewer than +`--max-keep-per-signature` (default 5, minimum 1) exemplar directories still +on disk in this lane. The cap counts directories, not rows, so restarting a +runner against the same output directory neither double-counts re-recorded +seeds nor deletes a previously retained exemplar. The first exemplar of every +new signature is always retained, so the watcher's minimization path keeps a +replay file to work from; subsequent repeats of a known signature add a +database row and nothing else. A failure whose replay document is missing +(worker killed before writing it) always keeps its directory — the files are +the only reproduction. Disk growth is therefore proportional to *new +distinct failures*, not to seeds executed. + +The cap applies per lane: the launcher passes the same value to every lane, +so a signature that appears in all lanes keeps up to `N × lanes` directories +across the run. Use `--keep-all-artifacts` (runner or launcher) to keep every +seed directory for debugging. + +The run-level Git metadata intentionally stays compact: full and short commit +hash, current branch when available, commit date, and a dirty flag for +tracked-file changes. The dirty flag is tri-state: `true`, `false`, or `null` +when Git is unavailable or dirty detection fails. Full `git status` or diff +output is not recorded because it is noisy, can expose local edits, and grows +indefinitely in long runs. Launcher and runner processes collect this metadata +once and pass it to workers so long runs do not invoke Git for every seed. +Replayed and minimized manifests keep current checkout metadata at the top level +and preserve discovery provenance in `sourceReplay`. + +The watcher writes triage state under `.triage-watcher` by default, or under +`--state-dir` when provided. Each signature gets a stable directory containing +`failure.json`, minimizer logs, and minimized replay/result artifacts. Failed +minimizations are retried on later scans, up to `--max-minimize-retries` +(default 3) attempts per signature. + +## Modes and Fragment Contexts + +- `fragment-body`: parse as a fragment. The selected oracle uses real fragment + parsing (the `innerHTML` setter on a context element of an empty document), + not a document-wrapping approximation. +- `full-document`: parse as a full HTML document. +- `auto`: weighted choice. + +In fragment mode a context element is selected per seed +(`--fragment-context TAG` on `worker.php` for replays). `` dominates; +the other contexts (`div`, `p`, `td`, `tr`, `table`, `caption`, `colgroup`, +`select`, `option`, `template`, `title`, `textarea`, `script`, `style`, +`svg`, `math`) receive a small probe weight. `WP_HTML_Processor::create_fragment()` +currently supports only ``, so non-body contexts are recorded as +`status: "unsupported"` today; when create_fragment() gains context support +the fuzzer picks up the new coverage with no changes. The selected oracle already +parses every context correctly. + +Unsupported `WP_HTML_Processor` cases are expected by default and are recorded +as successful attempts with `status: "unsupported"`. Use `--fail-unsupported` +when you want unsupported cases to become failures. + +## Invariants + +Each seed checks, in order, stopping at the first failing class: + +1. **Tag Processor invariants** (`tag-invariant-failed`): token loop + termination under the token ceiling; non-null token type/name/tag; + attribute getters and `class_list()` iteration do not throw; + `get_updated_html()` with no queued edits returns the input unchanged; a + simple `set_attribute()` mutation is visible to a re-scan; and + **seek consistency** — a bookmark set at a seed-chosen token, after + scanning to the end and seeking back, must reproduce the identical token + stream (`seek-token-stream-mismatch`). +2. **Differential tree comparison** (`tree-mismatch` / `encoding-mismatch`): + the WordPress tree must equal the selected oracle tree (see “Tree Comparison”). +3. **Breadcrumb consistency** (`breadcrumb-mismatch`): at every tag token, + `get_breadcrumbs()` must agree with the element stack derived from token + order and `expects_closer()`. +4. **Mutation differential** (`mutation-tree-mismatch` / + `mutation-delta-mismatch`), only on a clean baseline: after setting + `data-fuzz="1"` on the first tag, the mutated document must parse + identically in WordPress and the selected oracle, and the WordPress tree must + change by exactly the one attribute line (unless formatting-element + reconstruction clones the attribute, or tree construction legitimately + drops the mutated element, in which case the differential comparison alone + applies). +5. **Normalize tree preservation** (`normalize-tree-changed`), only on a + clean baseline: parsing `normalize()` output must produce the same tree as + the original input, modulo the documented scalar substitutions. This is + stricter than idempotence, which a consistently wrong serializer can pass. +6. **Normalize idempotence** (`normalize-invariant-failed`): + `normalize()` / `serialize()` run twice must be a fixed point, with no + PHP native errors or throwables. Full documents use + `create_full_parser()->serialize()`; non-body fragment contexts use + `create_fragment()->serialize()`. + +### Known invariant oracle follow-ups + +- The simple `set_attribute()` mutation oracle needs to handle inputs that + begin with `
` using the same tag-selection semantics as the mutator. + `next_tag()` skips the raw closing token and mutates the following tag, + while a verifier that scans with `next_token()` can see the spec-special + `BR` element synthesized from `
` first and incorrectly report + `mutation-attribute-missing`. Fix the verifier by selecting the first + mutable tag through `next_tag()` too. + +## Generator Profiles + +The generator uses a structural HTML grammar with weighted profiles: + +- `balanced` +- `full-document` (includes occasional frameset documents, quirks-mode + doctypes, and content after ``) +- `body-fragment` +- `tables` +- `template` +- `select` (option/optgroup nesting, select-ending elements such as `input`, + `textarea` and `button`, nested selects, select-in-table) +- `foreign-content` (MathML/SVG integration points, HTML breakout tags, + `` with and without breakout attributes, `annotation-xml` encoding + variants, CDATA sections in foreign content, case-mangled `foreignObject`) +- `rawtext-rcdata` (script/style/iframe/noembed/noframes/xmp/noscript, + title/textarea, occasional `plaintext`) +- `text-fragment` (standalone terminal payloads, biased toward exact + 0-10 byte inputs plus medium syntax-heavy text unless `stress-long` is + selected explicitly) +- `formatting-adoption` (random formatting elements plus explicit + adoption-agency shapes: misnested closers, block-boundary formatting, + reconstruction across siblings, nested anchors, Noah's Ark overflow, + repeated closers, and four-plus active-formatting reconstruction clusters + covering same tag/no attributes, same tag/distinct attributes, same + tag/matching attributes, mixed interleavings, and table-cell marker + isolation/clearing) +- `attributes-entities` +- `comments-doctype-bogus` +- `deep-nesting` +- `resource-stress` +- `incomplete-malformed` (includes spec-special closers such as `
` and + `

`, stray closers, and ``) + +All profiles can emit duplicate attribute names (first-wins coverage), +auto-closing chains (`li`, `dd`/`dt`, headings, `p`), and named character +references with longest-prefix-match ambiguity (`¬it;`, `©right;`, +`&ngE`, ...). + +Terminal payloads are selected by a separate policy: + +- `valid-utf8`: structural cases with ASCII, Unicode, controls, and + entity references, including NUL byte coverage, but no raw invalid bytes. +- `mostly-valid`: default-biased structural cases with valid UTF-8 Unicode, + controls, NUL bytes, and entity references. +- `ascii-structural`: ASCII-only terminal text and attributes for tokenizer and + tree-construction coverage, including NUL byte coverage. +- `stress-long`: long valid UTF-8 terminal payloads for deliberate + resource-stress runs. +- `auto`: weighted choice. Normal structural profiles favor valid UTF-8 and + mostly-valid payloads; `resource-stress` favors `stress-long`. + +Use `--payload-policy POLICY` on `worker.php`, `runner.php`, or `launcher.php`. +Use `--max-input-bytes N` to apply a post-generation byte cap before the worker +records replay metadata. The cap preserves UTF-8 byte boundaries, but it is not +grammar-aware and may cut through HTML tokens. Replay manifests and minimization +summaries preserve the original payload policy when it was recorded; old or +hand-supplied inputs leave `payloadPolicy` null unless an explicit policy label +is provided. Historical `invalid-byte-heavy` labels are accepted only as replay +metadata for direct inputs and are not selectable for generated runs. +Replayed and minimized manifests keep immediate `inputSource` metadata separate +from `originalGenerator` metadata. + +## Tree Comparison + +The tree renderer follows the html5lib test style used by +`tests/phpunit/tests/html-api/wpHtmlProcessorHtml5lib.php`: + +- attributes sorted by their spec-scrubbed names (so a raw-NUL name on the + WordPress side and its U+FFFD substitution on the oracle side sort identically), + rendered raw +- boolean attributes rendered as `=""` +- namespace-qualified element and attribute names +- template `content` marker +- only the narrow auto-generated `html/head/body` wrapper tolerance + +Trees are rendered without comparison-time scalar normalization. Current +trunk applies HTML input preprocessing at its read interfaces, so NUL is +exposed as U+FFFD and CR/CRLF as LF just as it is in the independent oracles. +A raw-byte difference is therefore reported as a finding; the older fuzz +branch's NUL/CR scalar tolerance is intentionally not carried forward. + +Invalid bytes are never normalized away. If WordPress and the selected oracle +surface different byte sequences, the first-difference record includes bounded +line previews, byte lengths, line hashes, the first differing byte offset, and +hex previews, including a diff-window hex preview around the differing byte, so +the mismatch remains inspectable even when JSON display substitutes replacement +characters. Full comparison lines are kept out of `result.json` to avoid large +artifacts from stress inputs. + +### Known divergence + +- **CDATA at SVG/MathML integration points is a real WordPress divergence + the fuzzer will keep reporting.** For + `a\0b` (likewise ``, + ``, HTML-encoded `annotation-xml`), WordPress substitutes + NUL with U+FFFD in CDATA text while the spec routes those characters + through the HTML insertion mode, which drops NUL — WordPress handles + plain text at integration points correctly; only the CDATA path diverges + (a known `@todo` in `WP_HTML_Tag_Processor`'s CDATA handling). The shape + is U+FFFD-versus-removed, which no tolerance covers in either direction, + so these report as genuine `tree-mismatch` findings. This is distinct + from older upstream parser integration-point reparenting bugs. + +## Minimization + +`minimize.php` reduces in three phases under a shared attempt budget +(`--max-attempts`, default 600): markup-aligned segment deletion, binary +byte-chunk deletion, then per-byte deletion and canonicalization (replacements +never grow the input). Every accepted candidate re-runs the worker and must +reproduce the original signature hash (or any failure with `--any-failure`). diff --git a/tools/html-api-fuzz/codex-triage-orchestrator.php b/tools/html-api-fuzz/codex-triage-orchestrator.php new file mode 100755 index 0000000000000..17a62d6d6ea93 --- /dev/null +++ b/tools/html-api-fuzz/codex-triage-orchestrator.php @@ -0,0 +1,935 @@ +#!/usr/bin/env php + $value ) { + if ( '_' === $name ) { + if ( ! empty( $value ) ) { + throw new InvalidArgumentException( 'Unexpected positional argument: ' . (string) reset( $value ) ); + } + continue; + } + + if ( ! in_array( $name, $allowed, true ) ) { + throw new InvalidArgumentException( 'Unknown option: --' . $name ); + } + if ( in_array( $name, $value_options, true ) && true === $value ) { + throw new InvalidArgumentException( 'Expected --' . $name . ' to have a value.' ); + } + if ( in_array( $name, $bool_options, true ) && true !== $value ) { + throw new InvalidArgumentException( 'Option --' . $name . ' does not accept a value.' ); + } + } +} + +function html_api_fuzz_codex_is_absolute_path( string $path ): bool { + return 1 === preg_match( '#^(?:/|[A-Za-z]:[\\\\/]|\\\\\\\\)#', $path ); +} + +function html_api_fuzz_codex_now_iso_z(): string { + return gmdate( 'Y-m-d\TH:i:s\Z' ); +} + +function html_api_fuzz_codex_normalize_path( string $path ): string { + if ( '' === $path ) { + throw new InvalidArgumentException( 'Expected a non-empty path.' ); + } + + if ( ! html_api_fuzz_codex_is_absolute_path( $path ) ) { + $cwd = getcwd(); + if ( false === $cwd ) { + throw new RuntimeException( 'Could not determine current working directory.' ); + } + $path = $cwd . DIRECTORY_SEPARATOR . $path; + } + + $real = realpath( $path ); + return false === $real ? rtrim( $path, DIRECTORY_SEPARATOR ) : $real; +} + +function html_api_fuzz_codex_require_dir( string $path, string $label ): string { + $normalized = html_api_fuzz_codex_normalize_path( $path ); + if ( ! is_dir( $normalized ) ) { + throw new InvalidArgumentException( "{$label} is not a directory: {$normalized}" ); + } + + return $normalized; +} + +function html_api_fuzz_codex_path_join( string $base, string ...$parts ): string { + $path = rtrim( $base, DIRECTORY_SEPARATOR ); + foreach ( $parts as $part ) { + $path .= DIRECTORY_SEPARATOR . ltrim( $part, DIRECTORY_SEPARATOR ); + } + + return $path; +} + +function html_api_fuzz_codex_signature_dir_name( string $signature_hash ): string { + $dir = preg_replace( '/[^a-zA-Z0-9._-]+/', '_', $signature_hash ); + return null === $dir || '' === $dir ? '_' : $dir; +} + +function html_api_fuzz_codex_valid_signature_hash( string $signature_hash ): bool { + return 1 === preg_match( '/^[A-Za-z0-9._-]+$/', $signature_hash ); +} + +function html_api_fuzz_codex_safe_prompt_line( $value, string $field ): string { + if ( null === $value ) { + return ''; + } + if ( ! is_scalar( $value ) ) { + throw new InvalidArgumentException( "Expected {$field} to be scalar." ); + } + + $text = (string) $value; + if ( 1 === preg_match( '/[\x00-\x1F\x7F]/', $text ) ) { + throw new InvalidArgumentException( "Unsafe control character in {$field}." ); + } + + return $text; +} + +function html_api_fuzz_codex_path_has_control_chars( string $path ): bool { + return 1 === preg_match( '/[\x00-\x1F\x7F]/', $path ); +} + +function html_api_fuzz_codex_path_is_under( string $path, string $root ): bool { + $real_path = realpath( $path ); + $real_root = realpath( $root ); + if ( false === $real_path || false === $real_root ) { + return false; + } + + $real_root = rtrim( $real_root, DIRECTORY_SEPARATOR ); + return $real_path === $real_root || 0 === strpos( $real_path, $real_root . DIRECTORY_SEPARATOR ); +} + +function html_api_fuzz_codex_allowed_artifact_roots( array $args ): array { + $roots = array( + $args['triageDir'], + $args['runDir'] ?? dirname( $args['triageDir'] ), + ); + $allowed = array(); + foreach ( $roots as $root ) { + if ( is_string( $root ) && '' !== $root && is_dir( $root ) && ! html_api_fuzz_codex_path_has_control_chars( $root ) ) { + $real = realpath( $root ); + if ( false !== $real ) { + $allowed[] = $real; + } + } + } + + return array_values( array_unique( $allowed ) ); +} + +function html_api_fuzz_codex_sort_json_value( $value ) { + if ( ! is_array( $value ) ) { + return $value; + } + + if ( array_is_list( $value ) ) { + return array_map( 'html_api_fuzz_codex_sort_json_value', $value ); + } + + foreach ( $value as $key => $item ) { + $value[ $key ] = html_api_fuzz_codex_sort_json_value( $item ); + } + ksort( $value, SORT_STRING ); + + return $value; +} + +function html_api_fuzz_codex_json_encode( $value ): string { + $json = json_encode( + html_api_fuzz_codex_sort_json_value( $value ), + JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE + ); + if ( false === $json ) { + throw new RuntimeException( 'JSON encode failed: ' . json_last_error_msg() ); + } + + return $json; +} + +function html_api_fuzz_codex_read_json( string $path, int $retries = 5 ) { + $last_error = null; + for ( $i = 0; $i < $retries; ++$i ) { + if ( ! file_exists( $path ) ) { + return null; + } + + $text = @file_get_contents( $path ); + if ( false === $text ) { + throw new RuntimeException( "Could not read JSON file: {$path}" ); + } + + $value = json_decode( $text, true ); + if ( JSON_ERROR_NONE === json_last_error() ) { + return $value; + } + + $last_error = json_last_error_msg(); + usleep( 200000 ); + } + + throw new RuntimeException( "Could not parse JSON {$path}: {$last_error}" ); +} + +function html_api_fuzz_codex_exit_code_from_done( $done ): int { + if ( ! is_array( $done ) || ! array_key_exists( 'exitCode', $done ) || ! is_numeric( $done['exitCode'] ) ) { + return 1; + } + + return (int) $done['exitCode']; +} + +function html_api_fuzz_codex_write_json( string $path, $value ): void { + $dir = dirname( $path ); + \HtmlApiFuzz\ensure_dir( $dir ); + $tmp = tempnam( $dir, basename( $path ) . '.tmp.' ); + if ( false === $tmp ) { + throw new RuntimeException( "Could not create temporary JSON file for: {$path}" ); + } + $json = html_api_fuzz_codex_json_encode( $value ) . "\n"; + if ( false === file_put_contents( $tmp, $json ) ) { + @unlink( $tmp ); + throw new RuntimeException( "Could not write temporary JSON file: {$tmp}" ); + } + if ( ! rename( $tmp, $path ) ) { + @unlink( $tmp ); + throw new RuntimeException( "Could not replace JSON file: {$path}" ); + } +} + +function html_api_fuzz_codex_archive_path( string $dir, string $prefix ): string { + $suffix = (string) time(); + $path = html_api_fuzz_codex_path_join( $dir, "{$prefix}.{$suffix}.json" ); + for ( $i = 1; file_exists( $path ); ++$i ) { + $path = html_api_fuzz_codex_path_join( $dir, "{$prefix}.{$suffix}.{$i}.json" ); + } + + return $path; +} + +function html_api_fuzz_codex_process_alive( int $pid ): bool { + if ( $pid < 1 || ! function_exists( 'posix_kill' ) ) { + return false; + } + + if ( @posix_kill( $pid, 0 ) ) { + return true; + } + + if ( function_exists( 'posix_get_last_error' ) && defined( 'POSIX_EPERM' ) ) { + return POSIX_EPERM === posix_get_last_error(); + } + + return false; +} + +function html_api_fuzz_codex_newest_minimize_dir( array $signature ): ?string { + $output_dir = $signature['minimizeOutputDir'] ?? null; + if ( is_string( $output_dir ) && '' !== $output_dir && is_dir( $output_dir ) ) { + return $output_dir; + } + + $result_path = $signature['minimizeResult'] ?? null; + if ( is_string( $result_path ) && '' !== $result_path && is_file( $result_path ) ) { + return dirname( $result_path ); + } + + return null; +} + +function html_api_fuzz_codex_claim_signature( string $diagnostics_dir, string $signature_hash, array $signature, int $stale_after_seconds ): ?string { + $signature_dir = html_api_fuzz_codex_path_join( $diagnostics_dir, html_api_fuzz_codex_signature_dir_name( $signature_hash ) ); + $done_path = html_api_fuzz_codex_path_join( $signature_dir, 'done.json' ); + $claim_path = html_api_fuzz_codex_path_join( $signature_dir, 'claim.json' ); + + if ( is_file( $done_path ) ) { + $done = html_api_fuzz_codex_read_json( $done_path ); + $exit_code = html_api_fuzz_codex_exit_code_from_done( $done ); + if ( 0 === $exit_code ) { + return null; + } + + if ( ! rename( $done_path, html_api_fuzz_codex_archive_path( $signature_dir, 'done.failed' ) ) && is_file( $done_path ) ) { + throw new RuntimeException( "Could not archive failed done file: {$done_path}" ); + } + if ( is_file( $claim_path ) && ! rename( $claim_path, html_api_fuzz_codex_archive_path( $signature_dir, 'claim.failed' ) ) && is_file( $claim_path ) ) { + throw new RuntimeException( "Could not archive failed claim file: {$claim_path}" ); + } + } + + if ( is_file( $claim_path ) ) { + $claim = html_api_fuzz_codex_read_json( $claim_path ) ?: array(); + $pid = (int) ( $claim['pid'] ?? 0 ); + $claimed_at = (float) ( $claim['claimedAtUnix'] ?? 0 ); + $stale = microtime( true ) - $claimed_at > $stale_after_seconds; + if ( ! $stale ) { + if ( $pid > 0 && html_api_fuzz_codex_process_alive( $pid ) ) { + return null; + } + return null; + } + if ( ! rename( $claim_path, html_api_fuzz_codex_archive_path( $signature_dir, 'claim.stale' ) ) && is_file( $claim_path ) ) { + throw new RuntimeException( "Could not archive stale claim file: {$claim_path}" ); + } + } + + \HtmlApiFuzz\ensure_dir( $signature_dir ); + $claim = array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-codex-claim', + 'hash' => $signature_hash, + 'pid' => getmypid(), + 'claimedAt' => html_api_fuzz_codex_now_iso_z(), + 'claimedAtUnix' => microtime( true ), + 'failureClass' => $signature['failureClass'] ?? null, + 'sourceReplayPath' => $signature['replayPath'] ?? null, + 'sourceResultPath' => $signature['resultPath'] ?? null, + ); + + $handle = @fopen( $claim_path, 'xb' ); + if ( false === $handle ) { + if ( is_file( $claim_path ) ) { + return null; + } + throw new RuntimeException( "Could not create claim file: {$claim_path}" ); + } + + $bytes = html_api_fuzz_codex_json_encode( $claim ) . "\n"; + $write = fwrite( $handle, $bytes ); + fclose( $handle ); + if ( false === $write || $write !== strlen( $bytes ) ) { + throw new RuntimeException( "Could not write claim file: {$claim_path}" ); + } + + return $signature_dir; +} + +function html_api_fuzz_codex_existing_paths( array $paths, array $allowed_roots ): array { + $existing = array(); + foreach ( $paths as $path ) { + if ( ! is_string( $path ) || '' === $path || html_api_fuzz_codex_path_has_control_chars( $path ) || ! file_exists( $path ) ) { + continue; + } + + foreach ( $allowed_roots as $root ) { + if ( html_api_fuzz_codex_path_is_under( $path, $root ) ) { + $real = realpath( $path ); + if ( false !== $real ) { + $existing[] = $real; + } + continue 2; + } + } + } + + return array_values( array_unique( $existing ) ); +} + +function html_api_fuzz_codex_prompt_for_signature( array $args, string $signature_hash, array $signature ): string { + if ( ! html_api_fuzz_codex_valid_signature_hash( $signature_hash ) ) { + throw new InvalidArgumentException( 'Unsafe signature hash: ' . $signature_hash ); + } + + $signature_dir_name = html_api_fuzz_codex_signature_dir_name( $signature_hash ); + $signature_triage_dir = html_api_fuzz_codex_path_join( $args['triageDir'], 'signatures', $signature_dir_name ); + $minimize_dir = html_api_fuzz_codex_newest_minimize_dir( $signature ); + $minimized_dir = null === $minimize_dir ? null : html_api_fuzz_codex_path_join( $minimize_dir, 'minimized' ); + $report_path = html_api_fuzz_codex_path_join( $args['diagnosticsDir'], $signature_dir_name, 'report.md' ); + + $paths = array( + html_api_fuzz_codex_path_join( $signature_triage_dir, 'failure.json' ), + is_string( $signature['minimizeResult'] ?? null ) ? $signature['minimizeResult'] : null, + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'result.json' ), + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'replay.json' ), + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'wordpress-tree.txt' ), + null === $minimized_dir ? null : html_api_fuzz_codex_path_join( $minimized_dir, 'dom-tree.txt' ), + is_string( $signature['replayPath'] ?? null ) ? $signature['replayPath'] : null, + is_string( $signature['resultPath'] ?? null ) ? $signature['resultPath'] : null, + ); + $existing_paths = html_api_fuzz_codex_existing_paths( $paths, html_api_fuzz_codex_allowed_artifact_roots( $args ) ); + + if ( 'classify' === $args['mode'] ) { + $fix_policy = 'This is classify-only mode. Do not edit files. Do not run write commands. Return the Markdown report as your final response; the Codex CLI will save it. If you believe a fix is needed, describe the smallest fix and the focused verification.'; + } else { + $fix_policy = 'You may implement the smallest justified fix. Preserve the minimized replay and run focused verification.'; + } + + $artifact_lines = implode( "\n", array_map( static fn( string $path ): string => '- ' . $path, $existing_paths ) ); + $classification_lines = implode( "\n", array_map( static fn( string $classification ): string => '- ' . $classification, HTML_API_FUZZ_CODEX_CLASSIFICATIONS ) ); + $failure_class = html_api_fuzz_codex_safe_prompt_line( $signature['failureClass'] ?? '', 'failureClass' ); + $triage_kind = html_api_fuzz_codex_safe_prompt_line( $signature['triageKind'] ?? 'failure', 'triageKind' ); + $oracle_type = html_api_fuzz_codex_safe_prompt_line( $signature['oracleFindingType'] ?? '', 'oracleFindingType' ); + $suspected_owner = html_api_fuzz_codex_safe_prompt_line( $signature['suspectedOwner'] ?? '', 'suspectedOwner' ); + $upstream_issue = html_api_fuzz_codex_safe_prompt_line( $signature['upstreamIssueUrl'] ?? '', 'upstreamIssueUrl' ); + + return << 1, + 'kind' => 'html-api-fuzz-codex-job', + 'hash' => $signature_hash, + 'startedAt' => html_api_fuzz_codex_now_iso_z(), + 'startedAtUnix' => microtime( true ), + 'mode' => $args['mode'], + 'sandbox' => $args['sandbox'], + 'command' => $command, + 'processCommand' => $process_command, + 'promptPath' => $prompt_path, + 'stdoutPath' => $stdout_path, + 'stderrPath' => $stderr_path, + 'reportPath' => $report_path, + 'donePath' => $done_path, + ); + html_api_fuzz_codex_write_json( html_api_fuzz_codex_path_join( $signature_dir, 'job.json' ), $metadata ); + + $descriptor_spec = array( + 0 => array( 'pipe', 'r' ), + 1 => array( 'file', $stdout_path, 'w' ), + 2 => array( 'file', $stderr_path, 'w' ), + ); + $process = proc_open( $process_command, $descriptor_spec, $pipes, $args['repoRoot'] ); + if ( ! is_resource( $process ) ) { + throw new RuntimeException( 'Could not start Codex subprocess.' ); + } + + try { + html_api_fuzz_codex_write_stream( $pipes[0], $prompt ); + fclose( $pipes[0] ); + } catch ( Throwable $error ) { + if ( is_resource( $pipes[0] ) ) { + fclose( $pipes[0] ); + } + proc_terminate( $process ); + proc_close( $process ); + throw $error; + } + $status = proc_get_status( $process ); + + return array( + 'process' => $process, + 'pid' => (int) ( $status['pid'] ?? 0 ), + 'processGroupPid' => $uses_process_group ? (int) ( $status['pid'] ?? 0 ) : null, + 'knownExitCode' => ( ! ( $status['running'] ?? false ) && array_key_exists( 'exitcode', $status ) && -1 !== $status['exitcode'] ) ? (int) $status['exitcode'] : null, + 'signatureDir' => $signature_dir, + 'hash' => $signature_hash, + ); +} + +function html_api_fuzz_codex_poll_exit_code( array &$job ): ?int { + if ( null !== ( $job['knownExitCode'] ?? null ) ) { + $exit_code = (int) $job['knownExitCode']; + proc_close( $job['process'] ); + return $exit_code; + } + + $status = proc_get_status( $job['process'] ); + if ( $status['running'] ?? false ) { + return null; + } + + $exit_code = null; + if ( array_key_exists( 'exitcode', $status ) && -1 !== $status['exitcode'] ) { + $exit_code = (int) $status['exitcode']; + } elseif ( array_key_exists( 'cached_exitcode', $status ) && false !== $status['cached_exitcode'] ) { + $exit_code = (int) $status['cached_exitcode']; + } + + $closed_code = proc_close( $job['process'] ); + if ( null === $exit_code ) { + $exit_code = (int) $closed_code; + } + + return $exit_code; +} + +function html_api_fuzz_codex_mark_done( array $job, int $exit_code, array $extra = array() ): void { + $signature_dir = $job['signatureDir']; + $signature_hash = $job['hash']; + $done = array_merge( + array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-codex-done', + 'hash' => $signature_hash, + 'finishedAt' => html_api_fuzz_codex_now_iso_z(), + 'finishedAtUnix' => microtime( true ), + 'exitCode' => $exit_code, + 'reportPath' => html_api_fuzz_codex_path_join( $signature_dir, 'report.md' ), + 'stdoutPath' => html_api_fuzz_codex_path_join( $signature_dir, 'codex.jsonl' ), + 'stderrPath' => html_api_fuzz_codex_path_join( $signature_dir, 'codex.stderr.log' ), + ), + $extra + ); + html_api_fuzz_codex_write_json( html_api_fuzz_codex_path_join( $signature_dir, 'done.json' ), $done ); +} + +function html_api_fuzz_codex_mark_launch_failed( string $signature_dir, string $signature_hash, Throwable $error ): void { + html_api_fuzz_codex_mark_done( + array( + 'signatureDir' => $signature_dir, + 'hash' => $signature_hash, + ), + 1, + array( + 'launchFailed' => true, + 'errorClass' => get_class( $error ), + 'errorMessage' => $error->getMessage(), + ) + ); +} + +function html_api_fuzz_codex_load_candidate_signatures( array $state ): array { + $signatures = $state['signatures'] ?? array(); + if ( ! is_array( $signatures ) ) { + return array(); + } + + ksort( $signatures, SORT_STRING ); + $candidates = array(); + foreach ( $signatures as $signature_hash => $signature ) { + if ( ! is_array( $signature ) ) { + continue; + } + $signature_hash = (string) $signature_hash; + if ( ! html_api_fuzz_codex_valid_signature_hash( $signature_hash ) ) { + continue; + } + if ( 'minimized' !== ( $signature['status'] ?? null ) ) { + continue; + } + if ( null === html_api_fuzz_codex_newest_minimize_dir( $signature ) ) { + continue; + } + $candidates[] = array( $signature_hash, $signature ); + } + + return $candidates; +} + +function html_api_fuzz_codex_parse_args( array $argv ): array { + if ( in_array( '-h', $argv, true ) ) { + html_api_fuzz_codex_usage(); + exit( 0 ); + } + + $options = \HtmlApiFuzz\parse_cli_options( $argv ); + html_api_fuzz_codex_validate_cli_options( $options ); + if ( \HtmlApiFuzz\option_bool( $options, 'help', false ) ) { + html_api_fuzz_codex_usage(); + exit( 0 ); + } + + $triage_dir = \HtmlApiFuzz\option_string( $options, 'triage-dir' ); + if ( null === $triage_dir ) { + html_api_fuzz_codex_usage(); + exit( 1 ); + } + + $mode = \HtmlApiFuzz\option_string( $options, 'mode', 'classify' ); + if ( ! in_array( $mode, array( 'classify', 'fix' ), true ) ) { + throw new InvalidArgumentException( 'Expected --mode to be classify or fix.' ); + } + + $sandbox = \HtmlApiFuzz\option_string( $options, 'sandbox', 'fix' === $mode ? 'workspace-write' : 'read-only' ); + if ( ! in_array( $sandbox, array( 'read-only', 'workspace-write', 'danger-full-access' ), true ) ) { + throw new InvalidArgumentException( 'Expected --sandbox to be read-only, workspace-write, or danger-full-access.' ); + } + if ( 'fix' === $mode && 'read-only' === $sandbox ) { + throw new InvalidArgumentException( 'Expected --mode fix to use a writable sandbox.' ); + } + + $repo_root = html_api_fuzz_codex_require_dir( \HtmlApiFuzz\option_string( $options, 'repo-root', getcwd() ?: '.' ), 'Repo root' ); + $triage_dir = html_api_fuzz_codex_require_dir( $triage_dir, 'Triage directory' ); + $diagnostics_dir = \HtmlApiFuzz\option_string( $options, 'diagnostics-dir' ); + if ( null === $diagnostics_dir ) { + $diagnostics_dir = html_api_fuzz_codex_path_join( dirname( $triage_dir ), 'diagnostics' ); + } + + $interval_seconds = \HtmlApiFuzz\option_float( $options, 'interval-seconds', 120.0 ); + $max_concurrent = \HtmlApiFuzz\option_int( $options, 'max-concurrent', 1 ); + $max_launch_per_pass = \HtmlApiFuzz\option_int( $options, 'max-launch-per-pass', 1 ); + $stale_after_seconds = \HtmlApiFuzz\option_int( $options, 'stale-after-seconds', 6 * 60 * 60 ); + if ( $interval_seconds < 0 ) { + throw new InvalidArgumentException( 'Expected --interval-seconds to be at least 0.' ); + } + if ( $max_concurrent < 0 ) { + throw new InvalidArgumentException( 'Expected --max-concurrent to be at least 0.' ); + } + if ( $max_launch_per_pass < 0 ) { + throw new InvalidArgumentException( 'Expected --max-launch-per-pass to be at least 0.' ); + } + if ( $stale_after_seconds < 0 ) { + throw new InvalidArgumentException( 'Expected --stale-after-seconds to be at least 0.' ); + } + + return array( + 'triageDir' => $triage_dir, + 'diagnosticsDir' => html_api_fuzz_codex_normalize_path( $diagnostics_dir ), + 'repoRoot' => $repo_root, + 'codexBin' => \HtmlApiFuzz\option_string( $options, 'codex-bin', 'codex' ), + 'model' => \HtmlApiFuzz\option_string( $options, 'model' ), + 'intervalSeconds' => $interval_seconds, + 'maxConcurrent' => $max_concurrent, + 'maxLaunchPerPass' => $max_launch_per_pass, + 'staleAfterSeconds' => $stale_after_seconds, + 'once' => \HtmlApiFuzz\option_bool( $options, 'once', false ), + 'mode' => $mode, + 'sandbox' => $sandbox, + ); +} + +function html_api_fuzz_codex_install_signal_handlers( bool &$stopping ): void { + if ( ! function_exists( 'pcntl_signal' ) ) { + throw new RuntimeException( 'The pcntl extension is required for signal-safe Codex job orchestration.' ); + } + + if ( function_exists( 'pcntl_async_signals' ) ) { + pcntl_async_signals( true ); + } + if ( defined( 'SIGTERM' ) ) { + pcntl_signal( + SIGTERM, + static function () use ( &$stopping ): void { + $stopping = true; + } + ); + } + if ( defined( 'SIGINT' ) ) { + pcntl_signal( + SIGINT, + static function () use ( &$stopping ): void { + $stopping = true; + } + ); + } +} + +function html_api_fuzz_codex_dispatch_signals(): void { + if ( function_exists( 'pcntl_signal_dispatch' ) ) { + pcntl_signal_dispatch(); + } +} + +function html_api_fuzz_codex_sleep( float $seconds, bool &$stopping ): void { + $deadline = microtime( true ) + max( 0.0, $seconds ); + do { + html_api_fuzz_codex_dispatch_signals(); + if ( $stopping ) { + return; + } + + $remaining = $deadline - microtime( true ); + if ( $remaining <= 0 ) { + return; + } + + usleep( (int) min( 250000, max( 1000, round( $remaining * 1000000 ) ) ) ); + } while ( true ); +} + +function html_api_fuzz_codex_stop_job( array &$job ): void { + $status = proc_get_status( $job['process'] ); + if ( $status['running'] ?? false ) { + if ( null !== ( $job['processGroupPid'] ?? null ) && function_exists( 'posix_kill' ) && defined( 'SIGTERM' ) ) { + @posix_kill( -1 * (int) $job['processGroupPid'], SIGTERM ); + } else { + proc_terminate( $job['process'] ); + } + usleep( 200000 ); + $status = proc_get_status( $job['process'] ); + if ( $status['running'] ?? false ) { + if ( null !== ( $job['processGroupPid'] ?? null ) && function_exists( 'posix_kill' ) ) { + @posix_kill( -1 * (int) $job['processGroupPid'], 9 ); + } else { + proc_terminate( $job['process'], 9 ); + } + } + } + proc_close( $job['process'] ); + html_api_fuzz_codex_mark_done( + $job, + 143, + array( + 'interrupted' => true, + ) + ); +} + +function html_api_fuzz_codex_main( array $argv ): int { + $args = html_api_fuzz_codex_parse_args( $argv ); + $state_path = html_api_fuzz_codex_path_join( $args['triageDir'], 'state.json' ); + $running = array(); + $stopping = false; + $scanned = false; + + html_api_fuzz_codex_install_signal_handlers( $stopping ); + \HtmlApiFuzz\ensure_dir( $args['diagnosticsDir'] ); + html_api_fuzz_codex_write_json( + html_api_fuzz_codex_path_join( $args['diagnosticsDir'], 'orchestrator-state.json' ), + array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-codex-orchestrator-state', + 'startedAt' => html_api_fuzz_codex_now_iso_z(), + 'triageDir' => $args['triageDir'], + 'repoRoot' => $args['repoRoot'], + 'mode' => $args['mode'], + 'sandbox' => $args['sandbox'], + ) + ); + + try { + while ( ! $stopping ) { + html_api_fuzz_codex_dispatch_signals(); + foreach ( array_keys( $running ) as $key ) { + $exit_code = html_api_fuzz_codex_poll_exit_code( $running[ $key ] ); + if ( null === $exit_code ) { + continue; + } + + html_api_fuzz_codex_mark_done( $running[ $key ], $exit_code ); + echo 'completed ' . $running[ $key ]['hash'] . ' exit=' . $exit_code . "\n"; + unset( $running[ $key ] ); + } + + /* + * Graceful stop: a STOP file in the run directory stops new + * launches; running jobs finish before the orchestrator exits. + * The run directory comes from the watcher state when available + * (the triage dir may live outside the run dir), matching how + * launches resolve it below. + */ + $stop_run_dir = dirname( $args['triageDir'] ); + $stop_state = html_api_fuzz_codex_read_json( $state_path ); + if ( is_array( $stop_state ) && is_string( $stop_state['runDir'] ?? null ) && ! html_api_fuzz_codex_path_has_control_chars( $stop_state['runDir'] ) ) { + $stop_run_dir = html_api_fuzz_codex_normalize_path( $stop_state['runDir'] ); + } + $stop_file_present = is_file( html_api_fuzz_codex_path_join( $stop_run_dir, 'STOP' ) ); + if ( $stop_file_present && empty( $running ) ) { + echo "stop requested; exiting\n"; + break; + } + + $can_scan = ( ! $args['once'] || ! $scanned ) && ! $stop_file_present; + if ( $can_scan ) { + $available = max( 0, $args['maxConcurrent'] - count( $running ) ); + $launched = 0; + if ( $available > 0 ) { + $state = html_api_fuzz_codex_read_json( $state_path ); + if ( is_array( $state ) ) { + $launch_args = $args; + if ( is_string( $state['runDir'] ?? null ) && ! html_api_fuzz_codex_path_has_control_chars( $state['runDir'] ) ) { + $launch_args['runDir'] = html_api_fuzz_codex_normalize_path( $state['runDir'] ); + } else { + $launch_args['runDir'] = dirname( $args['triageDir'] ); + } + + foreach ( html_api_fuzz_codex_load_candidate_signatures( $state ) as $candidate ) { + if ( $launched >= $available || $launched >= $args['maxLaunchPerPass'] ) { + break; + } + list( $signature_hash, $signature ) = $candidate; + $signature_dir = html_api_fuzz_codex_claim_signature( $args['diagnosticsDir'], $signature_hash, $signature, $args['staleAfterSeconds'] ); + if ( null === $signature_dir ) { + continue; + } + + try { + $job = html_api_fuzz_codex_launch( $launch_args, $signature_hash, $signature, $signature_dir ); + $running[] = $job; + ++$launched; + echo 'launched ' . $signature_hash . ' pid=' . $job['pid'] . "\n"; + } catch ( Throwable $error ) { + html_api_fuzz_codex_mark_launch_failed( $signature_dir, $signature_hash, $error ); + echo 'launch failed ' . $signature_hash . ' error=' . $error->getMessage() . "\n"; + } + } + } + } + $scanned = true; + } + + if ( $args['once'] && empty( $running ) ) { + break; + } + + $sleep_seconds = $args['once'] ? min( 1.0, $args['intervalSeconds'] ) : $args['intervalSeconds']; + html_api_fuzz_codex_sleep( $sleep_seconds, $stopping ); + } + } finally { + foreach ( array_keys( $running ) as $key ) { + html_api_fuzz_codex_stop_job( $running[ $key ] ); + unset( $running[ $key ] ); + } + } + + return 0; +} + +if ( ! defined( 'HTML_API_FUZZ_CODEX_SELF_TESTING' ) || ! HTML_API_FUZZ_CODEX_SELF_TESTING ) { + exit( html_api_fuzz_codex_main( $argv ) ); +} diff --git a/tools/html-api-fuzz/launcher.php b/tools/html-api-fuzz/launcher.php new file mode 100755 index 0000000000000..a4b843820be8c --- /dev/null +++ b/tools/html-api-fuzz/launcher.php @@ -0,0 +1,362 @@ +#!/usr/bin/env php + array( 'pipe', 'r' ), + 1 => array( 'pipe', 'w' ), + 2 => array( 'pipe', 'w' ), + ); + + $process = proc_open( $command, $spec, $pipes, $cwd ); + if ( ! is_resource( $process ) ) { + throw new RuntimeException( 'Could not start runner lane.' ); + } + + fclose( $pipes[0] ); + stream_set_blocking( $pipes[1], false ); + stream_set_blocking( $pipes[2], false ); + \HtmlApiFuzz\ensure_dir( dirname( $log_path ) ); + file_put_contents( $log_path, '' ); + + return array( + 'process' => $process, + 'pipes' => $pipes, + 'logPath' => $log_path, + 'startedAt' => microtime( true ), + 'stdout' => '', + 'stderr' => '', + ); +} + +function html_api_fuzz_launcher_drain_lane( array &$lane ): void { + $stdout = stream_get_contents( $lane['pipes'][1] ); + $stderr = stream_get_contents( $lane['pipes'][2] ); + if ( '' !== $stdout ) { + $lane['stdout'] .= $stdout; + file_put_contents( $lane['logPath'], $stdout, FILE_APPEND ); + } + if ( '' !== $stderr ) { + $lane['stderr'] .= $stderr; + file_put_contents( $lane['logPath'], $stderr, FILE_APPEND ); + } +} + +function html_api_fuzz_launcher_close_lane( array &$lane ): array { + html_api_fuzz_launcher_drain_lane( $lane ); + fclose( $lane['pipes'][1] ); + fclose( $lane['pipes'][2] ); + $code = proc_close( $lane['process'] ); + + return array( + 'code' => $code, + 'durationMs' => (int) round( ( microtime( true ) - $lane['startedAt'] ) * 1000 ), + 'stdoutTail' => substr( $lane['stdout'], -2000 ), + 'stderrTail' => substr( $lane['stderr'], -2000 ), + ); +} + +$options = \HtmlApiFuzz\parse_cli_options( $argv ); +if ( \HtmlApiFuzz\option_bool( $options, 'help', false ) || \HtmlApiFuzz\option_bool( $options, 'h', false ) ) { + html_api_fuzz_launcher_usage(); + exit( 0 ); +} + +$repo_root = \HtmlApiFuzz\repo_root(); +$output_dir = \HtmlApiFuzz\option_string( $options, 'output-dir', $repo_root . '/artifacts/html-api-fuzz/launch-' . \HtmlApiFuzz\timestamp() ); +$lanes = max( 1, \HtmlApiFuzz\option_int( $options, 'lanes', 2 ) ); +$start_seed = \HtmlApiFuzz\option_int( $options, 'start-seed', 1 ); +$max_seeds = \HtmlApiFuzz\option_int( $options, 'max-seeds', 0 ); +$duration_seconds = \HtmlApiFuzz\option_float( $options, 'duration-seconds', 60.0 ); +$timeout_ms = \HtmlApiFuzz\option_int( $options, 'timeout-ms', 2500 ); +$profile = \HtmlApiFuzz\option_string( $options, 'profile', 'auto' ); +$mode = \HtmlApiFuzz\option_string( $options, 'mode', 'auto' ); +$payload_policy = \HtmlApiFuzz\option_string( $options, 'payload-policy', 'auto' ); +$max_input_bytes = \HtmlApiFuzz\option_int( $options, 'max-input-bytes', 0 ); +$max_tokens = \HtmlApiFuzz\option_int( $options, 'max-tokens', 2000 ); +$max_nodes = \HtmlApiFuzz\option_int( $options, 'max-nodes', 3000 ); +$stop_on_failure = \HtmlApiFuzz\option_bool( $options, 'stop-on-failure', false ); +$fail_unsupported = \HtmlApiFuzz\option_bool( $options, 'fail-unsupported', false ); +$run_watcher = \HtmlApiFuzz\option_bool( $options, 'watcher', false ); +$triage_oracle_findings = \HtmlApiFuzz\option_bool( $options, 'triage-oracle-findings', false ); +$max_keep_per_signature = \HtmlApiFuzz\option_int( $options, 'max-keep-per-signature', 5 ); +$keep_all_artifacts = \HtmlApiFuzz\option_bool( $options, 'keep-all-artifacts', false ); +if ( $max_keep_per_signature < 1 ) { + throw new InvalidArgumentException( 'Expected --max-keep-per-signature to be at least 1.' ); +} +html_api_fuzz_launcher_validate_generator_options( $profile, $mode, $payload_policy ); +html_api_fuzz_launcher_validate_runtime_options( $max_seeds, $duration_seconds, $timeout_ms, $max_input_bytes, $max_tokens, $max_nodes ); + +if ( is_file( $output_dir . '/STOP' ) ) { + // A leftover stop request must not silently turn this launch into a + // 0-seed success; starting again is an explicit operator decision. + fwrite( STDERR, "Stop file already exists: {$output_dir}/STOP\nRemove it to start a run in this directory.\n" ); + exit( 1 ); +} + +\HtmlApiFuzz\ensure_dir( $output_dir ); +$events_path = $output_dir . '/events.ndjson'; +$state_path = $output_dir . '/launcher-state.json'; +$git_metadata = null === \HtmlApiFuzz\option_string( $options, 'git-metadata-base64', null ) + ? \HtmlApiFuzz\git_metadata() + : \HtmlApiFuzz\git_metadata_from_base64( \HtmlApiFuzz\option_string( $options, 'git-metadata-base64' ) ); +$git_metadata_base64 = \HtmlApiFuzz\git_metadata_base64( $git_metadata ); +$oracle_renderer = \HtmlApiFuzz\OracleRenderer::from_options( $options ); +$oracle_metadata = $oracle_renderer->metadata(); +if ( true !== ( $oracle_metadata['available'] ?? false ) ) { + throw new RuntimeException( 'Selected oracle is unavailable: ' . ( $oracle_metadata['error'] ?? $oracle_metadata['versionError'] ?? 'unknown error' ) ); +} +$oracle_worker_args = $oracle_renderer->worker_args(); + +$state = array( + 'schemaVersion' => 1, + 'kind' => 'html-api-fuzz-launcher-state', + 'startedAt' => gmdate( 'c' ), + 'updatedAt' => gmdate( 'c' ), + 'outputDir' => $output_dir, + 'lanes' => $lanes, + 'startSeed' => $start_seed, + 'seedStride' => $lanes, + 'profile' => $profile, + 'mode' => $mode, + 'payloadPolicy' => $payload_policy, + 'maxInputBytes' => $max_input_bytes > 0 ? $max_input_bytes : null, + 'git' => $git_metadata, + 'oracle' => $oracle_metadata, + 'finished' => false, + 'laneResults' => array(), +); +\HtmlApiFuzz\write_json_file( $state_path, $state ); +\HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'launcher-start', 'outputDir' => $output_dir, 'lanes' => $lanes, 'git' => $git_metadata, 'oracle' => $oracle_metadata ) ); + +$running = array(); +for ( $i = 0; $i < $lanes; ++$i ) { + $lane_max_seeds = 0; + if ( 0 !== $max_seeds ) { + $lane_max_seeds = intdiv( $max_seeds, $lanes ) + ( $i < ( $max_seeds % $lanes ) ? 1 : 0 ); + if ( 0 === $lane_max_seeds ) { + $lane_dir = $output_dir . '/lane-' . str_pad( (string) $i, 2, '0', STR_PAD_LEFT ); + $state['laneResults'][ $i ] = array( + 'lane' => $i, + 'status' => 'skipped', + 'outputDir' => $lane_dir, + 'reason' => 'no seeds assigned', + ); + continue; + } + } + + $lane_dir = $output_dir . '/lane-' . str_pad( (string) $i, 2, '0', STR_PAD_LEFT ); + $command = array( + PHP_BINARY, + __DIR__ . '/runner.php', + '--output-dir', + $lane_dir, + '--start-seed', + (string) ( $start_seed + $i ), + '--seed-stride', + (string) $lanes, + '--duration-seconds', + (string) $duration_seconds, + '--timeout-ms', + (string) $timeout_ms, + '--profile', + $profile, + '--mode', + $mode, + '--payload-policy', + $payload_policy, + '--max-tokens', + (string) $max_tokens, + '--max-nodes', + (string) $max_nodes, + '--git-metadata-base64', + $git_metadata_base64, + '--max-keep-per-signature', + (string) $max_keep_per_signature, + '--stop-file', + $output_dir . '/STOP', + ); + foreach ( $oracle_worker_args as $arg ) { + $command[] = $arg; + } + + if ( 0 !== $max_seeds ) { + $command[] = '--max-seeds'; + $command[] = (string) $lane_max_seeds; + } + if ( $keep_all_artifacts ) { + $command[] = '--keep-all-artifacts'; + } + if ( $stop_on_failure ) { + $command[] = '--stop-on-failure'; + } + if ( $fail_unsupported ) { + $command[] = '--fail-unsupported'; + } + if ( $max_input_bytes > 0 ) { + $command[] = '--max-input-bytes'; + $command[] = (string) $max_input_bytes; + } + + $running[ $i ] = html_api_fuzz_launcher_start_lane( $command, $repo_root, $lane_dir . '/runner.stdout.log' ); + $state['laneResults'][ $i ] = array( + 'lane' => $i, + 'status' => 'running', + 'command' => \HtmlApiFuzz\command_string( $command ), + 'outputDir' => $lane_dir, + 'logPath' => $lane_dir . '/runner.stdout.log', + ); + \HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'lane-start', 'lane' => $i, 'outputDir' => $lane_dir, 'oracle' => $oracle_metadata ) ); +} +\HtmlApiFuzz\write_json_file( $state_path, $state ); + +while ( $running ) { + foreach ( array_keys( $running ) as $lane_id ) { + html_api_fuzz_launcher_drain_lane( $running[ $lane_id ] ); + $status = proc_get_status( $running[ $lane_id ]['process'] ); + if ( $status['running'] ) { + continue; + } + + $closed = html_api_fuzz_launcher_close_lane( $running[ $lane_id ] ); + unset( $running[ $lane_id ] ); + $lane_dir = $state['laneResults'][ $lane_id ]['outputDir']; + $runner_state = \HtmlApiFuzz\read_json_file( $lane_dir . '/state.json' ); + $state['laneResults'][ $lane_id ] = array_merge( + $state['laneResults'][ $lane_id ], + array( + 'status' => 0 === $closed['code'] ? 'completed' : 'failed', + 'code' => $closed['code'], + 'durationMs' => $closed['durationMs'], + 'runnerState' => $runner_state, + 'stdoutTail' => $closed['stdoutTail'], + 'stderrTail' => $closed['stderrTail'], + ) + ); + $state['updatedAt'] = gmdate( 'c' ); + \HtmlApiFuzz\write_json_file( $state_path, $state ); + \HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'lane-stop', 'lane' => $lane_id, 'code' => $closed['code'] ) ); + } + usleep( 100000 ); +} + +$aggregate = array( + 'successes' => 0, + 'failures' => 0, + 'unsupported' => 0, + 'oracleParseErrors' => 0, + 'oracleUnsupported' => 0, + 'oracleTolerated' => 0, + 'oracleFindings' => 0, +); +foreach ( $state['laneResults'] as $lane ) { + $runner_state = $lane['runnerState'] ?? array(); + foreach ( $aggregate as $name => $count ) { + $aggregate[ $name ] += (int) ( $runner_state[ $name ] ?? 0 ); + } +} + +$state['finished'] = true; +$state['updatedAt'] = gmdate( 'c' ); +$state['aggregate'] = $aggregate; +\HtmlApiFuzz\write_json_file( $state_path, $state ); +\HtmlApiFuzz\append_ndjson( $events_path, array( 'at' => gmdate( 'c' ), 'kind' => 'launcher-stop', 'aggregate' => $aggregate ) ); + +$watcher_result = null; +if ( $run_watcher ) { + $triage_dir = $output_dir . '/triage'; + $minimize_timeout_ms = \HtmlApiFuzz\option_int( $options, 'minimize-timeout-ms', 300000 ); + $watcher_timeout_ms = \HtmlApiFuzz\option_int( $options, 'watcher-timeout-ms', max( 600000, $minimize_timeout_ms + 60000 ) ); + $proc = \HtmlApiFuzz\run_php_process( + array_values( + array_filter( + array( + __DIR__ . '/watcher.php', + '--run-dir', + $output_dir, + '--state-dir', + $triage_dir, + '--once', + '--minimize-timeout-ms', + (string) $minimize_timeout_ms, + '--timeout-ms', + (string) $timeout_ms, + array_key_exists( 'max-attempts', $options ) ? '--max-attempts' : null, + array_key_exists( 'max-attempts', $options ) ? (string) \HtmlApiFuzz\option_int( $options, 'max-attempts', 250 ) : null, + array_key_exists( 'max-minimize', $options ) ? '--max-minimize' : null, + array_key_exists( 'max-minimize', $options ) ? (string) \HtmlApiFuzz\option_int( $options, 'max-minimize', 0 ) : null, + array_key_exists( 'probe-mode', $options ) ? '--probe-mode' : null, + array_key_exists( 'probe-mode', $options ) ? \HtmlApiFuzz\option_string( $options, 'probe-mode', 'auto' ) : null, + \HtmlApiFuzz\option_bool( $options, 'keep-candidate-artifacts', false ) ? '--keep-candidate-artifacts' : null, + \HtmlApiFuzz\option_bool( $options, 'no-minimize', false ) ? '--no-minimize' : null, + \HtmlApiFuzz\option_bool( $options, 'any-failure', false ) ? '--any-failure' : null, + $triage_oracle_findings ? '--triage-oracle-findings' : null, + ), + static function ( $value ) { + return null !== $value; + } + ) + ), + $repo_root, + $watcher_timeout_ms, + $triage_dir . '/watcher.log' + ); + $watcher_result = array( + 'code' => $proc['code'], + 'timedOut' => $proc['timedOut'], + 'durationMs' => $proc['durationMs'], + 'logPath' => $proc['logPath'], + ); +} + +echo \HtmlApiFuzz\json_encode_safe( + array( + 'ok' => true, + 'outputDir' => $output_dir, + 'statePath' => $state_path, + 'aggregate' => $aggregate, + 'watcherResult' => $watcher_result, + ) +) . "\n"; diff --git a/tools/html-api-fuzz/lib/Corpus.php b/tools/html-api-fuzz/lib/Corpus.php new file mode 100644 index 0000000000000..005a91dc8a40b --- /dev/null +++ b/tools/html-api-fuzz/lib/Corpus.php @@ -0,0 +1,78 @@ + basename( $file ), + 'data' => $data, + ); + } + } + + if ( null === $directory ) { + self::$entries = $entries; + } + return $entries; + } + + /** + * Extracts #data sections from html5lib .dat content. A section runs from + * the line after `#data` to the line before the next `#` directive, with + * the trailing newline removed. + */ + private static function parse_dat_data_sections( string $contents ): array { + $sections = array(); + $lines = explode( "\n", $contents ); + $current = null; + foreach ( $lines as $line ) { + if ( '#data' === $line ) { + $current = array(); + continue; + } + if ( null !== $current ) { + if ( '' !== $line && '#' === $line[0] ) { + $sections[] = implode( "\n", $current ); + $current = null; + continue; + } + $current[] = $line; + } + } + if ( null !== $current && array() !== $current ) { + $sections[] = implode( "\n", $current ); + } + + return $sections; + } +} diff --git a/tools/html-api-fuzz/lib/Generator.php b/tools/html-api-fuzz/lib/Generator.php new file mode 100644 index 0000000000000..21eed89952d96 --- /dev/null +++ b/tools/html-api-fuzz/lib/Generator.php @@ -0,0 +1,1409 @@ +`; the others receive a small probe weight so the fuzzer + * exercises the unsupported-context path today and picks up real + * coverage automatically when create_fragment() gains context support. + */ + public static function fragment_contexts(): array { + return array( 'body', 'div', 'p', 'td', 'tr', 'table', 'caption', 'colgroup', 'select', 'option', 'template', 'title', 'textarea', 'script', 'style', 'svg', 'math' ); + } + + public static function generate( int $seed, string $profile = 'auto', string $mode = 'auto', string $payload_policy = 'auto', ?int $max_input_bytes = null ): array { + $rng = new Prng( $seed ); + $requested_profile = $profile; + $requested_mode = $mode; + $requested_payload_policy = $payload_policy; + if ( 'auto' === $profile ) { + $profile = $rng->weighted( + array( + 'balanced' => 22, + 'full-document' => 8, + 'body-fragment' => 8, + 'tables' => 11, + 'template' => 8, + 'select' => 8, + 'foreign-content' => 10, + 'rawtext-rcdata' => 8, + 'text-fragment' => 4, + 'formatting-adoption' => 10, + 'attributes-entities' => 7, + 'comments-doctype-bogus' => 5, + 'deep-nesting' => 2, + 'resource-stress' => 1, + 'incomplete-malformed' => 2, + ) + ); + } elseif ( ! in_array( $profile, self::profiles(), true ) ) { + throw new \InvalidArgumentException( 'Unknown generator profile: ' . $profile ); + } + + if ( 'auto' === $payload_policy ) { + $payload_policy = 'resource-stress' === $profile + ? $rng->weighted( + array( + 'stress-long' => 75, + 'mostly-valid' => 15, + 'valid-utf8' => 10, + ) + ) + : $rng->weighted( + array( + 'valid-utf8' => 52, + 'mostly-valid' => 33, + 'ascii-structural' => 15, + ) + ); + } elseif ( ! in_array( $payload_policy, self::payload_policies(), true ) ) { + throw new \InvalidArgumentException( 'Unknown generator payload policy: ' . $payload_policy ); + } + + if ( 'auto' === $mode ) { + $mode = 'full-document' === $profile ? self::MODE_FULL_DOCUMENT : ( in_array( $profile, array( 'body-fragment', 'text-fragment' ), true ) ? self::MODE_FRAGMENT_BODY : $rng->weighted( array( self::MODE_FRAGMENT_BODY => 70, self::MODE_FULL_DOCUMENT => 30 ) ) ); + } elseif ( ! in_array( $mode, self::modes(), true ) ) { + throw new \InvalidArgumentException( 'Unknown generator mode: ' . $mode ); + } + + $fragment_context = 'body'; + if ( self::MODE_FRAGMENT_BODY === $mode ) { + $weights = array( 'body' => 240 ); + foreach ( self::fragment_contexts() as $context ) { + if ( 'body' !== $context ) { + $weights[ $context ] = 1; + } + } + $fragment_context = $rng->weighted( $weights ); + } + + $generator = new self( $rng, $profile, $payload_policy ); + if ( 'body' !== $fragment_context ) { + $generator->mark_feature( 'fragment-context:' . $fragment_context ); + } + $max_depth = $generator->depth_for_profile(); + $body = 'text-fragment' === $profile ? $generator->text_fragment_input() : $generator->nodes( $max_depth, 'body' ); + + if ( self::MODE_FULL_DOCUMENT === $mode ) { + $html = $generator->full_document( $body ); + } else { + $html = $body; + } + $truncated = false; + if ( null !== $max_input_bytes && $max_input_bytes > 0 && strlen( $html ) > $max_input_bytes ) { + $html = self::trim_to_max_bytes( $html, $max_input_bytes ); + $truncated = true; + $generator->mark_feature( 'generator:truncated' ); + } + + return array( + 'input' => $html, + 'mode' => $mode, + 'profile' => $profile, + 'payloadPolicy' => $payload_policy, + 'fragmentContext' => $fragment_context, + 'parameters' => array( + 'seed' => $seed, + 'requestedProfile' => $requested_profile, + 'requestedMode' => $requested_mode, + 'requestedPayloadPolicy' => $requested_payload_policy, + 'profile' => $profile, + 'mode' => $mode, + 'payloadPolicy' => $payload_policy, + 'fragmentContext' => $fragment_context, + 'maxDepth' => $max_depth, + 'maxInputBytes' => $max_input_bytes, + 'truncated' => $truncated, + 'byteLength' => strlen( $html ), + 'features' => $generator->features(), + ), + ); + } + + private function __construct( Prng $rng, string $profile, string $payload_policy ) { + $this->rng = $rng; + $this->profile = $profile; + $this->payload_policy = $payload_policy; + } + + private static function trim_to_max_bytes( string $html, int $max_input_bytes ): string { + $trimmed = substr( $html, 0, $max_input_bytes ); + while ( '' !== $trimmed && 1 !== preg_match( '//u', $trimmed ) ) { + $trimmed = substr( $trimmed, 0, -1 ); + } + + return $trimmed; + } + + private function mark_feature( string $feature ): void { + $this->features[ $feature ] = true; + } + + private function features(): array { + $features = array_keys( $this->features ); + sort( $features ); + return $features; + } + + private function depth_for_profile(): int { + if ( 'resource-stress' === $this->profile ) { + return $this->rng->int( 10, 20 ); + } + if ( 'deep-nesting' === $this->profile ) { + return $this->rng->int( 8, 18 ); + } + if ( 'balanced' === $this->profile ) { + return $this->rng->int( 3, 5 ); + } + return $this->rng->int( 3, 6 ); + } + + private function full_document( string $body ): string { + $doctype = $this->rng->chance( 70 ) ? $this->doctype() : ''; + $head = $this->rng->chance( 65 ) ? '' . $this->head_nodes() . '' : $this->head_nodes(); + $attrs = $this->attrs(); + $body_at = $this->attrs(); + + if ( $this->rng->chance( 4 ) ) { + $this->mark_feature( 'frameset' ); + $frames = 'attrs() . '>' . $this->terminal_text( true ) . ''; + return $doctype . '' . $head . 'attrs() . '>' . $frames . ( $this->rng->chance( 70 ) ? '' : '' ) . ( $this->rng->chance( 40 ) ? $body : '' ); + } + + if ( $this->rng->chance( 20 ) ) { + return $doctype . $head . $body; + } + + $trailer = ''; + if ( $this->rng->chance( 8 ) ) { + $this->mark_feature( 'content-after-html' ); + $trailer = $this->rng->choice( array( $this->terminal_ascii( 6 ), '', '
tail
', '' ) ); + } + + return $doctype . '' . $head . '' . $body . ( $this->rng->chance( 75 ) ? '' . $trailer : '' ); + } + + private function head_nodes(): string { + $out = ''; + if ( $this->rng->chance( 45 ) ) { + $out .= '' . $this->terminal_text() . ''; + } + if ( $this->rng->chance( 35 ) ) { + $out .= 'attrs() . '>'; + } + if ( $this->rng->chance( 25 ) ) { + $out .= ''; + } + return $out; + } + + private function nodes( int $depth, string $context ): string { + if ( 'deep-nesting' === $this->profile ) { + $count = $this->rng->int( 1, 2 ); + } elseif ( $depth > 4 ) { + $count = $this->rng->int( 1, 3 ); + } elseif ( $depth > 2 ) { + $count = $this->rng->int( 1, 5 ); + } else { + $count = $this->rng->int( 1, 7 ); + } + + $out = ''; + for ( $i = 0; $i < $count; ++$i ) { + $out .= $this->node( $depth, $context ); + if ( strlen( $out ) > 131072 ) { + $this->mark_feature( 'generator:hard-truncated' ); + return self::trim_to_max_bytes( $out, 131072 ); + } + } + return $out; + } + + private function node( int $depth, string $context ): string { + if ( $depth <= 0 ) { + return $this->leaf(); + } + + $weights = array( + 'element' => 33, + 'text' => 16, + 'charref' => 1, + 'comment' => 7, + 'void' => 8, + 'raw' => 5, + 'template' => 5, + 'table' => 5, + 'select' => 2, + 'foreign' => 5, + 'adoption' => 2, + 'reconstruction' => 0, + 'list-chain' => 1, + 'special-closers' => 1, + 'weird-tag' => 0, + 'doctype' => 2, + 'bogus' => 2, + ); + + if ( 'tables' === $this->profile ) { + $weights['table'] = 35; + $weights['element'] = 15; + } elseif ( 'template' === $this->profile ) { + $weights['template'] = 35; + } elseif ( 'select' === $this->profile ) { + $weights['select'] = 35; + $weights['table'] = 8; + $weights['element'] = 15; + } elseif ( 'foreign-content' === $this->profile ) { + $weights['foreign'] = 35; + } elseif ( 'rawtext-rcdata' === $this->profile ) { + $weights['raw'] = 35; + } elseif ( 'comments-doctype-bogus' === $this->profile ) { + $weights['comment'] = 25; + $weights['doctype'] = 12; + $weights['bogus'] = 15; + } elseif ( 'attributes-entities' === $this->profile ) { + $weights['element'] = 50; + $weights['text'] = 22; + $weights['charref'] = 12; + $weights['weird-tag'] = 8; + } elseif ( 'formatting-adoption' === $this->profile ) { + $weights['element'] = 35; + $weights['adoption'] = 20; + $weights['reconstruction'] = 30; + } elseif ( 'incomplete-malformed' === $this->profile ) { + $weights['bogus'] = 25; + $weights['element'] = 28; + $weights['weird-tag'] = 15; + $weights['special-closers'] = 4; + } + + switch ( $this->rng->weighted( $weights ) ) { + case 'text': + $this->mark_feature( 'text' ); + return $this->terminal_text(); + case 'charref': + $this->mark_feature( 'text' ); + return $this->character_reference( 'text' ) . $this->terminal_payload(); + case 'comment': + $this->mark_feature( 'comment' ); + return $this->comment(); + case 'void': + $this->mark_feature( 'void-element' ); + return '<' . $this->rng->choice( $this->void_tags ) . $this->attrs() . ( $this->rng->chance( 25 ) ? '/>' : '>' ); + case 'raw': + return $this->raw_element(); + case 'template': + $this->mark_feature( 'template' ); + return 'attrs() . '>' . $this->nodes( $depth - 1, 'body' ) . ( $this->rng->chance( 80 ) ? '' : '' ); + case 'table': + return $this->table( $depth - 1 ); + case 'select': + return $this->select_stress( $depth - 1 ); + case 'foreign': + return $this->foreign( $depth - 1 ); + case 'adoption': + return $this->adoption_pattern( $depth - 1 ); + case 'reconstruction': + return $this->active_formatting_reconstruction_pattern( $depth - 1 ); + case 'list-chain': + return $this->auto_closing_chain( $depth - 1 ); + case 'special-closers': + return $this->special_closers(); + case 'weird-tag': + return $this->weird_element( $depth - 1 ); + case 'doctype': + $this->mark_feature( 'doctype' ); + return $this->doctype(); + case 'bogus': + $this->mark_feature( 'bogus-markup' ); + return $this->bogus(); + case 'element': + default: + return $this->element( $depth - 1 ); + } + } + + private function element( int $depth ): string { + if ( 'attributes-entities' === $this->profile && $this->rng->chance( 16 ) ) { + return $this->weird_element( $depth ); + } + if ( 'incomplete-malformed' === $this->profile && $this->rng->chance( 12 ) ) { + return $this->weird_element( $depth ); + } + + $tag = $this->tag_name(); + if ( 'formatting-adoption' === $this->profile ) { + $tag = $this->rng->choice( array( 'a', 'b', 'big', 'button', 'em', 'font', 'i', 'nobr', 'p', 'span', 'strong' ) ); + $this->mark_feature( 'formatting-adoption-candidate' ); + } + + $close = $this->rng->chance( 'incomplete-malformed' === $this->profile ? 55 : 85 ); + $end = $close ? 'rng->chance( 88 ) ? $tag : $this->tag_name() ) . '>' : ''; + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . $end; + } + + private function weird_element( int $depth ): string { + $this->mark_feature( 'tag:weird-syntax' ); + $case = $this->rng->weighted( + array( + 'unusual-name' => 36, + 'invalid-name' => 30, + 'boundary-spacing' => 18, + 'malformed-closer' => 16, + ) + ); + + if ( 'invalid-name' === $case ) { + $tag = $this->invalid_tag_name(); + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . ( $this->rng->chance( 40 ) ? '' : '' ); + } + + $tag = 'unusual-name' === $case ? $this->unusual_tag_name() : $this->tag_name(); + if ( 'boundary-spacing' === $case ) { + $this->mark_feature( 'tag:weird-spacing' ); + return '<' . $tag . $this->tag_gap() . $this->attrs() . $this->tag_gap() . ( $this->rng->chance( 25 ) ? '/' . $this->tag_gap() : '' ) . '>' . $this->nodes( $depth, 'body' ) . ( $this->rng->chance( 70 ) ? 'tag_gap() . '>' : '' ); + } + + if ( 'malformed-closer' === $case ) { + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . 'invalid_tag_name() . $this->tag_gap() . '>'; + } + + return '<' . $tag . $this->attrs() . '>' . $this->nodes( $depth, 'body' ) . ( $this->rng->chance( 75 ) ? '' : '' ); + } + + /** + * select/option/optgroup nesting stress, including elements that end or + * are disallowed in select (input, textarea, button, nested select) and + * select inside table structures. + */ + private function select_stress( int $depth ): string { + $this->mark_feature( 'select' ); + $out = ''; + $count = $this->rng->int( 1, 4 ); + for ( $i = 0; $i < $count; ++$i ) { + switch ( $this->rng->weighted( array( 'option' => 38, 'optgroup' => 22, 'breaker' => 18, 'nested-select' => 10, 'other' => 12 ) ) ) { + case 'option': + $this->mark_feature( 'select:option' ); + $out .= 'attrs() . '>' . $this->terminal_text() . ( $this->rng->chance( 55 ) ? '' : '' ); + break; + case 'optgroup': + $this->mark_feature( 'select:optgroup' ); + $out .= 'attrs() . '>