diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..d121cbc --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,88 @@ +# Times conversions through the page (tests/bench/bench.js) in each browser engine, Chromium, Firefox and +# WebKit (Safari's), on Ubuntu, against native chdman on the same machine (whose outputs each conversion must +# equal). The results are the run's summary and an artifact. When the page changes, weekly, and on demand. +name: Benchmarks + +on: + push: + branches: [main] + paths: ['dist/**', 'tests/bench/**', '.github/workflows/bench.yml'] + pull_request: + paths: ['dist/**', 'tests/bench/**', '.github/workflows/bench.yml'] + schedule: + - cron: '11 7 * * 4' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: bench-${{ github.ref }} + cancel-in-progress: true + +jobs: + bench: + strategy: + fail-fast: false + matrix: + include: + - browser: chromium + os: ubuntu-24.04 + native: true + - browser: firefox + os: ubuntu-24.04 + native: true + # (WebKit on Ubuntu too: the macOS runner this asked for was cancelled before any runner took it) + - browser: webkit + os: ubuntu-24.04 + native: true + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + name: bench (${{ matrix.browser }}) + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: tests/package-lock.json + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + - name: numpy, for the synthetic test discs + run: pip install numpy==2.4.6 + - name: Cache MAME 0.289 and its chdman + if: matrix.native + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + third_party/mame + build/chdman-0.289 + key: mame-0.289-${{ runner.os }}-${{ hashFiles('scripts/fetch-mame.sh', 'scripts/build-upstream.sh', 'wasm/**') }} + - name: Native chdman (0.289 and the engine) + if: matrix.native + run: | + [ -x build/chdman-0.289 ] || ./scripts/build-upstream.sh + ./scripts/build-native.sh + - name: Playwright and ${{ matrix.browser }} + working-directory: tests + run: | + npm ci --no-audit --no-fund + npx playwright install --with-deps ${{ matrix.browser }} + # (bench.js makes the synthetic discs, the benchmark images among them) + - name: Benchmark + working-directory: tests + run: > + node bench/bench.js --browser ${{ matrix.browser }} --label ci-${{ matrix.browser }} + --threads 1,4 --presets default,fast --cd-mb 150 --dvd-mb 512 + - name: Summary + if: always() + run: cat tests/.cache/bench/*.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + - name: Results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: bench-${{ matrix.browser }} + path: tests/.cache/bench + if-no-files-found: ignore + retention-days: 90 diff --git a/.github/workflows/browsers.yml b/.github/workflows/browsers.yml new file mode 100644 index 0000000..28c93ce --- /dev/null +++ b/.github/workflows/browsers.yml @@ -0,0 +1,89 @@ +# The UI suite (tests/README.md) on dist/discpress.html in Firefox and in WebKit, Safari's engine (Playwright's +# projects behind EXTRA_BROWSERS: webkit, iphone-webkit and firefox), against unmodified chdman 0.289 and the +# engine built natively, as ci.yml runs it in Chromium. A test that fails in one of them for a reason the page +# can't fix is pinned there with test.fail() or skipped with its reason. When the page, the tests or this +# workflow change, weekly, and on demand. +name: Browsers + +on: + push: + branches: [main] + paths: ['app/**', 'dist/**', 'tests/**', 'web/**', '.github/workflows/browsers.yml'] + pull_request: + paths: ['app/**', 'dist/**', 'tests/**', 'web/**', '.github/workflows/browsers.yml'] + schedule: + - cron: '37 6 * * 2' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: browsers-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests: + runs-on: ubuntu-24.04 + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - browser: firefox + projects: --project=firefox + - browser: webkit + projects: --project=webkit --project=iphone-webkit + name: tests (${{ matrix.browser }}) + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: tests/package-lock.json + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + - name: numpy, for the synthetic test discs + run: pip install numpy==2.4.6 + # the references the tests compare with, as in ci.yml (the same cache) + - name: Cache MAME 0.289 and its chdman + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + third_party/mame + build/chdman-0.289 + key: mame-0.289-${{ runner.os }}-${{ hashFiles('scripts/fetch-mame.sh', 'scripts/build-upstream.sh', 'wasm/**') }} + - name: Unmodified chdman 0.289 + run: '[ -x build/chdman-0.289 ] || ./scripts/build-upstream.sh' + - name: The engine built natively + run: ./scripts/build-native.sh + - name: Cache Playwright's ${{ matrix.browser }} + id: pwcache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ matrix.browser }}-${{ runner.os }}-${{ hashFiles('tests/package-lock.json') }} + - name: Playwright and ${{ matrix.browser }} + working-directory: tests + run: | + npm ci --no-audit --no-fund + if [ '${{ steps.pwcache.outputs.cache-hit }}' = 'true' ]; then npx playwright install-deps ${{ matrix.browser }}; else npx playwright install --with-deps ${{ matrix.browser }}; fi + - name: Synthetic test discs + run: python3 tests/fixtures/make_fixtures.py + - name: UI tests in ${{ matrix.browser }} + working-directory: tests + env: + EXTRA_BROWSERS: '1' + REQUIRE_NATIVE: '1' + run: npx playwright test ${{ matrix.projects }} + - name: Report of a failed run + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: playwright-report-${{ matrix.browser }} + path: | + tests/.cache/report + tests/.cache/test-results + retention-days: 14 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 0000000..03aa805 --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,69 @@ +# Fuzzes the engine (scripts/fuzz.sh, tests/fuzz/README.md): every seed, past finding and saved corpus +# input once, then each fuzzer for a while: 30 minutes weekly, as long as asked on demand, 5 minutes when +# engine/, wasm/ or the fuzzers change. The corpus grows from run to run in the Actions cache (pruned to +# the inputs that add coverage); a finding fails the run and is uploaded with it. +name: Fuzz + +on: + push: + branches: [main] + paths: ['engine/**', 'wasm/**', 'tests/fuzz/**', 'tests/support/lsan.supp', 'scripts/fuzz.sh', '.github/workflows/fuzz.yml'] + pull_request: + paths: ['engine/**', 'wasm/**', 'tests/fuzz/**', 'tests/support/lsan.supp', 'scripts/fuzz.sh', '.github/workflows/fuzz.yml'] + schedule: + - cron: '23 4 * * 3' + workflow_dispatch: + inputs: + seconds: + description: Seconds each fuzzer runs (the four run at once) + default: '1800' + +permissions: + contents: read + +concurrency: + group: fuzz-${{ github.ref }} + cancel-in-progress: true + +jobs: + fuzz: + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + CC: clang-18 + CXX: clang++-18 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - name: clang and its libFuzzer runtime + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends clang-18 libclang-rt-18-dev + - name: The fuzzers and their seeds + run: ./scripts/fuzz.sh build + - name: Restore the corpus + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: build/fuzz/corpus + key: fuzz-corpus-${{ github.run_id }} + restore-keys: fuzz-corpus- + - name: Every seed, regression and corpus input once + run: ./scripts/fuzz.sh check all + - name: Fuzz + run: ./scripts/fuzz.sh run all ${{ github.event_name == 'schedule' && '1800' || (github.event_name == 'workflow_dispatch' && inputs.seconds) || '300' }} + - name: Prune the corpus + if: success() + run: ./scripts/fuzz.sh prune all + # (a pull request's cache is its own: only main's runs keep the corpus) + - name: Save the corpus + if: always() && github.event_name != 'pull_request' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: build/fuzz/corpus + key: fuzz-corpus-${{ github.run_id }} + - name: Findings + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: fuzz-findings + path: | + build/fuzz/findings + build/fuzz/*.log + if-no-files-found: ignore diff --git a/CLAUDE.md b/CLAUDE.md index c113897..2cf1a4e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,6 +26,8 @@ python3 scripts/assemble.py # [output.html], default dist/discpress.html ./scripts/engine-diff.sh # after changing engine/mame: refresh engine/mame-0.289.diff (--check to verify) ./scripts/build-sanitized.sh # -> build/chdman-san: engine/ with ASan, LSan and UBSan; then in tests/: # CHDMAN_ENGINE=../build/chdman-san LSAN_OPTIONS=suppressions=$PWD/support/lsan.supp npx playwright test ui/engine.spec.js +./scripts/fuzz.sh build # libFuzzer under ASan and UBSan (needs clang + libclang-rt--dev): tests/fuzz/README.md +./scripts/fuzz.sh run all 600 # the chd, desc, codec and ecm fuzzers at once; `check all` replays seeds, regressions, corpus # releases (see Releases below) ./scripts/check-dist.sh # dist/discpress.html is what app/, db/ and the engine sources make (the release refuses otherwise) @@ -51,11 +53,11 @@ Compiler experiments: `make -C wasm ... EXTRA=""` adds flags to every com ## Testing workflow -When changing `app/`: `npm run dev-page` (in `tests/`) re-assembles `build/discpress-dev.html` in under a second (first taking the wasm from `dist/discpress.html` when that is newer than `build/`'s and differs: `extract-build.py --if-stale`, which leaves a fresher `./build.sh` output alone), `npm run test:dev` runs the suite on it, and `npm run bench:dev -- --compare latest` checks performance. Tests fail on any console error or network request. Review UI changes with the screenshots the layout spec writes to `tests/.cache/screens//-.png`. Once satisfied, `python3 scripts/assemble.py` updates `dist/discpress.html`, which must be committed with the source change. CI (`.github/workflows/ci.yml`) runs `check-dist.sh`, `check-licenses.sh` (only BSD-3-Clause MAME files in `engine/`, `md5.h` public domain) `engine-diff.sh --check` and the whole suite with `REQUIRE_NATIVE=1` on every pull request and push to `main`; locally, set `REQUIRE_NATIVE=1` too, since without `build/chdman-0.289` and a `build/chdman-native` newer than `engine/` the byte-for-byte checks and `engine.spec.js` skip. Bugs found but not fixed yet are pinned with `test.fail()` in the relevant spec, or in the `ACCEPTED` list of `tests/ui/a11y.spec.js`; remove the marker when fixing one. Fixtures are synthetic (`tests/fixtures/`); never add real game data to the repo. +When changing `app/`: `npm run dev-page` (in `tests/`) re-assembles `build/discpress-dev.html` in under a second (first taking the wasm from `dist/discpress.html` when that is newer than `build/`'s and differs: `extract-build.py --if-stale`, which leaves a fresher `./build.sh` output alone), `npm run test:dev` runs the suite on it, and `npm run bench:dev -- --compare latest` checks performance. Tests fail on any console error or network request. Review UI changes with the screenshots the layout spec writes to `tests/.cache/screens//-.png`. Once satisfied, `python3 scripts/assemble.py` updates `dist/discpress.html`, which must be committed with the source change. `.github/workflows/browsers.yml` runs the suite in Firefox and WebKit too (`EXTRA_BROWSERS=1`, the `firefox`, `webkit` and `iphone-webkit` projects) when the page or the tests change, and weekly. CI (`.github/workflows/ci.yml`) runs `check-dist.sh`, `check-licenses.sh` (only BSD-3-Clause MAME files in `engine/`, `md5.h` public domain) `engine-diff.sh --check` and the whole suite with `REQUIRE_NATIVE=1` on every pull request and push to `main`; locally, set `REQUIRE_NATIVE=1` too, since without `build/chdman-0.289` and a `build/chdman-native` newer than `engine/` the byte-for-byte checks and `engine.spec.js` skip. Bugs found but not fixed yet are pinned with `test.fail()` in the relevant spec, or in the `ACCEPTED` list of `tests/ui/a11y.spec.js`; remove the marker when fixing one. Fixtures are synthetic (`tests/fixtures/`); never add real game data to the repo. -The build is reproducible: a clean build must produce a byte-identical `dist/discpress.html` (gzip uses `mtime=0`; `build.sh` fails if the SIMD and non-SIMD JS glue differ, and refuses any Emscripten but 6.0.10, commit `d6c521a7f…`, unless `EMSCRIPTEN_ANY=1`, for experiments only: such a build records no sources hash, so `check-dist.sh` and a release refuse its page). `.github/workflows/sanitizers.yml` runs `engine.spec.js` on the sanitized build (`scripts/build-sanitized.sh`) when `engine/`, `wasm/`, the spec, the fixtures or the suppressions change, and weekly; any sanitizer report fails a test, whatever chdman's exit code, except leaks unmodified chdman 0.289 has too (`tests/support/lsan.supp`, each with how that was checked; only MAME's work queue, `osdsync.cpp`, so far). `.github/workflows/rebuild.yml` checks it on another machine: it rebuilds with emsdk 6.0.10 and requires the committed page byte for byte, when `engine/`, `wasm/`, `build.sh`, `dist/` or the scripts that hash and assemble the page change, weekly and on demand (a clean build takes about a minute on 4 cores; machine paths don't reach the wasm). `dist/discpress.html` is committed and is what releases ship, so **regenerate and commit it whenever `app/`, `wasm/`, `engine/` or `db/` change**: after `app/` or `db/`, `assemble.py` is enough; after `engine/`, `wasm/` or `build.sh`, only `./build.sh` will do. The page records the sources its wasm was built from: `build.sh` hashes them (`scripts/engine-sources.py`: the contents of `build.sh` and of `engine/` and `wasm/`'s files that git tracks or would (an untracked file it doesn't ignore counts too), without `*.md`, `engine/FILES` and the diff; paths sorted, no dates) into `build/engine-sources.sha256`, which `assemble.py` writes after `` and `extract-build.py` recovers, and `scripts/check-dist.sh` refuses a page whose hash isn't the current sources'. +The build is reproducible: a clean build must produce a byte-identical `dist/discpress.html` (gzip uses `mtime=0`; `build.sh` fails if the SIMD and non-SIMD JS glue differ, and refuses any Emscripten but 6.0.10, commit `d6c521a7f…`, unless `EMSCRIPTEN_ANY=1`, for experiments only: such a build records no sources hash, so `check-dist.sh` and a release refuse its page). `.github/workflows/sanitizers.yml` runs `engine.spec.js` on the sanitized build (`scripts/build-sanitized.sh`) when `engine/`, `wasm/`, the spec, the fixtures or the suppressions change, and weekly; any sanitizer report fails a test, whatever chdman's exit code, except leaks unmodified chdman 0.289 has too (`tests/support/lsan.supp`, each with how that was checked; only MAME's work queue, `osdsync.cpp`, so far). `.github/workflows/fuzz.yml` fuzzes the engine (`scripts/fuzz.sh`, `tests/fuzz/`): CHD files, descriptors, compressed hunks and ECM data, with the browser-only code built in (`__EMSCRIPTEN__` defined, `tests/fuzz/host.cpp` doing the page's part), weekly for 30 minutes per fuzzer and for 5 when `engine/`, `wasm/` or the fuzzers change; the corpus is kept in the Actions cache, and each finding, once fixed, becomes an input in `tests/fuzz/regressions/` and, where it can be made from a fixture, a test in `engine.spec.js`. `.github/workflows/rebuild.yml` checks it on another machine: it rebuilds with emsdk 6.0.10 and requires the committed page byte for byte, when `engine/`, `wasm/`, `build.sh`, `dist/` or the scripts that hash and assemble the page change, weekly and on demand (a clean build takes about a minute on 4 cores; machine paths don't reach the wasm). `dist/discpress.html` is committed and is what releases ship, so **regenerate and commit it whenever `app/`, `wasm/`, `engine/` or `db/` change**: after `app/` or `db/`, `assemble.py` is enough; after `engine/`, `wasm/` or `build.sh`, only `./build.sh` will do. The page records the sources its wasm was built from: `build.sh` hashes them (`scripts/engine-sources.py`: the contents of `build.sh` and of `engine/` and `wasm/`'s files that git tracks or would (an untracked file it doesn't ignore counts too), without `*.md`, `engine/FILES` and the diff; paths sorted, no dates) into `build/engine-sources.sha256`, which `assemble.py` writes after `` and `extract-build.py` recovers, and `scripts/check-dist.sh` refuses a page whose hash isn't the current sources'. -To try the app, open `dist/discpress.html` directly in a browser (`file://` works). Setting `localStorage['chdman-web-debug'] = '{"stage":1}'` makes the worker copy every input into storage with async reads before running, and uses the main-thread CRC; `"stage":2` also makes those reads fail, so the page streams inputs to the worker (the iOS web view path, see below). For tests, `"identDelay"` and `"crcDelay"` (ms) slow identification and its checksum step, `"sinkMax"` (bytes) lowers that 64 MB limit, `"memMax"` (bytes) lowers the size above which a job whose results are kept in memory asks first, and `"failHelpers"` (`"one"`, `"all"`, `"always"`) makes helper workers trap as a miscompiling WebAssembly engine does (`"lie"`: the first one answers for other hunks than it was given). `"crcSlow"` (ms) delays the worker's checksum, `"shareMax"` (bytes) lowers `SHARE_MAX`, `"failStorage"` (`"lock"`, or anything else) makes the worker's storage fail as a locked or missing OPFS does, `"unzipFull"` fills the storage while a zip is unpacked, `"savePickerMin"` (bytes) lowers `SAVE_PICKER_MIN`, and `"hosted"` makes the page act as the online version (service worker, manifest) without https. +To try the app, open `dist/discpress.html` directly in a browser (`file://` works). Setting `localStorage['chdman-web-debug'] = '{"stage":1}'` makes the worker copy every input into storage with async reads before running, and uses the main-thread CRC; `"stage":2` also makes those reads fail, so the page streams inputs to the worker (the iOS web view path, see below). For tests, `"identDelay"` and `"crcDelay"` (ms) slow identification and its checksum step, `"sinkMax"` (bytes) lowers that 64 MB limit, `"memMax"` (bytes) lowers the size above which a job whose results are kept in memory asks first, and `"failHelpers"` (`"one"`, `"all"`, `"always"`) makes helper workers trap as a miscompiling WebAssembly engine does (`"lie"`: the first one answers for other hunks than it was given). `"crcSlow"` (ms) delays the worker's checksum, `"shareMax"` (bytes) lowers `SHARE_MAX`, `"failStorage"` (`"lock"`, or anything else) makes the worker's storage fail as a locked or missing OPFS does, `"unzipFull"` fills the storage while a zip is unpacked, `"savePickerMin"` (bytes) lowers `SAVE_PICKER_MIN`, `"storeDelay"` (ms) slows the page's check for private storage (`Store.ready`, which jobs wait for), `"freeze"` (ms) keeps a conversion's job worker silent before chdman starts and `"stallMs"` lowers the no-progress watchdog's `STALL_MS`, and `"hosted"` makes the page act as the online version (service worker, manifest) without https. ## How the single file is assembled @@ -72,9 +74,9 @@ Consequences: app JS is plain browser script (no modules/imports, ES5-style `var ## Runtime architecture -- **`app/ui.js`** (main thread, one IIFE): job model (files grouped into jobs by descriptor `.cue`/`.gdi`/etc.; a CloneCD `.ccd` is turned into a cue sheet for its `.img` by `ccdToCue`; a `.cso`/`.zso` becomes an input flagged `ciso`, given to chdman as `.iso`; an ECM image `.ecm` stands for ``, the track file a descriptor names or a lone `.bin` (`trackFile`), as an input flagged `ecm`; a PS1 `EBOOT.PBP` makes a job per disc, `sniffPbp` reading each disc's TOC into a cue sheet and the disc's image an input flagged `pbp`; a `.zip`'s entries join the list as picked files, `zipEntries`, stored ones as slices of it and deflated ones unpacked by `Unzip` into OPFS `chdman-unzip/` (else memory up to `UNZIP_MEM_MAX`) and checked against its CRC-32; formats it can't read (`UNSUPPORTED`: `.cdi`, `.mds`/`.mdf`, `.isz`, `.7z`, `.rar`) get a card saying why), building chdman argument lists per job (`buildJob`; the type and hunk size an identified console needs, the Zstd and MiSTer warnings (`noZstd`: its readers that can't decode Zstandard, for every preset that uses it; `noMister`: MiSTer has no core for it, for the MiSTer preset) and the notes for consoles whose emulators or players don't load CHDs (`keep`: the system doesn't set the CHD type) and the DOSBox Pure hint come from `PROFILES`, applied by `applyIdent`; Settings → Compatibility mode for older emulators (`settings.compat`, which a stored `ps2dvd: 'cd'` of 1.8 and earlier turns on; `COMPAT_NAME`) makes a PS2 game on DVD a CD CHD, the only kind AetherSX2, NetherSX2 and PCSX2 builds from before April 2023 read, and makes the Zstd presets warn for every console (`ZSTD_OLD`); rules for particular consoles and discs are `QUIRKS` in `app/quirks.js` (each with a stable id, shown as `data-quirk` on its note; `act: 'nokeepcue'` makes `buildJob` leave out `--keepcue`, as for Saturn and Sega CD, whose Kronos, Yabause and jgenesis reject it; `act: 'norename'` keeps the file's name, as for NAOMI romsets, through `renameWanted`; `warn` shows a note in the warning colour, and more than two other notes are grouped), looking at `quirkCtx` (the identification, `discModel`: the cue sheet's or TOC's layout from `cueModel`/`tocModel` in `ident.js`: tracks, sessions, INDEX 00, PREGAP and POSTGAP lines); a manifest entry's `quirks` lists the rule ids its card must show (`convert.spec.js`); compression never changes by itself; a PAL PS1 disc's `.sbi` (or DuckStation's `.lsd`) is paired with its job by name, `sbiJob`, and saved next to the CHD under its name, or made from a CloneCD `.sub`'s Q subchannel (`scanSub`, `sbiFromSub`), and is left out again when the disc turns out to be another console's (`detachSbi`); a LibCrypt game (`GameFacts`, from `db/facts/libcrypt.tsv`) without one warns (`libcrypt-missing`); a finished disc of a set says how many discs the set has (`GameDB.discCount`; `data-quirk="set-incomplete"`, set by `renderResult`, not a `QUIRKS` rule) until all are converted; two or more "(Disc N)" results of one game get an `.m3u` on the first disc's card, `discSet`), the Advanced tab CLI builder (`CMDS`/`OPT`), OPFS bookkeeping (`Store`), settings, and the `Engine` that runs commands. `Engine.run` starts one **job worker** per chdman command plus N **helper workers** connected to it by `MessageChannel`s. N comes from the thread setting: `'auto'` (default) uses `Tuning`, a one-off per-device speed test (helpers compress synthetic DVD hunks at 1, 2, 3, 4, 6, 8… threads until a step adds under 8%, twice below the reported core count or once above it; cached in `localStorage['chdman-web-tuning']` keyed by core count + user agent; up to 8 threads on touch devices (4 on iPhones), 16 elsewhere, regardless of the reported core count), run before the first compressing job; `threadCount()` resolves the setting. A finished CHD can be read back (`verifyResult`: chdman `verify` on the result, the card's Verify button), after every conversion with the `verifyAfter` setting and by itself after one during which a helper failed or the job ran again. A job whose results are kept in memory (no OPFS) asks before it runs when they could be over 2 GB (1 GB on touch devices; `memoryNeed`/`confirmMemory`, in the Advanced tab too). Where the browser has `showSaveFilePicker` (Chrome, Edge), Save asks where to put a result of 1 GB or more (`savePicked`, `SAVE_PICKER_MIN`), so a save canceled there keeps it unsaved; a plain download counts as saved once started. A started job can be canceled or removed before chdman runs (loading, identification, the speed test: `abortStart`), and a helper that fails to start fails the job instead of leaving it waiting. +- **`app/ui.js`** (main thread, one IIFE): job model (files grouped into jobs by descriptor `.cue`/`.gdi`/etc.; a CloneCD `.ccd` is turned into a cue sheet for its `.img` by `ccdToCue`; a `.cso`/`.zso` becomes an input flagged `ciso`, given to chdman as `.iso`; an ECM image `.ecm` stands for ``, the track file a descriptor names or a lone `.bin` (`trackFile`), as an input flagged `ecm`; a PS1 `EBOOT.PBP` makes a job per disc, `sniffPbp` reading each disc's TOC into a cue sheet and the disc's image an input flagged `pbp`; a `.zip`'s entries join the list as picked files, `zipEntries`, stored ones as slices of it and deflated ones unpacked by `Unzip` into OPFS `chdman-unzip/` (else memory up to `UNZIP_MEM_MAX`) and checked against its CRC-32; formats it can't read (`UNSUPPORTED`: `.cdi`, `.mds`/`.mdf`, `.isz`, `.7z`, `.rar`) get a card saying why), building chdman argument lists per job (`buildJob`; the type and hunk size an identified console needs, the Zstd and MiSTer warnings (`noZstd`: its readers that can't decode Zstandard, for every preset that uses it; `noMister`: MiSTer has no core for it, for the MiSTer preset) and the notes for consoles whose emulators or players don't load CHDs (`keep`: the system doesn't set the CHD type) and the DOSBox Pure hint come from `PROFILES`, applied by `applyIdent`; Settings → Compatibility mode for older emulators (`settings.compat`, which a stored `ps2dvd: 'cd'` of 1.8 and earlier turns on; `COMPAT_NAME`) makes a PS2 game on DVD a CD CHD, the only kind AetherSX2, NetherSX2 and PCSX2 builds from before April 2023 read, and makes the Zstd presets warn for every console (`ZSTD_OLD`); rules for particular consoles and discs are `QUIRKS` in `app/quirks.js` (each with a stable id, shown as `data-quirk` on its note; `act: 'nokeepcue'` makes `buildJob` leave out `--keepcue`, as for Saturn and Sega CD, whose Kronos, Yabause and jgenesis reject it; `act: 'norename'` keeps the file's name, as for NAOMI romsets, through `renameWanted`; `warn` shows a note in the warning colour, and more than two other notes are grouped), looking at `quirkCtx` (the identification, `discModel`: the cue sheet's or TOC's layout from `cueModel`/`tocModel` in `ident.js`: tracks, sessions, INDEX 00, PREGAP and POSTGAP lines); a manifest entry's `quirks` lists the rule ids its card must show (`convert.spec.js`); compression never changes by itself; a PAL PS1 disc's `.sbi` (or DuckStation's `.lsd`) is paired with its job by name, `sbiJob`, and saved next to the CHD under its name, or made from a CloneCD `.sub`'s Q subchannel (`scanSub`, `sbiFromSub`), and is left out again when the disc turns out to be another console's (`detachSbi`); a LibCrypt game (`GameFacts`, from `db/facts/libcrypt.tsv`) without one warns (`libcrypt-missing`); a finished disc of a set says how many discs the set has (`GameDB.discCount`; `data-quirk="set-incomplete"`, set by `renderResult`, not a `QUIRKS` rule) until all are converted; two or more "(Disc N)" results of one game get an `.m3u` on the first disc's card, `discSet`), the Advanced tab CLI builder (`CMDS`/`OPT`), OPFS bookkeeping (`Store`), settings, and the `Engine` that runs commands. `Engine.run` starts one **job worker** per chdman command plus N **helper workers** connected to it by `MessageChannel`s. N comes from the thread setting: `'auto'` (default) uses `Tuning`, a one-off per-device speed test (helpers compress synthetic DVD hunks at 1, 2, 3, 4, 6, 8… threads until a step adds under 8%, twice below the reported core count or once above it; cached in `localStorage['chdman-web-tuning']` keyed by core count + user agent; up to 8 threads on touch devices (4 on iPhones), 16 elsewhere, regardless of the reported core count), run before the first compressing job; `threadCount()` resolves the setting. A finished CHD can be read back (`verifyResult`: chdman `verify` on the result, the card's Verify button), after every conversion with the `verifyAfter` setting and by itself after one during which a helper failed or the job ran again. A job whose results are kept in memory (no OPFS) asks before it runs when they could be over 2 GB (1 GB on touch devices; `memoryNeed`/`confirmMemory`, in the Advanced tab too). Where the browser has `showSaveFilePicker` (Chrome, Edge), Save asks where to put a result of 1 GB or more (`savePicked`, `SAVE_PICKER_MIN`), so a save canceled there keeps it unsaved; a plain download counts as saved once started. A started job can be canceled or removed before chdman runs (loading, identification, the speed test: `abortStart`), and a helper that fails to start fails the job instead of leaving it waiting. A running conversion the page hears nothing from for 90 s while visible gets a note saying it may be stuck, until it goes on (`Engine.run`'s watchdog, `STALL_MS`, `onStall`; a line in the Advanced tab). - **Input guard rails** (`app/ui.js`): chdman 0.289 reads descriptors byte for byte with case-sensitive keywords, and loops forever when it finds no tracks. So `descriptorJob` decodes a descriptor itself (`decodeText`: BOM, UTF-16, Windows-1252), and `fixDescriptor` either gives chdman a corrected copy (LF line ends, cue keywords upper-cased) or refuses the job (`job.invalid`: no tracks, track numbers outside 1–99, CD+G tracks, which 0.289 rejects). The engine reads TOC files with cdrdao's grammar (a missing length is the rest of the file) and Nero images' stored pregaps; `probePlan` in `ident.js` finds data tracks the same way. File names in descriptors are split into words as chdman does (`chdWords`: single and double quotes); typed Advanced tab commands as a shell does (`shellWords`). Binary `.toc` files are skipped, raw-sector `.iso` files get an `autoCue`, and a `nan% complete` progress line stops the run (`STALLED`), in the Advanced tab too. A descriptor takes each track from its own folder, the subfolder its reference names, or a file picked on its own, never a same-named file from another folder or one already claimed (`pickTrack`: every GDI has `track01.bin`); names are compared in NFC (`nameKey`: a macOS name's é is two characters), through one index per batch of added files (`nameIndex`), so a folder of thousands of games is grouped at once; the duplicate check keys files by path, size and date (`fileKey`). -- **`app/worker.js`** (Web Worker, appended to the Emscripten glue): message roles `run` (run a chdman command line with `chdman_begin`/`chdman_resume`), `helper` (compress or decompress hunks for the job worker), `reader` (open a CHD and serve sectors for identification) and `crc`. It mounts a custom Emscripten filesystem (`makeFS`) over storage backends: `BlobStore` (read inputs in place via `FileReaderSync`), `OpfsStore` (outputs to Origin Private File System), `MemStore` (fallback), `CisoStore` (a CSO v1/v2 or ZSO compressed ISO read as the ISO inside, over any of the others: deflate blocks through the wasm's `wasm_inflate_raw`, LZ4 in JS; also used by the `reader` and `crc` roles to identify it), `PbpStore` (a PS1 PBP's disc read as its raw image: 16-sector blocks, deflated through `wasm_inflate_raw` or stored, from the disc's block table), `EcmStore` (an ECM image read as the CD image inside: a scan of its chunk headers places a checkpoint every 1 MiB of the image, and each window is rebuilt, sync, headers, EDC and ECC, by `wasm_ecm_decode` in `wasm/ecm.cpp`; read in order to its end, the image's EDC must match the file's, else the job stops; the scan reports how far it got, `scan` messages, which the card shows and which keep identification's reader from timing out; the `reader` role serves ECM images' bytes for identification, the `crc` role their checksum), `CrcStore` (outMode `crc`: outputs only checksummed, which `redumpCheck` in `ui.js` uses after Verify to compare a CHD with Redump via `extractcd --redump`), and `StreamStore` (outputs streamed to a user-chosen folder: `s-*` messages handled by `makeSink` in `ui.js` via the File System Access API; a file already in the folder is replaced only after `confirmReplace` asks, and is never deleted on cancel or failure; the page acknowledges each write (`s-ack`), and while more than 64 MB is unwritten the worker pauses chdman at its pause points, via the browser-only engine hook `wasm_out_full`). When a worker can't read picked files (some iOS file-viewer web views), it sends `stage-request` and the page streams chunks (`stage-chunk`/`stage-ack`/`stage-end`). +- **`app/worker.js`** (Web Worker, appended to the Emscripten glue): message roles `run` (run a chdman command line with `chdman_begin`/`chdman_resume`), `helper` (compress or decompress hunks for the job worker), `reader` (open a CHD and serve sectors for identification) and `crc`. It mounts a custom Emscripten filesystem (`makeFS`) over storage backends: `BlobStore` (read inputs in place via `FileReaderSync`), `OpfsStore` (outputs to Origin Private File System), `MemStore` (fallback), `CisoStore` (a CSO v1/v2 or ZSO compressed ISO read as the ISO inside, over any of the others: deflate blocks through the wasm's `wasm_inflate_raw`, LZ4 in JS; also used by the `reader` and `crc` roles to identify it), `PbpStore` (a PS1 PBP's disc read as its raw image: 16-sector blocks, deflated through `wasm_inflate_raw` or stored, from the disc's block table), `EcmStore` (an ECM image read as the CD image inside: a scan of its chunk headers places a checkpoint every 1 MiB of the image, and each window is rebuilt, sync, headers, EDC and ECC, by `wasm_ecm_decode` in `wasm/ecm.cpp`; read in order to its end, the image's EDC must match the file's, else the job stops; the scan reports how far it got, `scan` messages, which the card shows and which keep identification's reader from timing out; the `reader` role serves ECM images' bytes for identification, the `crc` role their checksum), `CrcStore` (outMode `crc`: outputs only checksummed, which `redumpCheck` in `ui.js` uses after Verify to compare a CHD with Redump via `extractcd --redump`), and `StreamStore` (outputs streamed to a user-chosen folder: `s-*` messages handled by `makeSink` in `ui.js` via the File System Access API; a file already in the folder is replaced only after `confirmReplace` asks, and is never deleted on cancel or failure; the page acknowledges each write (`s-ack`), and while more than 64 MB is unwritten the worker pauses chdman at its pause points, via the browser-only engine hook `wasm_out_full`). When a worker can't read picked files (some iOS file-viewer web views), it sends `stage-request` and the page streams chunks (`stage-chunk`/`stage-ack`/`stage-end`). Every role instantiates the engine first and then says `wasm-ready`; the page terminates workers only through `Engine.stop`, which waits for that (or 10 s), since Firefox's tab crashes when a worker is terminated while it instantiates (a helper failing to start stops the job while the others start). - **Multi-core compression** doesn't use pthreads/SharedArrayBuffer (so no COOP/COEP headers are needed). The engine has hooks in `chd_file_compressor` (`engine/mame/src/lib/util/chd.cpp`); `wasm/par_lib.js` (Emscripten JS library) hands hunks to `Module.parSubmit`, each with the codec slots to try (all of them, unless `--codecplan`, the opt-in "Nearly as small, faster" preset, narrows them by what the hunk holds: `chd_file_compressor::codec_plan_codecs`; with `--libdeflate`, in both faster presets, the deflate codec uses `engine/libdeflate` and `wasm_par_enabled` tells the helpers), and helper workers run a second wasm instance whose `wasm_helper_*` exports (`wasm/wasm_helper.cpp`) compress hunks exactly as `chd_file_compressor` would. The job worker keeps each batch until a helper answers it, and takes an answer only for that batch, item by item, each no longer than a hunk (else the helper is dropped): a helper that fails (a WebAssembly trap; some WebKit versions' optimizing compiler miscompiles, `docs/ios/README.md`) is dropped and the others do its batches; with none left the page runs the job once more (`engineFailure`), then says to update the browser (an extract or verify job decompresses by itself instead). The commands that compress are C++20 coroutines (`chdman_task` in `chdman.cpp`, no Asyncify): the worker starts a command with `chdman_begin` and, while it returns -1 (paused after a compression step), awaits `Module.parWait()` and calls `chdman_resume`; natively `main` runs them straight through. New exported C functions must be added to `EXPORTED_FUNCTIONS` in `wasm/link.sh`. - **Multi-core extract and verify** work the same way in reverse. `verify` and the extract commands are coroutines too, and before each buffer's worth of reading they `co_await read_ahead(...)`. `chd_file::wasm_read_ahead` (`chd.cpp`, browser only) reads the stored data of that window and the next in one file read (small reads are expensive in the browser). It hands codec-compressed hunks to helpers (`wasm_rd_submit` → `Module.rdSubmit`; results come back through `wasm_rd_slot`/`wasm_rd_done`), and pauses the command until the window is in (`Module.rdWait()`). `read_hunk` then serves hunks from memory with chdman's usual CRC check, so errors are unchanged. With one thread it keeps the window's compressed bytes and decompresses them when read. Verify and extract jobs get `readHelpers()`: the thread setting, but at most one fewer than the cores, since the job worker stays busy. - **Game identification** (`app/ident.js`, runs on the main thread): reads boot data (IP.BIN, SYSTEM.CNF, PARAM.SFO, …) via ISO 9660 from raw images, or from inside CHDs via the worker `reader` role backed by `wasm_probe_*` in `wasm_helper.cpp`; matches serial, or size + CRC-32 (computed in a worker with zlib's `crc32` exported from the wasm, JS fallbacks in the worker and on the page), against the database. `detectTrack` checks, in order: Sega headers, 3DO, GameCube/Wii, PC Engine and PC-FX boot sectors, CD-i's own volume descriptor, then the ISO 9660 volume (PSP and UMD Video, PS1/PS2 with `hasUdf` for a PS2 DVD, PS3, Video CD, CD-i Bridge, Neo Geo CD, CDTV/CD32, Xbox, video discs, else a weak `pc`), and without one an Xbox partition; PC-98 has no detector and is found by size + CRC-32 only; a cue sheet of audio tracks only is checked for a Jaguar CD header (`detectJaguar`, `plan.audio`). Serials are keyed per system by `canonKey` (Sega's `MK-51064` is Redump's `51064`), a disc is looked up under `serialCandidates` (its serial, without a version suffix, a PlayStation boot file's name), and `bySerial` in `identifyJob` ranks the releases found: the disc of a set Sega headers name (`det.disc`), then size, how the serial matched, and region (`det.area`, `regionFits`); a release found only under a trimmed or base serial, without a matching size, names nothing when its region doesn't fit, and when a Sega header's title doesn't fit it either it is only offered (`serial-related`, with Use this name: Saturn's T-25406H is Hexen, T-25406H-50 Doom); the ident note's `data-method`, `data-how` (the serial that found it: `serial`, `trim`, `boot` or `base`), `data-disc` and `data-region` say what it found. Removing a card stops its checksum (`abortChecksums`), so the next card's identification goes on. Identification is two-phase: once console/serial/sizes are known, `onProvisional` lets a conversion start (`job.identKnown`); the checksum then confirms the exact release and `renameOutputs` renames finished results if it differs (not when results were already saved, named by the user, or written straight into a folder, which waits for the checksum). `db/db.json.gz` is `{version, systems: {key: "name\tserial\tsize\tcrc\ttrack\text" rows}}`, a release listed under several serials once, with them joined by `|` (`entry.serials`; `entry.serial` is the first); system keys in `db/mkdb.py` must be keys of `SYSTEMS` in `ident.js` (which also has systems without database rows: `gc`, `wii`, `xbox`, `ps3`, `vcd`, `video`, `pc`). The user's own DAT files (Logiqx XML or clrmamepro, Redump or No-Intro) are parsed by `parseDat` into `Dats` and kept in OPFS `chdman-dats/` (`DatStore` in `ui.js`); `datCheck` matches every track of a job by size and CRC-32 (`datMatch`), a CHD's after Verify from `extractcd --redump`'s checksums, and a full match names the result (`datApply`). The rules in `quirks.js` add what common emulators can't load (PCSX2: PS2 CDs with audio; Flycast 2.7: CD-based Dreamcast discs with pregaps). @@ -85,7 +87,7 @@ Consequences: app JS is plain browser script (no modules/imports, ES5-style `var - **Saving.** The share sheet loads the whole file into memory, so on iOS results over `SHARE_MAX` (512 MB) are downloaded instead. App web views (`iosWebView`) usually can't download, so `#iosTip` points to the hosted copy. - **Edge blur.** Since iOS 26 a Home Screen web app gets a system blur over its top edge unless a fixed or sticky box with a plain background colour and no backdrop filter spans it (WebKit's `LocalFrameView::fixedContainerEdges`), so in `mode-shell` the header is `position: sticky` (nothing scrolls it there). `tests/ui/mobile.spec.js` replays the rule. - **Threads.** The speed test tries at most 4 threads on iPhones (`iosThreadCap`), which report 4 cores whatever they have; a stored result that stopped at a lower limit (`cap`) is measured again. - - **Recovery.** `Recovery` keeps a `localStorage` record of the running job and unsaved results, so after iOS reloads the page, `Store.cleanupStale` keeps those results and `renderEarlier` lists them. Where results live in memory (Chrome and Edge give a `file://` page no OPFS), the next visit reports what was lost instead (`Recovery.lost`). Each visit holds a Web Lock named after its session, so another open tab's record is left alone. + - **Recovery.** `Recovery` keeps a `localStorage` record of the running job and unsaved results, so after iOS reloads the page, `Store.cleanupStale` keeps those results and `renderEarlier` lists them. Where results live in memory (Chrome and Edge give a `file://` page no OPFS), the next visit reports what was lost instead (`Recovery.lost`). Each visit holds a Web Lock named after its session, so another open tab's record is left alone. Once a reload or close has begun, WebKit fails a job worker's reads of the picked files; a job that fails after `beforeunload` keeps its record, which goes only if the page is still there 3 s later (`Recovery.ended`). - **`wasm/`**: `sources.mk` lists the engine sources that are compiled; `shim/SDL2/SDL.h` stubs the only SDL calls chdman needs; `version.cpp` supplies the version strings. ## Releases diff --git a/app/ident.js b/app/ident.js index cd8049c..7b18e1a 100644 --- a/app/ident.js +++ b/app/ident.js @@ -223,7 +223,7 @@ function readerWorker(msg, timeout, onScan) { clearTimeout(timer); if (dead) return; dead = err; - w.terminate(); + Engine.stop(w); if (!ready) reject(err); Object.keys(waiting).forEach(function (id) { var q = waiting[id]; delete waiting[id]; clearTimeout(q.timer); q.reject(err); }); } @@ -356,8 +356,8 @@ function crcWorker(blob, start, end, onProgress, how) { var w = new Worker(Engine.url); var entry = { blob: blob, - stop: function () { w.terminate(); var e = new Error('paused'); e.paused = true; reject(e); }, - abort: function () { w.terminate(); var e = new Error('stopped'); e.aborted = true; reject(e); } + stop: function () { Engine.stop(w); var e = new Error('paused'); e.paused = true; reject(e); }, + abort: function () { Engine.stop(w); var e = new Error('stopped'); e.aborted = true; reject(e); } }; var resolve = function (v) { crcRunning.delete(entry); resolveRaw(v); }; var reject = function (e) { crcRunning.delete(entry); rejectRaw(e); }; @@ -366,11 +366,11 @@ function crcWorker(blob, start, end, onProgress, how) { w.onmessage = function (e) { var m = e.data; if (m.type === 'crc-progress') onProgress && onProgress(m.done / m.total); - else if (m.type === 'crc') { w.terminate(); resolve(m.crc); } + else if (m.type === 'crc') { Engine.stop(w); resolve(m.crc); } else if (m.type === 'notice') { if (m.level === 'error' && !why) why = m.message; } - else if (m.type === 'fatal') { w.terminate(); reject(new Error(why || m.message)); } + else if (m.type === 'fatal') { Engine.stop(w); reject(new Error(why || m.message)); } }; - w.onerror = function (e) { w.terminate(); reject(new Error(e.message || 'worker error')); }; + w.onerror = function (e) { Engine.stop(w); reject(new Error(e.message || 'worker error')); }; var pbp = /^pbp@(\d+):(\d+)$/.exec(how || ''); // a PS1 PBP's disc: its header's offset and its sectors Engine.post(w, { type: 'crc', blob: blob, start: start || 0, end: end, ciso: how === 'ciso', ecm: how === 'ecm', pbp: pbp ? { at: +pbp[1], sectors: +pbp[2] } : null, debugSlow: DEBUG.crcSlow }); // with the compiled module, for zlib's crc32 }); diff --git a/app/ui.js b/app/ui.js index fe5569e..f052dea 100644 --- a/app/ui.js +++ b/app/ui.js @@ -324,10 +324,29 @@ var Engine = { this.url = URL.createObjectURL(new Blob([src], { type: 'text/javascript' })); return this; }, + // Firefox's tab crashes when a worker is terminated while it instantiates its WebAssembly (as when a helper + // that fails to start stops the job at once, its siblings still starting): a worker given the engine here is + // starting until it says otherwise ('wasm-ready', or 'fatal'), and Engine.stop terminates one still starting + // once it has, or in 10 s post: function (w, msg, transfer) { + if (!w.engineWatch) { + w.engineWatch = true; + w.engineStarting = true; + var started = function () { + w.engineStarting = false; + if (w.engineStop) { w.engineStop = false; w.terminate(); } + }; + w.addEventListener('message', function (e) { var t = e.data && e.data.type; if (t === 'wasm-ready' || t === 'fatal') started(); }); + w.addEventListener('error', started); + } try { msg.wasmModule = this.module; w.postMessage(msg, transfer || []); } catch (e) { delete msg.wasmModule; msg.wasmBytes = this.bytes; w.postMessage(msg, transfer || []); } }, + stop: function (w) { + if (!w.engineStarting) { w.terminate(); return; } + w.engineStop = true; + setTimeout(function () { w.terminate(); }, 10000); + }, // runs one chdman command; returns {promise, cancel} run: function (o) { var self = this, worker = null, helpers = [], finished = false, saving = false, rejectFn; @@ -344,6 +363,19 @@ var Engine = { } }) : null; var ackWait = null; + // no-progress watchdog: any message from the job worker is progress; only time the page is + // visible counts (a hidden page's workers may be paused, iOS pauses them all). o.onStall(true) + // after STALL_MS of none, o.onStall(false) when one comes again + var quiet = 0, lastTick = Date.now(), quietSaid = false, watch = o.onStall ? setInterval(function () { + var now = Date.now(); + if (document.visibilityState === 'visible') quiet += now - lastTick; + lastTick = now; + if (!quietSaid && quiet >= (DEBUG.stallMs || STALL_MS)) { quietSaid = true; o.onStall(true); } + }, 1000) : 0; + function heard() { + quiet = 0; + if (quietSaid) { quietSaid = false; o.onStall(false); } + } // the worker could not read an input itself: read it here and send it over async function feedStage(m) { var blob = o.inputs[m.index].blob, step = 4 << 20; @@ -360,10 +392,11 @@ var Engine = { } function cleanup() { finished = true; + if (watch) { clearInterval(watch); watch = 0; } // a chunk still waiting for the worker's answer: its staging ends (and lets go of the file) if (ackWait) { var a = ackWait; ackWait = null; a(); } - if (worker) worker.terminate(); - helpers.forEach(function (h) { h.terminate(); }); + if (worker) Engine.stop(worker); + helpers.forEach(function (h) { Engine.stop(h); }); helpers = []; } var promise = new Promise(function (resolve, reject) { @@ -394,6 +427,7 @@ var Engine = { worker.onmessage = function (e) { var m = e.data; if (finished) return; + heard(); if (m.type.charAt(0) === 's' && m.type.charAt(1) === '-') { if (sink) sink.handle(m, worker); } else if (m.type === 'stage-request') feedStage(m); else if (m.type === 'stage-ack') { var a = ackWait; ackWait = null; if (a) a(); } @@ -425,7 +459,8 @@ var Engine = { }; self.post(worker, { type: 'run', jobId: o.jobId, dirPath: o.dirPath, args: o.args, inputs: o.inputs || [], - writable: o.writable || [], slots: o.slots || 0, outMode: o.outMode || 'mem', ports: ports, debugStage: DEBUG.stage, debugSinkMax: DEBUG.sinkMax, debugStorage: DEBUG.failStorage + writable: o.writable || [], slots: o.slots || 0, outMode: o.outMode || 'mem', ports: ports, debugStage: DEBUG.stage, debugSinkMax: DEBUG.sinkMax, debugStorage: DEBUG.failStorage, + debugFreeze: o.onStall ? DEBUG.freeze : 0 }, ports); }); return { @@ -539,7 +574,7 @@ var Tuning = { else if (++misses >= (n > cores ? 1 : 2)) break; } } finally { - pool.forEach(function (h) { h.w.terminate(); }); + pool.forEach(function (h) { Engine.stop(h.w); }); document.removeEventListener('visibilitychange', onVisibility); } var t = { key: this.key(), threads: bestN, rate: +best.toFixed(2), steps: steps, cores: cores, cap: Tuning.cap, date: Date.now() }; @@ -582,6 +617,7 @@ async function heldLocks() { } var Store = { available: false, + ready: null, // init's promise: a job waits for it before choosing where its results go session: 's' + Date.now().toString(36) + uid(), init: async function () { // held while this visit lasts, so other visits can tell it is still open (Recovery) @@ -590,6 +626,7 @@ var Store = { await sleep(50); } var work = null; + if (DEBUG.storeDelay) await sleep(DEBUG.storeDelay); // testing: a slow check (Firefox's can be) try { if (!navigator.storage || !navigator.storage.getDirectory) throw new Error('no private storage'); var root = await navigator.storage.getDirectory(); @@ -714,8 +751,18 @@ var Recovery = { this.write(all); }, running: function (job) { + this.current = job || null; this.update(function (rec) { rec.running = job ? job.title : null; }); }, + leaving: false, // beforeunload was seen: the page may be going away (or the visitor chose to stay) + // the running job ended without results. Once a reload or close has begun, WebKit fails a job's reads + // before the page is gone, which ends the job: then its record stays, so that the next visit still says + // it didn't finish, and goes only if the page is still here 3 s later (and runs nothing newer) + ended: function (job) { + var self = this; + if (!this.leaving) { this.running(null); return; } + setTimeout(function () { if (self.current === job) self.running(null); }, 3000); + }, // results in memory ('blob') are recorded too: a reload loses them, and the next visit says so finished: function (job) { var outs = job.outputs.filter(function (o) { return (o.kind === 'opfs' || o.kind === 'blob') && !o.downloaded && !o.input; }); @@ -2323,6 +2370,8 @@ function renderControls(job) { if (settings.compat && /(^|,)(cdzs|zstd)(,|$)/.test(cur[2] || '')) { chint += ' ' + ZSTD_OLD; zwarn = true; } if (cur[0] === 'mister' && pfc.noMister) { chint += ' MiSTer has no core that plays ' + sysName(job.ident.sys) + ' games from CHDs.'; zwarn = true; } if (pfc.hint && cur[0] !== 'none') chint += ' ' + pfc.hint; + // phones and tablets take minutes over a disc: the preset that saves a third of that, suggested (never chosen) + if (isMobile && cur[0] === 'default' && presets.some(function (p) { return p[0] === 'plan'; })) chint += ' ' + PLAN_HINT; var cfield = selectField('Compression', presets.map(function (p) { return [p[0], p[1]]; }), o.preset, function (v) { o.preset = v; refresh(); }, chint, busy); if (zwarn) cfield.lastChild.style.color = 'var(--warn)'; fields.append(cfield); @@ -2393,6 +2442,7 @@ function renderNotes(job) { else box.append(el('div', { class: 'note warn' }, 'This CHD depends on a parent CHD (SHA-1 ' + job.info.parent.slice(0, 12) + '…). Add the parent .chd file too.')); } if (job.state === 'error' && job.errorText) box.append(el('div', { class: 'note err' }, job.errorText)); + if (job.state === 'running' && job.noProgress) box.append(el('div', { class: 'note warn', 'data-note': 'no-progress' }, el('b', null, 'No progress for over a minute. '), STALL_HINT)); if (job.state === 'canceled') box.append(el('div', { class: 'note' }, 'Canceled.')); box.hidden = !box.childNodes.length; } @@ -2719,6 +2769,11 @@ async function pump() { } } +// the compression hint on phones and tablets (renderControls) +var PLAN_HINT = 'On a phone or tablet, Nearly as small, faster takes about a third less time, for files at most 0.3% bigger.'; +// the no-progress watchdog (Engine.run): a running job's card says so after this long without a word from it +var STALL_MS = 90000; +var STALL_HINT = 'The browser may have paused it, or it may be stuck. You can wait, or cancel it and try again, with fewer CPU threads (Settings) if it stops again.'; // chdman never finishes when it finds no data: its progress then reads "nan% complete" (0 of 0) var STALLED = 'chdman found no data to convert in these files and would never have finished, so it was stopped. Check that the .cue, .gdi or .toc file lists the disc\u2019s tracks.'; function stalledProgress(t) { return /\bnan% complete/i.test(t); } @@ -2737,6 +2792,7 @@ async function runJobNow(job) { return true; } job.stalled = false; + job.noProgress = false; job.log = job.retryLog || []; // a run again after the engine failed keeps the first run's log job.retryLog = null; job.outputs = []; @@ -2765,6 +2821,8 @@ async function runJobNow(job) { if (job.state !== 'running') return; } if (job.subScan && await stopped(job.subScan)) return; + // (whether private storage is there, which a job started just after the page opened might not know yet) + if (Store.ready) await Store.ready; var spec = buildJob(job); if (settings.storage === 'folder' && outDir && (spec.rename || spec.outMode === 'stream')) { var oi = spec.args.indexOf('-o'), main = spec.rename ? outBase(job) + '.chd' : oi >= 0 ? spec.args[oi + 1].replace(/^.*\//, '') : ''; @@ -2846,6 +2904,7 @@ async function runJobNow(job) { if (/^A helper thread stopped/.test(m.message)) job.helperStopped = true; }, onScan: function (f) { setProgress(job, null, 'Reading the ECM image\u2026 ' + Math.floor(f * 100) + '%'); }, + onStall: function (on) { job.noProgress = on; if (job.ui) renderNotes(job); }, onStorage: function (mode, reason) { job.storage = mode; if (spec.outMode !== 'opfs' || mode !== 'mem') return; @@ -2884,7 +2943,7 @@ async function runJobNow(job) { await verifyResult(job); } } else { - Recovery.running(null); + Recovery.ended(job); job.state = 'error'; if (res.readFail) { job.errorText = readFailHint(res.error); Store.removeJob(job.id); refreshJob(job, true); return; } // (a helper that stopped, whose work the others did, isn't why) @@ -2897,7 +2956,7 @@ async function runJobNow(job) { Store.removeJob(job.id); } } catch (e) { - Recovery.running(null); + Recovery.ended(job); if (e.canceled && job.stalled) { job.state = 'error'; job.errorText = STALLED; } else if (e.canceled && job.storageRefused) { job.state = 'error'; job.errorText = 'The browser\u2019s storage wasn\u2019t available for this job, so it stopped rather than keep its results in memory. Try again in a moment.'; } else if (e.canceled) { job.state = 'canceled'; } @@ -3824,6 +3883,7 @@ async function cliRun() { // results kept in memory, as for a conversion: ask first when they could be too large (what it reads, // twice that for an extraction, and the copies of inputs chdman changes) var cliOut = /^(info|listtemplates|verify|addmeta|delmeta|dumpmeta)$/.test(args[0]) ? 0 : (/^extract/.test(args[0]) ? 2 : 1) * inputs.reduce(function (n, i) { return n + i.blob.size; }, 0); + if (Store.ready) await Store.ready; var cliMem = !(settings.storage === 'folder' && outDir) && (settings.storage === 'memory' || !Store.available); if (!confirmMemory(memoryNeed({ outMode: cliMem ? 'mem' : 'opfs', slots: cliOut ? 1 : 0, expected: cliOut, writable: writable }))) return; // the last command's results give way to this one's (only one command's results are shown) @@ -3865,7 +3925,8 @@ async function cliRun() { var m = /([\d.]+)% complete/.exec(t); if (m) { prog.querySelector('.bar').classList.remove('indet'); prog.querySelector('.bar').setAttribute('aria-valuenow', Math.round(+m[1])); barI.style.width = m[1] + '%'; ptext.textContent = t.trim(); } }, - onNotice: function (m) { if (m.level !== 'debug') con.textContent += m.message + '\n'; } + onNotice: function (m) { if (m.level !== 'debug') con.textContent += m.message + '\n'; }, + onStall: function (on) { if (on) con.textContent += '[No progress for over a minute] ' + STALL_HINT + '\n'; } }); var r = await cli.run.promise; con.textContent += '\n[exit code ' + r.code + (r.error ? ': ' + r.error : '') + ']\n'; @@ -4347,6 +4408,7 @@ function init() { }); window.addEventListener('beforeunload', function (e) { + Recovery.leaving = true; var busy = jobs.some(function (j) { return j.state === 'running' || j.state === 'queued'; }) || !!cli.run; var left = jobs.some(unsaved) || (cli.last && unsaved(cli.last)); if (busy || left) { e.preventDefault(); e.returnValue = ''; } @@ -4356,7 +4418,8 @@ function init() { updateDock(); updateChips(); - Store.init().then(function () { updateChips(); renderEarlier(); DatStore.load(); }); + Store.ready = Store.init(); + Store.ready.then(function () { updateChips(); renderEarlier(); DatStore.load(); }); $('#datAdd').addEventListener('click', function () { pickFiles('datInput', function (l) { addEntries(filesFromList(l)); }); }); initHosted(); initIosTip(); diff --git a/app/worker.js b/app/worker.js index 706b307..e604b57 100644 --- a/app/worker.js +++ b/app/worker.js @@ -37,7 +37,8 @@ function instantiate(extra) { var opts = { noInitialRun: true, instantiateWasm: function (imports, receive) { - WebAssembly.instantiate(wasmModule, imports).then(function (inst) { receive(inst, wasmModule); }, + // (the page may terminate this worker once it hears 'wasm-ready', or 'fatal': Engine.stop) + WebAssembly.instantiate(wasmModule, imports).then(function (inst) { postMessage({ type: 'wasm-ready' }); receive(inst, wasmModule); }, function (err) { postMessage({ type: 'fatal', message: 'Could not start WebAssembly: ' + err }); }); return {}; }, @@ -1334,8 +1335,18 @@ async function runChdman(M, args) { async function runJob(msg) { if (msg.debugSinkMax) SINK_MAX = msg.debugSinkMax; // testing: a small limit await loadModule(msg); + // (the engine first: until it is instantiated the page holds off terminating this worker, Engine.stop) + var out = makeLineSink(1), err = makeLineSink(2), aborted = null; + var M = await instantiate({ + stdin: function () { return null; }, + stdout: function (c) { out.put(c); }, + stderr: function (c) { err.put(c); }, + onAbort: function (what) { aborted = 'chdman stopped unexpectedly: ' + what; } + }); var backing = await makeBacking(msg); postMessage({ type: 'storage', mode: backing.mode, reason: memReason }); + // testing: silent this long, as a stuck job is (the page's no-progress watchdog, DEBUG.freeze) + if (msg.debugFreeze) await new Promise(function (r) { setTimeout(r, msg.debugFreeze); }); // copies of inputs chdman must modify in place (verify --fix, addmeta, delmeta) var outFiles = []; @@ -1387,14 +1398,6 @@ async function runJob(msg) { } } - var out = makeLineSink(1), err = makeLineSink(2), aborted = null; - var M = await instantiate({ - stdin: function () { return null; }, - stdout: function (c) { out.put(c); }, - stderr: function (c) { err.put(c); }, - onAbort: function (what) { aborted = 'chdman stopped unexpectedly: ' + what; } - }); - inStores.forEach(function (st) { if (st.store.setModule) st.store.setModule(M); }); backing.M = M; M.outFull = sinkFull; // wasm_out_full: chdman pauses while a folder is behind diff --git a/dist/discpress.html b/dist/discpress.html index f9bec8b..9cc4ebf 100644 --- a/dist/discpress.html +++ b/dist/discpress.html @@ -1306,6 +1306,18 @@

About and licenses

if(bps >= 0.0) return bps; else +--- a/3rdparty/flac/src/libFLAC/stream_encoder.c ++++ b/3rdparty/flac/src/libFLAC/stream_encoder.c +@@ -4126,7 +4126,8 @@ + residual, + #endif + abs_residual_partition_sums+sum, +- raw_bits_per_partition+sum, ++ /* Discpress: null without escape coding, which C doesn't let an offset be added to */ ++ raw_bits_per_partition != NULL ? raw_bits_per_partition+sum : NULL, + residual_samples, + predictor_order, + rice_parameter_limit, --- a/3rdparty/flac/src/libFLAC/window.c +++ b/3rdparty/flac/src/libFLAC/window.c @@ -39,6 +39,7 @@ @@ -1496,6 +1508,63 @@

About and licenses

*destLen -= (SizeT)outStream.rem; if (outStream.overflow) +--- a/src/lib/util/avhuff.cpp ++++ b/src/lib/util/avhuff.cpp +@@ -67,6 +67,7 @@ + #include <cmath> + #include <cstdlib> + #include <cstring> ++#include <iterator> + #include <new> + + +@@ -723,6 +724,11 @@ + uint32_t width = get_u16be(&source[4]); + uint32_t height = get_u16be(&source[6]); + ++ // Discpress: no more channels than there are places for (the encoder takes up to 16): 0.289 went past ++ // the end of audiostart and of m_config.audio ++ if (channels > std::size(m_config.audio)) ++ return AVHERR_INVALID_DATA; ++ + // validate that the sizes make sense + if (complength < 10 + 2 * channels) + return AVHERR_INVALID_DATA; +@@ -932,6 +938,10 @@ + // if no huffman length, just copy the data + if (treesize == 0) + { ++ // Discpress: as much as the samples take (as stored): 0.289 read past the channel's ++ // data, and past the end of the hunk's ++ if (size < 2 * samples) ++ return AVHERR_INVALID_DATA; + const uint8_t *cursource = source; + for (int sampnum = 0; sampnum < samples; sampnum++) + { +--- a/src/lib/util/bitstream.h ++++ b/src/lib/util/bitstream.h +@@ -146,7 +146,8 @@ + + inline void bitstream_in::remove(int numbits) + { +- m_buffer <<= numbits; ++ // (all 32 bits, which a crafted v5 map's field width can ask for: the buffer is empty, not shifted by 32) ++ m_buffer = (numbits < 32) ? (m_buffer << numbits) : 0; + m_bits -= numbits; + } + +@@ -226,6 +227,11 @@ + + inline void bitstream_out::write(uint32_t newbits, int numbits) + { ++ // Discpress: nothing to write, as for a CHD map with no copies of hunks (0.289 went on to shift ++ // the value by 32 bits, which C++ leaves undefined, and then returned without using it) ++ if (numbits <= 0) ++ return; ++ + newbits <<= 32 - numbits; + + // flush the buffer if we're going to overflow it --- a/src/lib/util/cdrom.cpp +++ b/src/lib/util/cdrom.cpp @@ -28,6 +28,8 @@ @@ -1561,7 +1630,85 @@

About and licenses

const uint32_t index_offset = frame - track_start; int index = 0; -@@ -1153,204 +1177,106 @@ +@@ -897,6 +921,13 @@ + * @return A std::error_condition. + */ + ++// Discpress: chd.cpp's formats for the track metadata, with a width for each string that fills its 16-byte ++// buffer at most (the longest a CD has is 15 characters, VMODE2_FORM_MIX); a longer one, in damaged or ++// crafted metadata, doesn't match. 0.289 read them with %s, which wrote past the end of the buffer. ++static const char CDROM_TRACK_METADATA_FORMAT_READ[] = "TRACK:%d TYPE:%15s SUBTYPE:%15s FRAMES:%d"; ++static const char CDROM_TRACK_METADATA2_FORMAT_READ[] = "TRACK:%d TYPE:%15s SUBTYPE:%15s FRAMES:%d PREGAP:%d PGTYPE:%15s PGSUB:%15s POSTGAP:%d"; ++static const char GDROM_TRACK_METADATA_FORMAT_READ[] = "TRACK:%d TYPE:%15s SUBTYPE:%15s FRAMES:%d PAD:%d PREGAP:%d PGTYPE:%15s PGSUB:%15s POSTGAP:%d"; ++ + std::error_condition cdrom_file::parse_metadata(chd_file *chd, toc &toc) + { + std::string metadata; +@@ -924,12 +955,12 @@ + // fetch the metadata for this track + if (!chd->read_metadata(CDROM_TRACK_METADATA_TAG, toc.numtrks, metadata)) + { +- if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA_FORMAT, &tracknum, type, subtype, &frames) != 4) ++ if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA_FORMAT_READ, &tracknum, type, subtype, &frames) != 4) + return chd_file::error::INVALID_DATA; + } + else if (!chd->read_metadata(CDROM_TRACK_METADATA2_TAG, toc.numtrks, metadata)) + { +- if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA2_FORMAT, &tracknum, type, subtype, &frames, &pregap, pgtype, pgsub, &postgap) != 8) ++ if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA2_FORMAT_READ, &tracknum, type, subtype, &frames, &pregap, pgtype, pgsub, &postgap) != 8) + return chd_file::error::INVALID_DATA; + } + else +@@ -944,7 +975,7 @@ + if (err) + break; + +- if (sscanf(metadata.c_str(), GDROM_TRACK_METADATA_FORMAT, &tracknum, type, subtype, &frames, &padframes, &pregap, pgtype, pgsub, &postgap) != 9) ++ if (sscanf(metadata.c_str(), GDROM_TRACK_METADATA_FORMAT_READ, &tracknum, type, subtype, &frames, &padframes, &pregap, pgtype, pgsub, &postgap) != 9) + return chd_file::error::INVALID_DATA; + + toc.flags |= CD_FLAG_GDROM; +@@ -995,7 +1026,14 @@ + + /* if we got any tracks this way, we're done */ + if (toc.numtrks > 0) ++ { ++ // Discpress: with each track once: a track number that repeats or skips one (damaged or crafted ++ // metadata) left a track of nothing, of 0-byte sectors, which extractcd divided by ++ for (uint32_t i = 0; i < toc.numtrks; i++) ++ if (toc.tracks[i].datasize == 0) ++ return chd_file::error::INVALID_DATA; + return std::error_condition(); ++ } + + osd_printf_info("toc.numtrks = %u?!\n", toc.numtrks); + +@@ -1004,6 +1042,9 @@ + err = chd->read_metadata(CDROM_OLD_METADATA_TAG, 0, oldmetadata); + if (err) + return err; ++ // Discpress: as long as the table it holds (0.289 read past the end of shorter metadata) ++ if (oldmetadata.size() < 4 + MAX_TRACKS * 6 * 4) ++ return chd_file::error::INVALID_DATA; + + /* reconstruct the TOC from it */ + auto *mrp = reinterpret_cast<uint32_t *>(&oldmetadata[0]); +@@ -1044,6 +1085,14 @@ + } + } + ++ // Discpress: no more tracks than the table holds, with sectors a CD has: anything else (damaged or ++ // crafted metadata) made 0.289 read past the end of the table, and of a sector's buffer ++ if (toc.numtrks > MAX_TRACKS) ++ return chd_file::error::INVALID_DATA; ++ for (uint32_t i = 0; i < toc.numtrks; i++) ++ if (toc.tracks[i].datasize == 0 || toc.tracks[i].datasize > MAX_SECTOR_DATA || toc.tracks[i].subsize > MAX_SUBCODE_DATA) ++ return chd_file::error::INVALID_DATA; ++ + return std::error_condition(); + } + +@@ -1153,204 +1202,106 @@ 0x50, 0xa4, 0xa5, 0x51, 0xa7, 0x53, 0x52, 0xa6, 0xa3, 0x57, 0x56, 0xa2, 0x54, 0xa0, 0xa1, 0x55 }; @@ -1801,10 +1948,7 @@

About and licenses

+ uint64_t a[11] = { }, b[11] = { }; + int const count = q ? ECC_Q_NUM_BYTES : ECC_P_NUM_BYTES; + if (!q) - { -- val1 ^= ecc_source_byte(sector, row[component]); -- val2 ^= ecc_source_byte(sector, row[component]); -- val1 = ecclow[val1]; ++ { + // the columns of the first 24 rows (a row's last word also reads 2 bytes of the next row, unused) + for (int row = 0; row < ECC_P_COMP; row++) + for (int l = 0; l < 11; l++) @@ -1839,7 +1983,10 @@

About and licenses

+ memcpy(av, a, sizeof(a)); + memcpy(bv, b, sizeof(b)); + for (int i = 0; i < count; i++) -+ { + { +- val1 ^= ecc_source_byte(sector, row[component]); +- val2 ^= ecc_source_byte(sector, row[component]); +- val1 = ecclow[val1]; + uint8_t const p0 = ecchigh[ecclow[av[i]] ^ bv[i]]; + parity[i] = p0; + parity[count + i] = p0 ^ bv[i]; @@ -1849,7 +1996,7 @@

About and licenses

} /** -@@ -1367,24 +1293,17 @@ +@@ -1367,24 +1318,17 @@ bool cdrom_file::ecc_verify(const uint8_t *sector) { @@ -1882,7 +2029,7 @@

About and licenses

} /** -@@ -1400,13 +1319,13 @@ +@@ -1400,13 +1344,13 @@ void cdrom_file::ecc_generate(uint8_t *sector) { @@ -1902,7 +2049,7 @@

About and licenses

} /** -@@ -1578,22 +1497,25 @@ +@@ -1578,22 +1522,25 @@ int cdrom_file::msf_to_frames(const char *token) { @@ -1942,7 +2089,7 @@

About and licenses

} /*------------------------------------------------- -@@ -1604,17 +1526,18 @@ +@@ -1604,17 +1551,18 @@ -------------------------------------------------*/ /** @@ -1963,7 +2110,7 @@

About and licenses

{ unsigned long offset = 0; uint32_t length, rate, filesize; -@@ -1633,7 +1556,7 @@ +@@ -1633,7 +1581,7 @@ } /* read the core header and make sure it's a WAVE file */ @@ -1972,7 +2119,7 @@

About and licenses

offset += actual; if (offset < 4) { -@@ -1647,7 +1570,7 @@ +@@ -1647,7 +1595,7 @@ } /* get the total size */ @@ -1981,7 +2128,7 @@

About and licenses

offset += actual; if (offset < 8) { -@@ -1657,7 +1580,7 @@ +@@ -1657,7 +1605,7 @@ filesize = little_endianize_int32(filesize); /* read the RIFF file type and make sure it's a WAVE file */ @@ -1990,7 +2137,7 @@

About and licenses

offset += actual; if (offset < 12) { -@@ -1673,9 +1596,9 @@ +@@ -1673,9 +1621,9 @@ /* seek until we find a format tag */ while (true) { @@ -2002,7 +2149,7 @@

About and licenses

offset += actual; length = little_endianize_int32(length); if (memcmp(&buf[0], "fmt ", 4) == 0) -@@ -1691,7 +1614,7 @@ +@@ -1691,7 +1639,7 @@ } /* read the format -- make sure it is PCM */ @@ -2011,7 +2158,7 @@

About and licenses

offset += actual; temp16 = little_endianize_int16(temp16); if (temp16 != 1) -@@ -1701,7 +1624,7 @@ +@@ -1701,7 +1649,7 @@ } /* number of channels -- only stereo is supported */ @@ -2020,7 +2167,7 @@

About and licenses

offset += actual; temp16 = little_endianize_int16(temp16); if (temp16 != 2) -@@ -1711,7 +1634,7 @@ +@@ -1711,7 +1659,7 @@ } /* sample rate */ @@ -2029,7 +2176,7 @@

About and licenses

offset += actual; rate = little_endianize_int32(rate); if (rate != 44100) -@@ -1721,11 +1644,11 @@ +@@ -1721,11 +1669,11 @@ } /* bytes/second and block alignment are ignored */ @@ -2043,7 +2190,7 @@

About and licenses

offset += actual; bits = little_endianize_int16(bits); if (bits != 16) -@@ -1740,9 +1663,9 @@ +@@ -1740,9 +1688,9 @@ /* seek until we find a data tag */ while (true) { @@ -2055,7 +2202,7 @@

About and licenses

offset += actual; length = little_endianize_int32(length); if (memcmp(&buf[0], "data", 4) == 0) -@@ -1764,7 +1687,12 @@ +@@ -1764,7 +1712,12 @@ return 0; } @@ -2069,7 +2216,7 @@

About and licenses

return length; } -@@ -1864,11 +1792,11 @@ +@@ -1864,11 +1817,11 @@ outtoc.numsessions = 1; @@ -2085,7 +2232,7 @@

About and licenses

{ osd_printf_error("ERROR: Not a Nero 5.5 or later image!\n"); fclose(infile); -@@ -1888,8 +1816,13 @@ +@@ -1888,8 +1841,13 @@ while (!done) { @@ -2101,7 +2248,7 @@

About and licenses

chunk_size = get_u32be(&buffer[4]); -@@ -1901,15 +1834,20 @@ +@@ -1901,15 +1859,20 @@ // skip second chunk size and UPC code fseek(infile, 20, SEEK_CUR); @@ -2124,7 +2271,7 @@

About and licenses

for (int track = start; track <= end; track++) { uint32_t size, mode; -@@ -1925,7 +1863,15 @@ +@@ -1925,7 +1888,15 @@ // printf("Track %d: sector size %d mode %x index0 %llx index1 %llx track_end %llx (pregap %d sectors, length %d sectors)\n", track, size, mode, index0, index1, track_end, (uint32_t)(index1-index0)/size, (uint32_t)(track_end-index1)/size); outinfo.track[track-1].fname.assign(tocfname); @@ -2141,7 +2288,7 @@

About and licenses

outinfo.track[track-1].idx[0] = outinfo.track[track-1].idx[1] = 0; switch (mode) -@@ -1976,21 +1922,29 @@ +@@ -1976,21 +1947,29 @@ return chd_file::error::UNSUPPORTED_FORMAT; } @@ -2176,7 +2323,7 @@

About and licenses

} } -@@ -2000,7 +1954,15 @@ +@@ -2000,7 +1979,15 @@ } else { @@ -2193,7 +2340,7 @@

About and licenses

} } -@@ -2042,6 +2004,20 @@ +@@ -2042,6 +2029,20 @@ outinfo.reset(); uint64_t size = get_file_size(tocfname); @@ -2214,7 +2361,7 @@

About and licenses

fclose(infile); -@@ -2052,7 +2028,15 @@ +@@ -2052,7 +2053,15 @@ outinfo.track[0].offset = 0; outinfo.track[0].idx[0] = outinfo.track[0].idx[1] = 0; @@ -2231,7 +2378,7 @@

About and licenses

{ outtoc.tracks[0].trktype = CD_TRACK_MODE1; outtoc.tracks[0].frames = size / 2048; -@@ -2149,7 +2133,7 @@ +@@ -2149,7 +2158,7 @@ const int numtracks = atoi(token); @@ -2240,7 +2387,7 @@

About and licenses

{ osd_printf_error("GDI expects too many tracks. Expected %d tracks but only up to %zu tracks allowed\n", numtracks, std::size(outinfo.track) + 1); return chd_file::error::INVALID_DATA; -@@ -2178,7 +2162,7 @@ +@@ -2178,7 +2187,7 @@ paramcnt++; const int trknum = atoi(token) - 1; @@ -2249,7 +2396,7 @@

About and licenses

{ osd_printf_error("Track %d is out of expected range of 1 to %d\n", trknum + 1, numtracks); return chd_file::error::INVALID_DATA; -@@ -2222,7 +2206,7 @@ +@@ -2222,7 +2231,7 @@ outtoc.tracks[trknum].trktype = CD_TRACK_MODE1; outtoc.tracks[trknum].datasize = 2048; } @@ -2258,7 +2405,7 @@

About and licenses

{ outtoc.tracks[trknum].trktype = CD_TRACK_AUDIO; outtoc.tracks[trknum].datasize = 2352; -@@ -2252,6 +2236,12 @@ +@@ -2252,6 +2261,12 @@ outinfo.track[trknum].fname.assign(path).append(token); const uint64_t sz = get_file_size(outinfo.track[trknum].fname); @@ -2271,7 +2418,7 @@

About and licenses

outtoc.tracks[trknum].frames = sz / trksize; outtoc.tracks[trknum].padframes = 0; -@@ -2345,6 +2335,19 @@ +@@ -2345,6 +2360,19 @@ enum gdi_area current_area = SINGLE_DENSITY; bool is_multibin = false; int leadin = -1; @@ -2291,7 +2438,7 @@

About and licenses

FILE *infile = fopen(path.c_str(), "rt"); if (!infile) -@@ -2371,6 +2374,14 @@ +@@ -2371,6 +2399,14 @@ char linebuffer[512]; memset(linebuffer, 0, sizeof(linebuffer)); @@ -2306,7 +2453,7 @@

About and licenses

while (!feof(infile)) { /* get the next line */ -@@ -2427,6 +2438,8 @@ +@@ -2427,6 +2463,8 @@ /* get lead-out time */ TOKENIZE int leadout_offset = msf_to_frames(token); @@ -2315,7 +2462,7 @@

About and licenses

outinfo.track[trknum].leadout = leadout_offset; } else if (!strncmp(linebuffer+i, "LEAD-IN", 7)) -@@ -2472,13 +2485,18 @@ +@@ -2472,13 +2510,18 @@ /* get the file type */ TOKENIZE @@ -2336,7 +2483,7 @@

About and licenses

} else if (!strcmp(token, "WAVE")) { -@@ -2489,6 +2507,8 @@ +@@ -2489,6 +2532,8 @@ osd_printf_error("ERROR: couldn't read [%s] or not a valid .WAV\n", lastfname); return chd_file::error::INVALID_DATA; } @@ -2345,7 +2492,7 @@

About and licenses

} else { -@@ -2502,6 +2522,12 @@ +@@ -2502,6 +2547,12 @@ /* get the track number */ TOKENIZE trknum = strtoul(token, nullptr, 10) - 1; @@ -2358,7 +2505,7 @@

About and licenses

/* next token on the line is the track type */ TOKENIZE -@@ -2541,6 +2567,10 @@ +@@ -2541,6 +2592,10 @@ outinfo.track[trknum].offset = wavoffs; wavoffs = wavlen = 0; } @@ -2369,7 +2516,7 @@

About and licenses

outinfo.track[trknum].fname.assign(lastfname); /* default filename to the last one */ -@@ -2586,6 +2616,54 @@ +@@ -2586,6 +2641,54 @@ osd_printf_error("ERROR: encountered invalid index %d\n", idx); return chd_file::error::INVALID_DATA; } @@ -2424,7 +2571,7 @@

About and licenses

outinfo.track[trknum].idx[idx] = frames; -@@ -2610,6 +2688,8 @@ +@@ -2610,6 +2713,8 @@ /* get index */ TOKENIZE frames = msf_to_frames(token); @@ -2433,7 +2580,7 @@

About and licenses

outtoc.tracks[trknum].pregap = frames; } -@@ -2620,11 +2700,15 @@ +@@ -2620,11 +2725,15 @@ /* get index */ TOKENIZE frames = msf_to_frames(token); @@ -2449,7 +2596,7 @@

About and licenses

outtoc.tracks[trknum].control_flags = 0; /* keep looping over remaining tokens in FLAGS line until there's no more to read */ -@@ -2667,18 +2751,51 @@ +@@ -2667,18 +2776,51 @@ } /* this is true for cue/bin and cue/iso, and we need it for cue/wav since .WAV is little-endian */ @@ -2503,7 +2650,7 @@

About and licenses

tlen = get_file_size(outinfo.track[trknum].fname); if (tlen == 0) { -@@ -2700,11 +2817,17 @@ +@@ -2700,11 +2842,17 @@ return chd_file::error::INVALID_DATA; } @@ -2522,7 +2669,7 @@

About and licenses

} else if (outtoc.tracks[trknum].frames == 0) { -@@ -2777,6 +2900,77 @@ +@@ -2777,6 +2925,77 @@ } } @@ -2600,7 +2747,7 @@

About and licenses

if (is_gdrom) { /* -@@ -2787,6 +2981,18 @@ +@@ -2787,6 +3006,18 @@ uint32_t this_pregap = outtoc.tracks[trknum].pregap; uint32_t this_offset = this_pregap * (outtoc.tracks[trknum].datasize + outtoc.tracks[trknum].subsize); @@ -2619,7 +2766,7 @@

About and licenses

outtoc.tracks[trknum-1].frames += this_pregap; outtoc.tracks[trknum-1].splitframes += this_pregap; -@@ -2808,7 +3014,7 @@ +@@ -2808,7 +3039,7 @@ outtoc.tracks[trknum].physframeofs = 45000; int dif=outtoc.tracks[trknum].physframeofs-(outtoc.tracks[trknum-1].frames+outtoc.tracks[trknum-1].physframeofs); outtoc.tracks[trknum-1].frames += dif; @@ -2628,7 +2775,7 @@

About and licenses

} else { -@@ -2965,6 +3171,67 @@ +@@ -2965,6 +3196,67 @@ char linebuffer[512]; memset(linebuffer, 0, sizeof(linebuffer)); @@ -2696,7 +2843,7 @@

About and licenses

while (!feof(infile)) { /* get the next line */ -@@ -2989,6 +3256,8 @@ +@@ -2989,6 +3281,8 @@ */ if (!strcmp(token, "NO")) { @@ -2705,7 +2852,7 @@

About and licenses

TOKENIZE if (!strcmp(token, "COPY")) outtoc.tracks[trknum].control_flags &= ~CD_FLAG_CONTROL_DIGITAL_COPY_PERMITTED; -@@ -2997,100 +3266,210 @@ +@@ -2997,100 +3291,210 @@ } else if (!strcmp(token, "COPY")) { @@ -2959,7 +3106,7 @@

About and licenses

/* next token on the line is the track type */ TOKENIZE -@@ -3117,22 +3496,128 @@ +@@ -3117,22 +3521,128 @@ } else if (!strcmp(token, "START")) { @@ -3163,7 +3310,11 @@

About and licenses

static uint64_t read_uint64(FILE *infile); --- a/src/lib/util/chd.cpp +++ b/src/lib/util/chd.cpp -@@ -28,6 +28,20 @@ +@@ -25,9 +25,24 @@ + #include <cstdlib> + #include <cstring> + #include <ctime> ++#include <limits> #include <new> #include <tuple> @@ -3184,7 +3335,7 @@

About and licenses

//************************************************************************** // CONSTANTS -@@ -853,6 +867,17 @@ +@@ -853,6 +868,17 @@ void chd_file::close() { @@ -3202,7 +3353,7 @@

About and licenses

// reset file characteristics m_file.reset(); m_allow_reads = false; -@@ -892,6 +917,48 @@ +@@ -892,6 +918,48 @@ m_cachehunk = ~0; } @@ -3251,17 +3402,20 @@

About and licenses

std::error_condition chd_file::codec_process_hunk(uint32_t hunknum) { // punt if no file -@@ -919,6 +986,9 @@ +@@ -919,6 +987,12 @@ case V34_MAP_ENTRY_TYPE_COMPRESSED: { uint32_t const blocklen = get_u16be(&rawmap[12]) | (uint32_t(rawmap[14]) << 16); + // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end + if (UNEXPECTED(blocklen > m_compressed.size())) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: and a hunk compressed in a CHD that says it isn't: 0.289 called a codec it doesn't have ++ if (UNEXPECTED(!m_decompressor[0])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; -@@ -931,7 +1001,14 @@ +@@ -931,11 +1005,21 @@ return std::error_condition(error::UNSUPPORTED_FORMAT); case V34_MAP_ENTRY_TYPE_SELF_HUNK: @@ -3277,17 +3431,27 @@

About and licenses

case V34_MAP_ENTRY_TYPE_PARENT_HUNK: if (UNEXPECTED(m_parent_missing)) -@@ -958,6 +1035,9 @@ + return std::error_condition(error::REQUIRES_PARENT); ++ // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer ++ if (UNEXPECTED(!m_parent)) ++ return std::error_condition(error::INVALID_DATA); + return m_parent->codec_process_hunk(blockoffs); + } + } +@@ -958,6 +1042,12 @@ case COMPRESSION_TYPE_2: case COMPRESSION_TYPE_3: { + // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end + if (UNEXPECTED(blocklen > m_compressed.size())) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: and a codec slot the header leaves empty: 0.289 called a codec it doesn't have ++ if (UNEXPECTED(!m_decompressor[rawmap[0]])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; -@@ -970,7 +1050,14 @@ +@@ -970,11 +1060,24 @@ return std::error_condition(error::UNSUPPORTED_FORMAT); case COMPRESSION_SELF: @@ -3303,17 +3467,30 @@

About and licenses

case COMPRESSION_PARENT: if (UNEXPECTED(m_parent_missing)) -@@ -1045,6 +1132,9 @@ + return std::error_condition(error::REQUIRES_PARENT); ++ // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer ++ if (UNEXPECTED(!m_parent)) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: a parent whose units are larger than its hunks has no hunk for a unit (0.289 divided by zero) ++ if (UNEXPECTED(m_parent->unit_bytes() > m_parent->hunk_bytes())) ++ return std::error_condition(error::INVALID_DATA); + return m_parent->codec_process_hunk(blockoffs / (m_parent->hunk_bytes() / m_parent->unit_bytes())); + } + break; +@@ -1045,6 +1148,12 @@ case V34_MAP_ENTRY_TYPE_COMPRESSED: { uint32_t const blocklen = get_u16be(&rawmap[12]) | (uint32_t(rawmap[14]) << 16); + // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end + if (UNEXPECTED(blocklen > m_compressed.size())) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: and a hunk compressed in a CHD that says it isn't: 0.289 called a codec it doesn't have ++ if (UNEXPECTED(!m_decompressor[0])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; -@@ -1073,7 +1163,14 @@ +@@ -1073,11 +1182,21 @@ return std::error_condition(); case V34_MAP_ENTRY_TYPE_SELF_HUNK: @@ -3329,7 +3506,14 @@

About and licenses

case V34_MAP_ENTRY_TYPE_PARENT_HUNK: if (UNEXPECTED(m_parent_missing)) -@@ -1115,6 +1212,14 @@ + return std::error_condition(error::REQUIRES_PARENT); ++ // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer ++ if (UNEXPECTED(!m_parent)) ++ return std::error_condition(error::INVALID_DATA); + return m_parent->read_hunk(blockoffs, dest); + } + } +@@ -1115,6 +1234,17 @@ case COMPRESSION_TYPE_2: case COMPRESSION_TYPE_3: { @@ -3340,11 +3524,14 @@

About and licenses

+#endif + // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end + if (UNEXPECTED(blocklen > m_compressed.size())) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: and a codec slot the header leaves empty: 0.289 called a codec it doesn't have ++ if (UNEXPECTED(!m_decompressor[rawmap[0]])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; -@@ -1130,16 +1235,29 @@ +@@ -1130,20 +1260,36 @@ case COMPRESSION_NONE: { @@ -3375,7 +3562,169 @@

About and licenses

case COMPRESSION_PARENT: if (UNEXPECTED(m_parent_missing)) -@@ -2173,6 +2291,10 @@ + return std::error_condition(error::REQUIRES_PARENT); ++ // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer ++ if (UNEXPECTED(!m_parent)) ++ return std::error_condition(error::INVALID_DATA); + return m_parent->read_bytes(blockoffs * m_parent->unit_bytes(), dest, m_hunkbytes); + } + } +@@ -1307,6 +1453,11 @@ + + std::error_condition chd_file::read_bytes(uint64_t offset, void *buffer, uint32_t bytes) + { ++ // Discpress: nothing to read, as for a CD track without data (a damaged or crafted CHD's): 0.289's ++ // last hunk wrapped around at offset 0, and it read a whole hunk into the buffer ++ if (bytes == 0) ++ return std::error_condition(); ++ + // iterate over hunks + uint32_t const first_hunk = offset / m_hunkbytes; + uint32_t const last_hunk = (offset + bytes - 1) / m_hunkbytes; +@@ -1360,6 +1511,10 @@ + + std::error_condition chd_file::write_bytes(uint64_t offset, const void *buffer, uint32_t bytes) + { ++ // Discpress: nothing to write (as read_bytes) ++ if (bytes == 0) ++ return std::error_condition(); ++ + // iterate over hunks + uint32_t const first_hunk = offset / m_hunkbytes; + uint32_t const last_hunk = (offset + bytes - 1) / m_hunkbytes; +@@ -1453,7 +1608,7 @@ + // read the metadata + try { output.resize(metaentry.length); } + catch (std::bad_alloc const &) { return std::errc::not_enough_memory; } +- return file_read(metaentry.offset + METADATA_HEADER_SIZE, &output[0], metaentry.length); ++ return file_read(metaentry.offset + METADATA_HEADER_SIZE, output.data(), metaentry.length); // Discpress: data(), as metadata can be empty + } + + /** +@@ -1515,7 +1670,7 @@ + // read the metadata + try { output.resize(metaentry.length); } + catch (std::bad_alloc const &) { return std::errc::not_enough_memory; } +- err = file_read(metaentry.offset + METADATA_HEADER_SIZE, &output[0], metaentry.length); ++ err = file_read(metaentry.offset + METADATA_HEADER_SIZE, output.data(), metaentry.length); // Discpress: data(), as metadata can be empty + if (UNEXPECTED(err)) + return err; + resulttag = metaentry.metatag; +@@ -1678,12 +1833,12 @@ + // read the metadata item + try { filedata.resize(metaentry.length); } + catch (std::bad_alloc const &) { return std::errc::not_enough_memory; } +- err = source.file_read(metaentry.offset + METADATA_HEADER_SIZE, &filedata[0], metaentry.length); ++ err = source.file_read(metaentry.offset + METADATA_HEADER_SIZE, filedata.data(), metaentry.length); // Discpress: data(), as metadata can be empty + if (UNEXPECTED(err)) + return err; + + // write it to the destination +- err = write_metadata(metaentry.metatag, (uint32_t)-1, &filedata[0], metaentry.length, metaentry.flags); ++ err = write_metadata(metaentry.metatag, (uint32_t)-1, filedata.data(), metaentry.length, metaentry.flags); + if (UNEXPECTED(err)) + return err; + } +@@ -1725,14 +1880,14 @@ + + // allocate memory and read the data + filedata.resize(metaentry.length); +- err = file_read(metaentry.offset + METADATA_HEADER_SIZE, &filedata[0], metaentry.length); ++ err = file_read(metaentry.offset + METADATA_HEADER_SIZE, filedata.data(), metaentry.length); // Discpress: data(), as metadata can be empty + if (UNEXPECTED(err)) + throw err; + + // create an entry for this metadata and add it + metadata_hash hashentry; + put_u32be(hashentry.tag, metaentry.metatag); +- hashentry.sha1 = util::sha1_creator::simple(&filedata[0], metaentry.length); ++ hashentry.sha1 = util::sha1_creator::simple(filedata.data(), metaentry.length); + hasharray.push_back(hashentry); + } + if (err != error::METADATA_NOT_FOUND) +@@ -1880,6 +2035,41 @@ + } + + /** ++ * @fn bool chd_file::past_end(uint64_t offset, uint64_t length) const ++ * @brief past_end - Discpress: whether length bytes from offset would run past the end of the ++ * file (false when its length is unknown, so that the read reports it) ++ */ ++bool chd_file::past_end(uint64_t offset, uint64_t length) const ++{ ++ uint64_t filelength; ++ if (!m_file || m_file->length(filelength)) ++ return false; ++ return (offset > filelength) || (length > filelength - offset); ++} ++ ++/** ++ * @fn void chd_file::check_sizes(uint64_t hunkcount) const ++ * ++ * @brief ------------------------------------------------- ++ * check_sizes - Discpress: a header whose sizes no data can have (a damaged or ++ * crafted file) is an error. 0.289 divided by a hunk or unit size of 0, and with a ++ * logical size of more hunks than a map can count, it made a map too small for them ++ * -------------------------------------------------. ++ * ++ * @exception CHDERR_INVALID_FILE Thrown when the sizes can't be right. ++ * ++ * @param hunkcount The number of hunks the header gives or implies. ++ */ ++ ++void chd_file::check_sizes(uint64_t hunkcount) const ++{ ++ if (UNEXPECTED(m_hunkbytes == 0 || m_unitbytes == 0)) ++ throw std::error_condition(error::INVALID_FILE); ++ if (UNEXPECTED(hunkcount > std::numeric_limits<uint32_t>::max())) ++ throw std::error_condition(error::INVALID_FILE); ++} ++ ++/** + * @fn void chd_file::parse_v3_header(uint8_t *rawheader, sha1_t &parentsha1) + * + * @brief ------------------------------------------------- +@@ -1938,7 +2128,8 @@ + parentsha1 = be_read_sha1(&rawheader[m_parentsha1_offset]); + + // guess at the units based on snooping the metadata +- m_unitbytes = guess_unitbytes(); ++ m_unitbytes = (m_hunkbytes != 0) ? guess_unitbytes() : 0; ++ check_sizes(m_hunkcount); + m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes; + } + +@@ -2001,7 +2192,8 @@ + parentsha1 = be_read_sha1(&rawheader[m_parentsha1_offset]); + + // guess at the units based on snooping the metadata +- m_unitbytes = guess_unitbytes(); ++ m_unitbytes = (m_hunkbytes != 0) ? guess_unitbytes() : 0; ++ check_sizes(m_hunkcount); + m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes; + } + +@@ -2029,8 +2221,9 @@ + m_mapoffset = get_u64be(&rawheader[40]); + m_metaoffset = get_u64be(&rawheader[48]); + m_hunkbytes = get_u32be(&rawheader[56]); +- m_hunkcount = (m_logicalbytes + m_hunkbytes - 1) / m_hunkbytes; + m_unitbytes = get_u32be(&rawheader[60]); ++ check_sizes((m_hunkbytes != 0) ? ((m_logicalbytes / m_hunkbytes) + ((m_logicalbytes % m_hunkbytes) != 0)) : 0); ++ m_hunkcount = (m_logicalbytes + m_hunkbytes - 1) / m_hunkbytes; + m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes; + + // determine compression +@@ -2073,7 +2266,7 @@ + try + { + // first get a CRC-16 of the original rawmap +- util::crc16_t mapcrc = util::crc16_creator::simple(&m_rawmap[0], m_hunkcount * 12); ++ util::crc16_t mapcrc = util::crc16_creator::simple(m_rawmap.data(), m_hunkcount * 12); + + // create a buffer to hold the RLE data + std::vector<uint8_t> compression_rle(m_hunkcount); +@@ -2173,6 +2366,10 @@ // for COMPRESSION_PARENT: parentbits // the overall size is clamped later with bitbuf.flush() int nbits_needed = (8*16) + (12 + std::max<int>({lengthbits+16, selfbits, parentbits}))*m_hunkcount; @@ -3386,7 +3735,7 @@

About and licenses

std::vector<uint8_t> compressed(nbits_needed / 8 + 1); bitstream_out bitbuf(&compressed[16], compressed.size() - 16); -@@ -2255,7 +2377,8 @@ +@@ -2255,7 +2452,8 @@ // write the map header uint32_t complen = bitbuf.flush(); @@ -3396,7 +3745,75 @@

About and licenses

put_u32be(&compressed[0], complen); put_u48be(&compressed[4], firstoffs); put_u16be(&compressed[10], mapcrc); -@@ -2966,6 +3089,13 @@ +@@ -2298,7 +2496,7 @@ + // if no offset, we haven't written it yet + if (m_mapoffset == 0) + { +- memset(&m_rawmap[0], 0xff, m_rawmap.size()); ++ memset(m_rawmap.data(), 0xff, m_rawmap.size()); + return; + } + +@@ -2314,13 +2512,21 @@ + uint8_t const lengthbits = rawbuf[12]; + uint8_t const selfbits = rawbuf[13]; + uint8_t const parentbits = rawbuf[14]; ++ // Discpress: the bit reader reads at most 32 bits at a time, as many as compress_v5_map writes: ++ // more (a damaged or crafted map) shifted by a negative count, which C++ leaves undefined ++ if (UNEXPECTED((lengthbits > 32) || (selfbits > 32) || (parentbits > 32))) ++ throw std::error_condition(error::DECOMPRESSION_ERROR); + + // now read the map ++ // Discpress: one that would run past the end of the file is the error reading it gives, without ++ // allocating it first (up to 4 GB, for a damaged or crafted header) ++ if (UNEXPECTED(past_end(m_mapoffset + 16, mapbytes))) ++ throw std::error_condition(std::errc::io_error); + std::vector<uint8_t> compressed(mapbytes); +- ioerr = file_read(m_mapoffset + 16, &compressed[0], mapbytes); ++ ioerr = file_read(m_mapoffset + 16, compressed.data(), mapbytes); + if (UNEXPECTED(ioerr)) + throw ioerr; +- bitstream_in bitbuf(&compressed[0], compressed.size()); ++ bitstream_in bitbuf(compressed.data(), compressed.size()); + + // first decode the compression types + huffman_decoder<16, 8> decoder; +@@ -2409,7 +2615,7 @@ + } + + // verify the final CRC +- if (UNEXPECTED(util::crc16_creator::simple(&m_rawmap[0], m_hunkcount * 12) != mapcrc)) ++ if (UNEXPECTED(util::crc16_creator::simple(m_rawmap.data(), m_hunkcount * 12) != mapcrc)) + throw std::error_condition(error::DECOMPRESSION_ERROR); + } + +@@ -2636,14 +2842,23 @@ + } + + // read the map; v5+ compressed drives need to read and decompress their map +- m_rawmap.resize(m_hunkcount * m_mapentrybytes); ++ // Discpress: a map larger than 32 bits can count (a damaged or crafted header) is an error: 0.289 ++ // computed its size in 32 bits, and with the size wrapped, read and wrote past the end of the map ++ uint64_t const mapsize = uint64_t(m_hunkcount) * m_mapentrybytes; ++ if (UNEXPECTED(mapsize > std::numeric_limits<uint32_t>::max())) ++ throw std::error_condition(error::INVALID_FILE); ++ // Discpress: and a stored map that would run past the end of the file is the error reading it ++ // gives, without allocating it first ++ if (!(m_version >= 5 && compressed()) && UNEXPECTED(past_end(m_mapoffset, mapsize))) ++ throw std::error_condition(std::errc::io_error); ++ m_rawmap.resize(mapsize); + if (m_version >= 5 && compressed()) + { + decompress_v5_map(); + } + else + { +- std::error_condition err = file_read(m_mapoffset, &m_rawmap[0], m_rawmap.size()); ++ std::error_condition err = file_read(m_mapoffset, m_rawmap.data(), m_rawmap.size()); + if (UNEXPECTED(err)) + throw err; + } +@@ -2966,6 +3181,13 @@ chd_file_compressor::~chd_file_compressor() { @@ -3410,7 +3827,7 @@

About and licenses

// free the work queues osd_work_queue_free(m_read_queue); osd_work_queue_free(m_work_queue); -@@ -3022,6 +3152,10 @@ +@@ -3022,6 +3244,10 @@ // reset write state m_write_hunk = 0; @@ -3421,7 +3838,7 @@

About and licenses

} /** -@@ -3123,6 +3257,11 @@ +@@ -3123,6 +3349,11 @@ } else { @@ -3433,7 +3850,7 @@

About and licenses

// otherwise, append it compressed and add to the self map hunk_write_compressed(item.m_hunknum, item.m_compression, item.m_compressed, item.m_complen, item.m_hash[0].m_crc16); m_total_out += item.m_complen; -@@ -3263,12 +3402,44 @@ +@@ -3263,12 +3494,44 @@ // TODO: data race if ((m_current_map.find(item.m_hash[0].m_crc16, item.m_hash[0].m_sha1) == hashmap::NOT_FOUND) && (m_parent_map.find(item.m_hash[0].m_crc16, item.m_hash[0].m_sha1) == hashmap::NOT_FOUND)) @@ -3479,7 +3896,7 @@

About and licenses

/** * @fn void *chd_file_compressor::async_read_static(void *param, int threadid) * -@@ -3345,6 +3516,16 @@ +@@ -3345,6 +3608,16 @@ assert(item.m_status == WS_READING); item.m_status = WS_QUEUED; item.m_hunknum = hunknum; @@ -3496,7 +3913,7 @@

About and licenses

item.m_osd = osd_work_item_queue(m_work_queue, m_walking_parent ? async_walk_parent_static : async_compress_hunk_static, &item, 0); } -@@ -3512,3 +3693,184 @@ +@@ -3512,3 +3785,184 @@ metadata_entry metaentry; return metadata_find(AV_METADATA_TAG, 0, metaentry); } @@ -3664,7 +4081,7 @@

About and licenses

+ memcpy(dest, &ahead.data[0], m_hunkbytes); + return true; + case 3: // compressed bytes -+ m_decompressor[type]->decompress(&ahead.data[0], ahead.data.size(), reinterpret_cast<uint8_t *>(dest), m_hunkbytes); ++ m_decompressor[type]->decompress(ahead.data.data(), ahead.data.size(), reinterpret_cast<uint8_t *>(dest), m_hunkbytes); + return true; + default: + return false; @@ -3724,7 +4141,16 @@

About and licenses

private: struct metadata_entry; struct metadata_hash; -@@ -391,6 +411,7 @@ +@@ -380,6 +400,8 @@ + + // internal helpers + uint32_t guess_unitbytes(); ++ void check_sizes(uint64_t hunkcount) const; // Discpress ++ bool past_end(uint64_t offset, uint64_t length) const; // Discpress + void parse_v3_header(uint8_t *rawheader, util::sha1_t &parentsha1); + void parse_v4_header(uint8_t *rawheader, util::sha1_t &parentsha1); + void parse_v5_header(uint8_t *rawheader, util::sha1_t &parentsha1); +@@ -391,6 +413,7 @@ std::error_condition verify_proper_compression_append(uint32_t hunknum) const noexcept; void hunk_write_compressed(uint32_t hunknum, int8_t compression, const uint8_t *compressed, uint32_t complength, util::crc16_t crc16); void hunk_copy_from_self(uint32_t hunknum, uint32_t otherhunk); @@ -3732,7 +4158,7 @@

About and licenses

void hunk_copy_from_parent(uint32_t hunknum, uint64_t parentunit); std::error_condition metadata_find(chd_metadata_tag metatag, int32_t metaindex, metadata_entry &metaentry, bool resume = false) const noexcept; std::error_condition metadata_set_previous_next(uint64_t prevoffset, uint64_t nextoffset) noexcept; -@@ -433,6 +454,21 @@ +@@ -433,6 +456,21 @@ // caching std::vector<uint8_t> m_cache; // single-hunk cache for partial reads/writes uint32_t m_cachehunk; // which hunk is in the cache? @@ -3754,7 +4180,7 @@

About and licenses

}; -@@ -450,11 +486,28 @@ +@@ -450,11 +488,28 @@ void compress_begin(); std::error_condition compress_continue(double &progress, double &ratio); @@ -3783,7 +4209,7 @@

About and licenses

// hash map for looking up values class hashmap { -@@ -524,6 +577,7 @@ +@@ -524,6 +579,7 @@ , m_complen(0) , m_compression(0) , m_codecs(nullptr) @@ -3791,7 +4217,7 @@

About and licenses

{ } osd_work_item * m_osd; // OSD work item running on this block -@@ -537,6 +591,7 @@ +@@ -537,6 +593,7 @@ uint32_t m_complen; // compressed data length int8_t m_compression; // type of compression used chd_compressor_group *m_codecs; // codec instance @@ -3799,7 +4225,7 @@

About and licenses

std::vector<hash_pair> m_hash; // array of hashes }; -@@ -549,6 +604,7 @@ +@@ -549,6 +606,7 @@ void async_read(); // current compression status @@ -3874,7 +4300,24 @@

About and licenses

return header_bytes + complen + m_subcode_compressor.compress(&m_buffer[frames * cdrom_file::MAX_SECTOR_DATA], frames * cdrom_file::MAX_SUBCODE_DATA, &dest[header_bytes + complen]); } -@@ -735,17 +751,33 @@ +@@ -426,8 +442,16 @@ + uint32_t ecc_bytes = (frames + 7) / 8; + uint32_t header_bytes = ecc_bytes + complen_bytes; + ++ // Discpress: a header longer than the data, or a base longer than what follows the header (a ++ // damaged or crafted hunk), is an error: 0.289 decoded past the end of the data, and the ++ // subcode's length wrapped ++ if (UNEXPECTED(complen < header_bytes)) ++ throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR); ++ + // extract compressed length of base + uint32_t complen_base = (complen_bytes > 2) ? get_u24be(&src[ecc_bytes]) : get_u16be(&src[ecc_bytes]); ++ if (UNEXPECTED(complen_base > complen - header_bytes)) ++ throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR); + + // reset and decode + m_base_decompressor.decompress(&src[header_bytes], complen_base, &m_buffer[0], frames * cdrom_file::MAX_SECTOR_DATA); +@@ -735,17 +759,33 @@ // compression for this hunk //------------------------------------------------- @@ -3910,7 +4353,7 @@

About and licenses

// if this is the best one, copy the data into the permanent buffer uint32_t compbytes = m_compressor[codecnum]->compress(src, m_hunkbytes, &m_compress_test[0]); #if CHDCODEC_VERIFY_COMPRESSION -@@ -772,7 +804,7 @@ +@@ -772,7 +812,7 @@ } printf(" codec%d=%d bytes \n", codecnum, compbytes); #endif @@ -3919,7 +4362,7 @@

About and licenses

{ compression = codecnum; complen = compbytes; -@@ -787,6 +819,7 @@ +@@ -787,6 +827,7 @@ // if the best is none, copy it over if (compression == -1) memcpy(compressed, src, m_hunkbytes); @@ -3927,7 +4370,7 @@

About and licenses

return compression; } -@@ -852,12 +885,12 @@ +@@ -852,12 +893,12 @@ { // set the low bit of the size so we don't match next time *ptr |= 1; @@ -3942,7 +4385,7 @@

About and licenses

for (int scan = 0; scan < MAX_ZLIB_ALLOCS; scan++) if (codec->m_allocptr[scan] == nullptr) { -@@ -867,7 +900,7 @@ +@@ -867,7 +908,7 @@ // set the low bit of the size so we don't match next time *ptr = size | 1; @@ -3951,7 +4394,7 @@

About and licenses

} -@@ -881,7 +914,7 @@ +@@ -881,7 +922,7 @@ auto *codec = reinterpret_cast<chd_zlib_allocator *>(opaque); // find the hunk @@ -3960,7 +4403,7 @@

About and licenses

for (int scan = 0; scan < MAX_ZLIB_ALLOCS; scan++) if (ptr == codec->m_allocptr[scan]) { -@@ -915,6 +948,13 @@ +@@ -915,6 +956,13 @@ throw std::bad_alloc(); else if (zerr != Z_OK) throw std::error_condition(chd_file::error::CODEC_ERROR); @@ -3974,7 +4417,7 @@

About and licenses

} -@@ -925,6 +965,8 @@ +@@ -925,6 +973,8 @@ chd_zlib_compressor::~chd_zlib_compressor() { deflateEnd(&m_deflater); @@ -3983,7 +4426,7 @@

About and licenses

} -@@ -934,12 +976,24 @@ +@@ -934,12 +984,24 @@ uint32_t chd_zlib_compressor::compress(const uint8_t *src, uint32_t srclen, uint8_t *dest) { @@ -4009,7 +4452,7 @@

About and licenses

m_deflater.total_out = 0; int zerr = deflateReset(&m_deflater); if (zerr != Z_OK) -@@ -949,7 +1003,7 @@ +@@ -949,7 +1011,7 @@ zerr = deflate(&m_deflater, Z_FINISH); // if we ended up with more data than we started with, return an error @@ -4018,7 +4461,7 @@

About and licenses

throw std::error_condition(chd_file::error::COMPRESSION_ERROR); // otherwise, return the length -@@ -1066,7 +1120,7 @@ +@@ -1066,7 +1128,7 @@ // do it ZSTD_inBuffer input{ src, srclen, 0 }; @@ -4027,7 +4470,7 @@

About and licenses

while (output.pos < output.size) { result = ZSTD_compressStream2(m_stream, &output, &input, ZSTD_e_end); -@@ -1197,12 +1251,12 @@ +@@ -1197,12 +1259,12 @@ { // set the low bit of the size so we don't match next time *ptr |= 1; @@ -4042,7 +4485,7 @@

About and licenses

for (int scan = 0; scan < MAX_LZMA_ALLOCS; scan++) if (codec->m_allocptr[scan] == nullptr) { -@@ -1212,7 +1266,7 @@ +@@ -1212,7 +1274,7 @@ // set the low bit of the size so we don't match next time *ptr = size | 1; @@ -4051,7 +4494,7 @@

About and licenses

} -@@ -1229,7 +1283,7 @@ +@@ -1229,7 +1291,7 @@ auto *const codec = static_cast<chd_lzma_allocator *>(const_cast<ISzAlloc *>(p)); // find the hunk @@ -4060,7 +4503,7 @@

About and licenses

for (int scan = 0; scan < MAX_LZMA_ALLOCS; scan++) if (ptr == codec->m_allocptr[scan]) { -@@ -1284,8 +1338,8 @@ +@@ -1284,8 +1346,8 @@ if (res != SZ_OK) throw std::error_condition(chd_file::error::COMPRESSION_ERROR); @@ -4071,7 +4514,7 @@

About and licenses

res = LzmaEnc_MemEncode(encoder, dest, &complen, src, srclen, 0, nullptr, &m_allocator, &m_allocator); if (res != SZ_OK) throw std::error_condition(chd_file::error::COMPRESSION_ERROR); -@@ -1419,7 +1473,7 @@ +@@ -1419,7 +1481,7 @@ uint32_t chd_huffman_compressor::compress(const uint8_t *src, uint32_t srclen, uint8_t *dest) { uint32_t complen; @@ -4080,7 +4523,7 @@

About and licenses

throw std::error_condition(chd_file::error::COMPRESSION_ERROR); return complen; } -@@ -1474,6 +1528,7 @@ +@@ -1474,6 +1536,7 @@ m_encoder.set_num_channels(2); m_encoder.set_block_size(blocksize(hunkbytes)); m_encoder.set_strip_metadata(true); @@ -4088,7 +4531,7 @@

About and licenses

} -@@ -1483,32 +1538,47 @@ +@@ -1483,32 +1546,47 @@ uint32_t chd_flac_compressor::compress(const uint8_t *src, uint32_t srclen, uint8_t *dest) { @@ -4149,7 +4592,7 @@

About and licenses

} return complen + 1; } -@@ -1642,14 +1712,16 @@ +@@ -1642,14 +1720,16 @@ memcpy(&m_buffer[frames * cdrom_file::MAX_SECTOR_DATA + framenum * cdrom_file::MAX_SUBCODE_DATA], &src[framenum * cdrom_file::FRAME_SIZE + cdrom_file::MAX_SECTOR_DATA], cdrom_file::MAX_SUBCODE_DATA); } @@ -4169,6 +4612,18 @@

About and licenses

// deflate the subcode data m_deflater.next_in = const_cast<Bytef *>(&m_buffer[frames * cdrom_file::MAX_SECTOR_DATA]); +@@ -1950,6 +2030,11 @@ + + void chd_avhuff_decompressor::decompress(const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) + { ++ // Discpress: a frame larger than the hunk (damaged or crafted data) is an error: 0.289 decoded it ++ // past the end of the hunk (its size, as raw_data_size has it: header, metadata, audio, video) ++ if ((complen >= 8) && (12 + uint64_t(src[0]) + 2 * uint64_t(src[1]) * get_u16be(&src[2]) + 2 * uint64_t(get_u16be(&src[4])) * get_u16be(&src[6]) > destlen)) ++ throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR); ++ + // decode the audio and video + avhuff_error averr = m_decoder.decode_data(src, complen, dest); + if (averr != AVHERR_NONE) --- a/src/lib/util/chdcodec.h +++ b/src/lib/util/chdcodec.h @@ -71,6 +71,13 @@ @@ -4547,6 +5002,42 @@

About and licenses

m_accum.m_raw = crc; } +--- a/src/lib/util/huffman.cpp ++++ b/src/lib/util/huffman.cpp +@@ -631,6 +631,10 @@ + uint32_t nextstart = (curstart + bithisto[codelen]) >> 1; + if (codelen != 1 && nextstart * 2 != (curstart + bithisto[codelen])) + return HUFFERR_INTERNAL_INCONSISTENCY; ++ // Discpress: more codes than one bit has room for (a damaged or crafted tree): 0.289 gave codes ++ // more bits than their length, and build_lookup_table wrote past the end of its table ++ if (codelen == 1 && (curstart + bithisto[codelen]) > 2) ++ return HUFFERR_INTERNAL_INCONSISTENCY; + bithisto[codelen] = curstart; + curstart = nextstart; + } +@@ -654,6 +658,7 @@ + void huffman_context_base::build_lookup_table() + { + // iterate over all codes ++ uint32_t covered = 0; + for (int curcode = 0; curcode < m_numcodes; curcode++) + { + // process all nodes which have non-zero bits +@@ -669,8 +674,14 @@ + lookup_value *destend = &m_lookup[((node.m_bits + 1) << shift) - 1]; + while (dest <= destend) + *dest++ = value; ++ covered = std::max<uint32_t>(covered, (node.m_bits + 1) << shift); + } + } ++ ++ // Discpress: the entries no code covers decode as code 0 of no bits. Canonical codes cover the table ++ // from 0 up, all of it for a complete tree (what an encoder writes); 0.289 left the rest as it was, ++ // whatever was in that memory, and a damaged tree's data read bit counts from it ++ std::fill(m_lookup + covered, m_lookup + (uint32_t(1) << m_maxbits), lookup_value(0)); + } + + --- a/src/lib/util/unicode.cpp +++ b/src/lib/util/unicode.cpp @@ -16,71 +16,12 @@ @@ -7267,7 +7758,8 @@

About and licenses

var opts = { noInitialRun: true, instantiateWasm: function (imports, receive) { - WebAssembly.instantiate(wasmModule, imports).then(function (inst) { receive(inst, wasmModule); }, + // (the page may terminate this worker once it hears 'wasm-ready', or 'fatal': Engine.stop) + WebAssembly.instantiate(wasmModule, imports).then(function (inst) { postMessage({ type: 'wasm-ready' }); receive(inst, wasmModule); }, function (err) { postMessage({ type: 'fatal', message: 'Could not start WebAssembly: ' + err }); }); return {}; }, @@ -8564,8 +9056,18 @@

About and licenses

async function runJob(msg) { if (msg.debugSinkMax) SINK_MAX = msg.debugSinkMax; // testing: a small limit await loadModule(msg); + // (the engine first: until it is instantiated the page holds off terminating this worker, Engine.stop) + var out = makeLineSink(1), err = makeLineSink(2), aborted = null; + var M = await instantiate({ + stdin: function () { return null; }, + stdout: function (c) { out.put(c); }, + stderr: function (c) { err.put(c); }, + onAbort: function (what) { aborted = 'chdman stopped unexpectedly: ' + what; } + }); var backing = await makeBacking(msg); postMessage({ type: 'storage', mode: backing.mode, reason: memReason }); + // testing: silent this long, as a stuck job is (the page's no-progress watchdog, DEBUG.freeze) + if (msg.debugFreeze) await new Promise(function (r) { setTimeout(r, msg.debugFreeze); }); // copies of inputs chdman must modify in place (verify --fix, addmeta, delmeta) var outFiles = []; @@ -8617,14 +9119,6 @@

About and licenses

} } - var out = makeLineSink(1), err = makeLineSink(2), aborted = null; - var M = await instantiate({ - stdin: function () { return null; }, - stdout: function (c) { out.put(c); }, - stderr: function (c) { err.put(c); }, - onAbort: function (what) { aborted = 'chdman stopped unexpectedly: ' + what; } - }); - inStores.forEach(function (st) { if (st.store.setModule) st.store.setModule(M); }); backing.M = M; M.outFull = sinkFull; // wasm_out_full: chdman pauses while a folder is behind @@ -8833,8 +9327,8 @@

About and licenses

if (p) p.catch(function (err) { postMessage({ type: 'fatal', message: err && typeof err.errno === 'number' ? errText(err) : String(err && err.stack || err && err.message || err) }); }); }; - - + + - + diff --git a/docs/chd/fork-plan.md b/docs/chd/fork-plan.md index d247ff4..e867c6d 100644 --- a/docs/chd/fork-plan.md +++ b/docs/chd/fork-plan.md @@ -115,7 +115,7 @@ Discpress knows the system of every job, so policies can be **system-aware**. Ex ## 4. Roadmap -Effort estimates assume one developer working with the existing test harness. The statuses are as of Discpress 1.8.1. +Effort estimates assume one developer working with the existing test harness. The statuses are as of Discpress 1.8.2. | Milestone | Status | |---|---| @@ -127,6 +127,7 @@ Effort estimates assume one developer working with the existing test harness. Th | 5. What users ask for | Done (1.5.0) | | 6. Detection that knows the game | Done (1.7.0) | | 7. Layouts each emulator reads right | Planned; nothing built (no new conversion engine, but steps 2 and 5 change the fork) | +| 8. Faster, more efficient, more reliable | Step 1 done (fuzzing and crafted headers found over 20 faults that damaged files reach, all but one also chdman 0.289's); step 2's workflows in place; steps 3 and 4 planned | The two releases after milestone 6 are outside the milestones, and their CHDs are the same: - **1.8.0** fixed what a full review of the app found: discs that took another game's tracks or name, results that could be lost, slow adding of big folders. @@ -406,6 +407,64 @@ Needed by every step: reader models in the tests (a Flycast 2.7 model, "assume s Separate from the engine question, and only as users ask: writers for other formats, starting with CSO/ZSO for real PSP and PS2 hardware (about 1 person-week; the page already reads both), then PBP for PS1 on PSP and Vita, and Xbox ISO for xemu (1 to 2 each). RVZ for Dolphin (5 to 8, with licence and key questions) only if Dolphin users ask, since Dolphin converts to RVZ itself. +### Milestone 8: faster, more efficient, more reliable (research of 2026-11-11) + +**Decision: reliability first, then speed with the same bytes.** The default compression stays chdman's +(the byte-for-byte contract, §3); phones are only told about "Nearly as small, faster"; Verify after +converting stays off by default. + +Where the time goes, measured on the benchmark discs (300 MB CD, 1 GB DVD; 4 cores; `bench.js`): +- **Native, `-np 4`, wall/CPU seconds:** DVD with the default codecs, 0.289 58.1/169.9, the engine 50.0/151.2, + the codec plan with libdeflate 26.6/81.5, `zlib,huff` with libdeflate 4.5/14.0. CD: 0.289 15.0/46.7, the + engine 13.0/39.3, plan 7.4/22.6, `cdzl,cdfl` 4.9/15.0. +- **Each codec alone on a DVD hunk** (a native harness timing `find_best_compressor`): LZMA 310.6 µs, FLAC + 228.6, zlib 47.8, Huffman 20.5; all four 586.5. FLAC almost never wins on data, which is what the codec plan + skips. In FLAC, autocorrelation is 31% of its instructions, `residual_from_qlp` 21%, the partition sums + 13%, the fixed predictor 8%. +- **The page in Chromium at 1, 2 and 4 threads:** DVD default 157.3, 82.2 and 49.3 s; plan 99.5, 53.5 and + 34.0; fast 20.7 and 11.2 (1 and 4). CD default 41.4, 20.5 and 11.8; plan 24.6, 11.4 and 7.4; fast 14.0 and + 5.2. At 4 threads the page is within 10–12% of native with the default; "Faster to create" on a DVD takes + 11.2 s against native's 4.5, so its time is in the page's own serial path, not in compression. +- **Ruled out:** clearing LZMA's tables lazily (12.8% of its instructions are a `memset`; the lazy version + measured slower), C++ exceptions for the codecs that lose (14,247 throws on the DVD's 6,186 hunks, under + 1 µs each in Chromium), `-flto` (no gain, and it breaks exceptions). + +1. **Reliability** (done, but for what CI's Firefox and WebKit runs show). + - The three header faults (hunk or unit size 0, more hunks than a map counts) are errors. + - libFuzzer under ASan and UBSan (`tests/fuzz/`, `scripts/fuzz.sh`, weekly in `.github/workflows/fuzz.yml` + with a kept corpus): CHD files read as identification and every reading command does, the browser's + read-ahead and helper decompression included; descriptors through `createcd`; each codec's hunks; ECM + windows. It found, and the engine fixes: a Huffman tree's lookup table written past its end (the stack, + for a map or a `huff` hunk), a map size computed in 32 bits (written past its end), maps larger than the + file allocated first, map bit widths over 32, CD codec sector data longer than the hunk, LaserDisc + frames larger than the hunk and more than 16 channels, a read of 0 bytes that read a whole hunk, CD + metadata parsed with `%s` into 16-byte buffers, CD metadata missing a track (a division by zero), + old-style CD metadata read past its end, hunks naming a codec or a parent the CHD doesn't have (null + pointers), and four kinds of undefined behavior with unchanged results (`tests/fuzz/README.md`, + `engine/README.md`). Every CHD the fixtures make is unchanged. + - The page's own readers of CSO/ZSO, PBP, ECM, zip and CloneCD files get damaged files of each kind + (`tests/ui/readers.spec.js`): each ends as a card that says why, or a conversion that finishes or fails, + never an uncaught error or a card left waiting. Open: a damaged file stored (not deflated) in a zip + converts, since stored files are read in place and the zip's CRC-32 is checked for deflated ones only. + - A running conversion the page hears nothing from for 90 s, while visible, says so (`STALL_MS`). + - The suite runs in Firefox and WebKit in CI too (`.github/workflows/browsers.yml`). +2. **Measure on real browsers** (workflows in place). `.github/workflows/bench.yml` times the page in + Chromium and Firefox (Ubuntu) and WebKit (an Apple-silicon Mac). Whether Firefox's or Safari's numbers + come before step 3 is decided from its results. +3. **Speed with the same bytes** (planned). + - The job worker's serial path first: what "Faster to create" waits on (11.2 s against native's 4.5 on a + DVD), and extract's and verify's, profiled with counters the page reads in a debug build. It reads the + input, keeps the running SHA-1 of all of it, hands hunks to the helpers and writes their results. + - Exact SIMD for FLAC's loops: Emscripten already vectorizes the two largest, autocorrelation (373 SIMD + instructions in the build) and the residual from the LPC coefficients (349), without changing a result + (each lag's sum keeps its order; WebAssembly SIMD has no fused multiply-add). What is left is mostly + scalar: the partition sums (15) and the fixed predictor (56), integer loops that would give the same + results by construction. Only worth writing if a measurement shows them in the page's time. + - Whether FLAC can stop after the first channel when it already can't win: only if that is exact. +4. **Later** (as needed): a CI build with WebAssembly's new exception format (`-sWASM_LEGACY_EXCEPTIONS=0`, + exnref), so that switching once every browser has it is one line. A battery saver isn't needed: the + thread setting already is one. + ## 5. Test and verification strategy - **Oracles.** Each output is checked against a set of oracles, not one: diff --git a/docs/chd/format.md b/docs/chd/format.md index 213353f..98fd11d 100644 --- a/docs/chd/format.md +++ b/docs/chd/format.md @@ -372,7 +372,7 @@ emit tree, all tokens, then pass-2 fields; flush; header as above; append the ma | hunkcount | u32 | MAME computes `m_hunkcount*12` in 32 bits, which overflows beyond 357,913,941 hunks. libchdr caps the materialized map at 256 MiB, about 22.37 M hunks: roughly 91 GB at 4 KiB hunks or 438 GB at CD geometry. | `chd.cpp:2217,2780`; `libchdr_chd.c:113,864-884` | | compressed length | 24 bits in the expanded entry | **≤ hunkbytes** (see §4.4) | `chd.cpp:2793`; `libchdr_chd.c:3024` | | file offsets | 48 bits (256 TiB) | | `chd.h:186` | -| lengthbits/selfbits/parentbits | stored as u8; MAME writes the bit width of the maximum value | **≤25** for libchdr (all versions); ≤31 for MAME, since `bitstream_in::remove(32)` is an undefined shift; ≤32 for chd-rs **[tested G3, O_map]** | `libchdr_bitstream.c:52`; `bitstream.h:147-151` | +| lengthbits/selfbits/parentbits | stored as u8; MAME writes the bit width of the maximum value | **≤25** for libchdr (all versions); ≤31 for MAME 0.289, since `bitstream_in::remove(32)` is an undefined shift (Discpress's engine reads 32: it empties the buffer); ≤32 for chd-rs **[tested G3, O_map]** | `libchdr_bitstream.c:52`; `bitstream.h:147-151` | | explicit self target | selfbits | < 2^25 hunks for libchdr | derived | | explicit parent unit | parentbits | < 2^25 units: 68.7 GB at 2048-byte units, 17 GB at 512-byte units (large HDD parents) | derived | | mapbytes | u32 | | | diff --git a/engine/README.md b/engine/README.md index 92167bb..717c7fa 100644 --- a/engine/README.md +++ b/engine/README.md @@ -27,7 +27,7 @@ The paths mirror MAME's, so every file can be compared with its original. **[`ma - **Keeping the cue sheet (`createcd --keepcue`, `-kc`; `chdman.cpp`), opt-in:** the input `.cue` is stored as it is, as metadata (`CUES`) after the CD metadata and without the checksum flag, so neither SHA-1 changes and most readers ignore it; Kronos and Yabause (Saturn) and jgenesis (Sega CD) reject a CHD with any tag they don't know, so the page leaves `--keepcue` out for those consoles, and chdman 0.289 reads, verifies and extracts such CHDs as usual. `extractcd --redump` writes that sheet back, with the names of the files it writes, when those are laid out as the sheet's files (one `BINARY` file per track, or one for a single track); otherwise it writes its own. So what a CHD's tracks don't hold comes back: CATALOG, FLAGS, ISRC, INDEX 02 and later, `CDI/2352`. The CHD's bytes then differ from 0.289's (the plan's tier C), which is why it is off by default, in the page too. - **The codec plan (`createcd`, `createdvd`, `createhd`, `createraw` with `--codecplan`, `-cp`; `chd.cpp`, `chd.h`, `chdcodec.cpp`, `chdman.cpp`), opt-in:** chdman tries every codec in the list on every hunk and keeps the smallest result, though most of them hardly ever win. With the plan, each hunk tries only the codecs meant for what it holds: a CD hunk in audio tracks `cdfl`, one in data tracks `cdlz` (`cdzl` wins about one data hunk in ten, by a few bytes), and a hunk of any other CHD every codec but `flac`. A CD hunk that holds both, or a codec list without those codecs, tries them all. The codec list in the header is unchanged, so every reader decodes the result, and the data and both SHA-1s are the same; the bytes differ where a codec left out would have won (the plan's tier C), so it is off by default. The choice depends only on where a hunk lies, so the CHD is the same at any thread count, and in the page (the codec slots travel with each hunk to the helper workers). Natively, `-np 4`, against the default: CD data 1.8 times less CPU and 0.22% larger, CD audio 1.9 times less and the same size, a mixed CD 2.0 times less and 0.11% larger, a DVD 1.4 times less and 0.06% larger (`docs/chd/measurements.md`). - **libdeflate for the deflate codec (`--libdeflate`, `-ld` on the same four commands; `chd.h`, `chdcodec.cpp`, `chdman.cpp`), opt-in:** `chd_zlib_compressor`, which does `zlib`, `cdzl` and the subcode of `cdlz`, encodes with [libdeflate](libdeflate/README.md) at its level 9 instead of zlib's level 9: standard raw deflate that every reader decodes with zlib, so the checksums are the same, but not zlib's bytes (tier C). Natively, `-np 4`, CPU: with the default codecs 1.1–1.5 times less and a little smaller; with `-c cdzl,cdfl` (the page's "Faster to create") 1.5–1.8 times less and 0.3–0.7% smaller; with the codec plan on a DVD 1.17 times less than the plan alone, and smaller than chdman's default. Helper workers are told with the codecs (`wasm_par_enabled`'s flags). -- **Fixes to 0.289's failure handling and to the descriptors it reads wrong (`chdman.cpp`, `cdrom.cpp`, `cdrom.h`, `chd.cpp`, `chdcodec.cpp`, libFLAC's `lpc.c` and `fixed.c`, LZMA SDK `LzmaEnc.c`); output for input that 0.289 reads right is unchanged, except the `.toc` that `extractcd` writes for a CD with a pregap it holds, and the GD-ROM outputs below that 0.289 can't read back:** +- **Fixes to 0.289's failure handling and to the descriptors it reads wrong (`chdman.cpp`, `cdrom.cpp`, `cdrom.h`, `chd.cpp`, `chdcodec.cpp`, `huffman.cpp`, `avhuff.cpp`, `bitstream.h`, libFLAC's `lpc.c`, `fixed.c` and `stream_encoder.c`, LZMA SDK `LzmaEnc.c`); output for input that 0.289 reads right is unchanged, except the `.toc` that `extractcd` writes for a CD with a pregap it holds, and the GD-ROM outputs below that 0.289 can't read back:** - `verify` exits 1 when the data doesn't match the header's SHA-1, unless `--fix` corrects it. Upstream exits 0, so scripts and the page reported success. - An output is never written over an input: `copy -i x -o x -f` used to truncate `x` and then delete it. - `extractcd` of a CHD that isn't a CD, and a unit or sector size of 0, are errors (exit 1). Upstream aborts on `throw nullptr` or divides by zero. @@ -41,7 +41,7 @@ The paths mirror MAME's, so every file can be compared with its original. **[`ma - Cue sheets (`parse_cue`): a track's `INDEX` points must be in order and not go back in the same file, a track in the same file as the next must end before the next starts, and every track must have frames from its `INDEX 01` on. Upstream made negative pregaps or lengths of them (exit 0, or a read error), and tracks of their pregap alone, whose `.toc` doesn't read back. - `cdrom_file`'s logical reads (`read_data` and `read_subcode` without `phys`, `get_track`), which no chdman command uses but the page's identification does (`wasm_probe_read`): a track's LBAs start at its `INDEX 00`, so a pregap the CHD holds is read from the track's own data, and a pregap or postgap it doesn't hold (or a gap between sessions of a mounted image) reads as zeros. Upstream counted a track's pregap as the track before's, so it read that track's padding and then data shifted by it, and for a gap the CHD doesn't hold the next track's data. LBAs from `INDEX 01` on, which is all the page reads, are read as before. `tests/support/cdprobe.cpp` (`make -C wasm T=native cdprobe`) reads every LBA as the page does. - libFLAC's LPC and fixed-predictor restores (`lpc.c`, `FLAC__lpc_restore_signal`; `fixed.c`, `FLAC__fixed_restore_signal`) compute in unsigned 32-bit arithmetic: on damaged `cdfl` data its signed sums and products overflowed, which C leaves undefined (UBSan flags it). The build wrapped them as two's complement, and unsigned arithmetic gives the same bits, so decoding is unchanged (each checked against the old function, built with `-fwrapv`, on 200,000 random blocks with gcc and clang); the encoder never calls them. - - Undefined behavior UBSan reports, with the same results: the LZMA encoder stored its properties' dictionary size (`LzmaEnc_WriteProperties`, `LzmaEnc.c`) through a `UInt32` pointer at an odd address, now with `memcpy`; the zlib and LZMA allocators (`chdcodec.cpp`) kept each block's size in 4 bytes in front of it, so every block, the codecs' state structures among them, was only 4-byte aligned, and now keep it in 16 (`alignof(std::max_align_t)`); `msf_to_frames` (`cdrom.cpp`) overflowed an `int` on huge minutes, and now clamps (the same results for any time that fits). + - Undefined behavior UBSan reports, with the same results: the LZMA encoder stored its properties' dictionary size (`LzmaEnc_WriteProperties`, `LzmaEnc.c`) through a `UInt32` pointer at an odd address, now with `memcpy`; the zlib and LZMA allocators (`chdcodec.cpp`) kept each block's size in 4 bytes in front of it, so every block, the codecs' state structures among them, was only 4-byte aligned, and now keep it in 16 (`alignof(std::max_align_t)`); `msf_to_frames` (`cdrom.cpp`) overflowed an `int` on huge minutes, and now clamps (the same results for any time that fits); the bit writer (`bitstream.h`) shifted a 32-bit value by 32 when writing no bits (as for a map without copies of hunks), and now returns, as it did after the shift; libFLAC's encoder (`stream_encoder.c`) added an offset to a null pointer when it doesn't use escape codes, and now passes the null pointer; a map or metadata of 0 bytes took element 0 of an empty vector (`chd.cpp`), now `data()`. - Memory that LeakSanitizer reports: the command line's option values (`chdman_run`), `extractcd`'s `cdrom_file`, and after an error, the compression work items still out (`~chd_file_compressor`). Other allocations on the error paths are freed as `report_error` unwinds, and descriptor parsers that return early without closing their file (upstream's) are left as they are: each runs once per command. - `cdrom_file::get_track_index` (not used by chdman or the page) gives 0 for a frame in a track's pregap: since `logical_to_chd_lba` gives the pregap to its own track (above), counting back from `INDEX 01` there wrapped. - An `.iso` of 2352-byte sectors (`parse_iso`) is typed by its first two sectors' sync pattern and mode byte as well as its size: upstream typed it by size alone, so an image of 128·n sectors (also a multiple of 2048 bytes) came out as 2048-byte `MODE1` sectors, and one of 146·n (a multiple of 2336) as 2336-byte `MODE2`. Such an image now gives `MODE2_RAW`, as upstream types every other image of 2352-byte sectors; images of 2048- and 2336-byte sectors are upstream's. @@ -52,11 +52,21 @@ The paths mirror MAME's, so every file can be compared with its original. **[`ma - `extractcd` of a CHD with a damaged hunk is an error, as with the other extract commands. Upstream writes whatever it last read and exits 0. - A FLAC stream that ends before its hunk is complete (damaged `flac` or `cdfl` data) is a decompression error: upstream's decoder asks for more forever (`flac.cpp`). - A hunk stored as a copy of another hunk of the same file is followed in a loop, to the hunk that holds the data, and a chain that leads back to itself is an error (`self_target`, `chd.cpp`). Upstream calls `read_hunk` again for each step, so a map that makes a hunk a copy of itself, as a damaged or crafted file can, overflows the stack. Copies of copies that end are read as before. - - A hunk whose stored length is larger than a hunk is an error (`read_hunk`, `codec_process_hunk`): upstream reads that many bytes into its hunk-sized buffer. + - A hunk whose stored length is larger than a hunk is an error (`read_hunk`, `codec_process_hunk`): upstream reads that many bytes into its hunk-sized buffer. So is a hunk compressed with a codec the CHD has none for (a v3/v4 CHD without compression whose map says a hunk is compressed, a v5 map naming an empty codec slot), and a hunk in a parent in a CHD that has none: upstream calls through a null pointer. - A CHD of a hunk or three can be read: upstream sizes the map's buffer without the Huffman tree that opens it, so the map overflows and is cut short, and the CHD it made (exit 0) fails with "Decompression error" (`compress_v5_map`, `chd.cpp`). Where upstream's map fits, which is every CHD of more than a few hunks, the bytes are the same. - The CD codecs compress the subcode into what is left of the hunk-sized buffer; upstream lets it run past the end when the sectors hardly compress (a heap overflow). Such a result is too large to win either way, so the CHD is the same (`chdcodec.cpp`). - `dumpmeta` without `-o` prints its banner on stderr, so stdout carries only the metadata. + - A header with a hunk or unit size of 0, or a logical size of more hunks than a map can count (2^32), is an error ("Invalid file"), and so is a map whose size in memory doesn't fit in 32 bits (`check_sizes`, `create_open_common`, `chd.cpp`). Upstream divides by zero, or computes the size in 32 bits and makes a map too small for its hunks, then reads and writes past its end (a segfault at 358 million hunks). + - A map that would run past the end of the file is the error reading it gives ("Input/output error") before its buffer is allocated: upstream allocated what the header said, up to 4 GB, first (`past_end`, `chd.cpp`). + - The Huffman lookup table's entries that no code covers (only a damaged tree leaves any) decode as code 0 of no bits (`build_lookup_table`, `huffman.cpp`): upstream never set them, so decoding took bit counts from whatever that memory held. + - A Huffman tree with more codes than one bit has room for (a damaged `huff` hunk, LaserDisc audio, or a v5 map's tree) is a decompression error (`assign_canonical_codes`, `huffman.cpp`): upstream gave codes more bits than their length and wrote the lookup table past its end, on the stack for a map or a `huff` hunk ("stack smashing detected"). + - A CD codec's hunk (`cdlz`, `cdzl`, `cdzs`) whose header is longer than the hunk's data, or says its sector data is, is a decompression error (`chdcodec.cpp`): upstream decoded past the end of the data, and the subcode's length wrapped around. So is a v5 map that reads more than 32 bits at a time (`decompress_v5_map`): upstream's bit reader shifted by a negative count. A field of exactly 32 bits is read as such (`bitstream_in::remove` empties the buffer): upstream shifted the buffer by 32, which C++ leaves undefined, and in practice kept its bits, so the next field was read with them. + - CD track metadata is read with a width for each string (`%15s`, the longest a CD has being 15 characters): upstream's `%s` wrote a longer one past the end of its 16-byte buffer, on the stack (`parse_metadata`, `cdrom.cpp`). + - CD metadata that leaves out a track (a track number repeated or skipped), and old-style binary CD metadata (`CHCD`) shorter than its table, with more than 99 tracks or with sectors no CD has, are invalid data (`parse_metadata`, `cdrom.cpp`): upstream made a track of 0-byte sectors, which `extractcd` divided by, or read past the end of the metadata and of a sector's buffer. + - Reading or writing 0 bytes (`read_bytes`, `write_bytes`) does nothing: upstream's last hunk wrapped around at offset 0, and it read a whole hunk into the caller's buffer. The page's identification reads a CD track that has no metadata (a damaged CHD's) that way. + - LaserDisc (`avhu`) hunks: a frame larger than the hunk, more than 16 audio channels, or raw audio shorter than its samples is a decompression error (`chdcodec.cpp`, `avhuff.cpp`). Upstream decoded past the end of the hunk, of its tables of channels, and of the data. - Any other unexpected exception is an error instead of a crash. + - `tests/fuzz/` fuzzes the engine (libFuzzer, ASan and UBSan; `scripts/fuzz.sh`, `.github/workflows/fuzz.yml`), and found the faults above in maps, Huffman trees and the CD and LaserDisc codecs (the map size in 32 bits while looking at one), and the undefined behavior below. - `tests/ui/engine.spec.js` compares each of these with upstream. ## Contents diff --git a/engine/mame-0.289.diff b/engine/mame-0.289.diff index 05e76b8..6f7d80d 100644 --- a/engine/mame-0.289.diff +++ b/engine/mame-0.289.diff @@ -467,6 +467,18 @@ if(bps >= 0.0) return bps; else +--- a/3rdparty/flac/src/libFLAC/stream_encoder.c ++++ b/3rdparty/flac/src/libFLAC/stream_encoder.c +@@ -4126,7 +4126,8 @@ + residual, + #endif + abs_residual_partition_sums+sum, +- raw_bits_per_partition+sum, ++ /* Discpress: null without escape coding, which C doesn't let an offset be added to */ ++ raw_bits_per_partition != NULL ? raw_bits_per_partition+sum : NULL, + residual_samples, + predictor_order, + rice_parameter_limit, --- a/3rdparty/flac/src/libFLAC/window.c +++ b/3rdparty/flac/src/libFLAC/window.c @@ -39,6 +39,7 @@ @@ -657,6 +669,63 @@ *destLen -= (SizeT)outStream.rem; if (outStream.overflow) +--- a/src/lib/util/avhuff.cpp ++++ b/src/lib/util/avhuff.cpp +@@ -67,6 +67,7 @@ + #include + #include + #include ++#include + #include + + +@@ -723,6 +724,11 @@ + uint32_t width = get_u16be(&source[4]); + uint32_t height = get_u16be(&source[6]); + ++ // Discpress: no more channels than there are places for (the encoder takes up to 16): 0.289 went past ++ // the end of audiostart and of m_config.audio ++ if (channels > std::size(m_config.audio)) ++ return AVHERR_INVALID_DATA; ++ + // validate that the sizes make sense + if (complength < 10 + 2 * channels) + return AVHERR_INVALID_DATA; +@@ -932,6 +938,10 @@ + // if no huffman length, just copy the data + if (treesize == 0) + { ++ // Discpress: as much as the samples take (as stored): 0.289 read past the channel's ++ // data, and past the end of the hunk's ++ if (size < 2 * samples) ++ return AVHERR_INVALID_DATA; + const uint8_t *cursource = source; + for (int sampnum = 0; sampnum < samples; sampnum++) + { +--- a/src/lib/util/bitstream.h ++++ b/src/lib/util/bitstream.h +@@ -146,7 +146,8 @@ + + inline void bitstream_in::remove(int numbits) + { +- m_buffer <<= numbits; ++ // (all 32 bits, which a crafted v5 map's field width can ask for: the buffer is empty, not shifted by 32) ++ m_buffer = (numbits < 32) ? (m_buffer << numbits) : 0; + m_bits -= numbits; + } + +@@ -226,6 +227,11 @@ + + inline void bitstream_out::write(uint32_t newbits, int numbits) + { ++ // Discpress: nothing to write, as for a CHD map with no copies of hunks (0.289 went on to shift ++ // the value by 32 bits, which C++ leaves undefined, and then returned without using it) ++ if (numbits <= 0) ++ return; ++ + newbits <<= 32 - numbits; + + // flush the buffer if we're going to overflow it --- a/src/lib/util/cdrom.cpp +++ b/src/lib/util/cdrom.cpp @@ -28,6 +28,8 @@ @@ -722,7 +791,85 @@ const uint32_t index_offset = frame - track_start; int index = 0; -@@ -1153,204 +1177,106 @@ +@@ -897,6 +921,13 @@ + * @return A std::error_condition. + */ + ++// Discpress: chd.cpp's formats for the track metadata, with a width for each string that fills its 16-byte ++// buffer at most (the longest a CD has is 15 characters, VMODE2_FORM_MIX); a longer one, in damaged or ++// crafted metadata, doesn't match. 0.289 read them with %s, which wrote past the end of the buffer. ++static const char CDROM_TRACK_METADATA_FORMAT_READ[] = "TRACK:%d TYPE:%15s SUBTYPE:%15s FRAMES:%d"; ++static const char CDROM_TRACK_METADATA2_FORMAT_READ[] = "TRACK:%d TYPE:%15s SUBTYPE:%15s FRAMES:%d PREGAP:%d PGTYPE:%15s PGSUB:%15s POSTGAP:%d"; ++static const char GDROM_TRACK_METADATA_FORMAT_READ[] = "TRACK:%d TYPE:%15s SUBTYPE:%15s FRAMES:%d PAD:%d PREGAP:%d PGTYPE:%15s PGSUB:%15s POSTGAP:%d"; ++ + std::error_condition cdrom_file::parse_metadata(chd_file *chd, toc &toc) + { + std::string metadata; +@@ -924,12 +955,12 @@ + // fetch the metadata for this track + if (!chd->read_metadata(CDROM_TRACK_METADATA_TAG, toc.numtrks, metadata)) + { +- if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA_FORMAT, &tracknum, type, subtype, &frames) != 4) ++ if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA_FORMAT_READ, &tracknum, type, subtype, &frames) != 4) + return chd_file::error::INVALID_DATA; + } + else if (!chd->read_metadata(CDROM_TRACK_METADATA2_TAG, toc.numtrks, metadata)) + { +- if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA2_FORMAT, &tracknum, type, subtype, &frames, &pregap, pgtype, pgsub, &postgap) != 8) ++ if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA2_FORMAT_READ, &tracknum, type, subtype, &frames, &pregap, pgtype, pgsub, &postgap) != 8) + return chd_file::error::INVALID_DATA; + } + else +@@ -944,7 +975,7 @@ + if (err) + break; + +- if (sscanf(metadata.c_str(), GDROM_TRACK_METADATA_FORMAT, &tracknum, type, subtype, &frames, &padframes, &pregap, pgtype, pgsub, &postgap) != 9) ++ if (sscanf(metadata.c_str(), GDROM_TRACK_METADATA_FORMAT_READ, &tracknum, type, subtype, &frames, &padframes, &pregap, pgtype, pgsub, &postgap) != 9) + return chd_file::error::INVALID_DATA; + + toc.flags |= CD_FLAG_GDROM; +@@ -995,7 +1026,14 @@ + + /* if we got any tracks this way, we're done */ + if (toc.numtrks > 0) ++ { ++ // Discpress: with each track once: a track number that repeats or skips one (damaged or crafted ++ // metadata) left a track of nothing, of 0-byte sectors, which extractcd divided by ++ for (uint32_t i = 0; i < toc.numtrks; i++) ++ if (toc.tracks[i].datasize == 0) ++ return chd_file::error::INVALID_DATA; + return std::error_condition(); ++ } + + osd_printf_info("toc.numtrks = %u?!\n", toc.numtrks); + +@@ -1004,6 +1042,9 @@ + err = chd->read_metadata(CDROM_OLD_METADATA_TAG, 0, oldmetadata); + if (err) + return err; ++ // Discpress: as long as the table it holds (0.289 read past the end of shorter metadata) ++ if (oldmetadata.size() < 4 + MAX_TRACKS * 6 * 4) ++ return chd_file::error::INVALID_DATA; + + /* reconstruct the TOC from it */ + auto *mrp = reinterpret_cast(&oldmetadata[0]); +@@ -1044,6 +1085,14 @@ + } + } + ++ // Discpress: no more tracks than the table holds, with sectors a CD has: anything else (damaged or ++ // crafted metadata) made 0.289 read past the end of the table, and of a sector's buffer ++ if (toc.numtrks > MAX_TRACKS) ++ return chd_file::error::INVALID_DATA; ++ for (uint32_t i = 0; i < toc.numtrks; i++) ++ if (toc.tracks[i].datasize == 0 || toc.tracks[i].datasize > MAX_SECTOR_DATA || toc.tracks[i].subsize > MAX_SUBCODE_DATA) ++ return chd_file::error::INVALID_DATA; ++ + return std::error_condition(); + } + +@@ -1153,204 +1202,106 @@ 0x50, 0xa4, 0xa5, 0x51, 0xa7, 0x53, 0x52, 0xa6, 0xa3, 0x57, 0x56, 0xa2, 0x54, 0xa0, 0xa1, 0x55 }; @@ -962,10 +1109,7 @@ + uint64_t a[11] = { }, b[11] = { }; + int const count = q ? ECC_Q_NUM_BYTES : ECC_P_NUM_BYTES; + if (!q) - { -- val1 ^= ecc_source_byte(sector, row[component]); -- val2 ^= ecc_source_byte(sector, row[component]); -- val1 = ecclow[val1]; ++ { + // the columns of the first 24 rows (a row's last word also reads 2 bytes of the next row, unused) + for (int row = 0; row < ECC_P_COMP; row++) + for (int l = 0; l < 11; l++) @@ -1000,7 +1144,10 @@ + memcpy(av, a, sizeof(a)); + memcpy(bv, b, sizeof(b)); + for (int i = 0; i < count; i++) -+ { + { +- val1 ^= ecc_source_byte(sector, row[component]); +- val2 ^= ecc_source_byte(sector, row[component]); +- val1 = ecclow[val1]; + uint8_t const p0 = ecchigh[ecclow[av[i]] ^ bv[i]]; + parity[i] = p0; + parity[count + i] = p0 ^ bv[i]; @@ -1010,7 +1157,7 @@ } /** -@@ -1367,24 +1293,17 @@ +@@ -1367,24 +1318,17 @@ bool cdrom_file::ecc_verify(const uint8_t *sector) { @@ -1043,7 +1190,7 @@ } /** -@@ -1400,13 +1319,13 @@ +@@ -1400,13 +1344,13 @@ void cdrom_file::ecc_generate(uint8_t *sector) { @@ -1063,7 +1210,7 @@ } /** -@@ -1578,22 +1497,25 @@ +@@ -1578,22 +1522,25 @@ int cdrom_file::msf_to_frames(const char *token) { @@ -1103,7 +1250,7 @@ } /*------------------------------------------------- -@@ -1604,17 +1526,18 @@ +@@ -1604,17 +1551,18 @@ -------------------------------------------------*/ /** @@ -1124,7 +1271,7 @@ { unsigned long offset = 0; uint32_t length, rate, filesize; -@@ -1633,7 +1556,7 @@ +@@ -1633,7 +1581,7 @@ } /* read the core header and make sure it's a WAVE file */ @@ -1133,7 +1280,7 @@ offset += actual; if (offset < 4) { -@@ -1647,7 +1570,7 @@ +@@ -1647,7 +1595,7 @@ } /* get the total size */ @@ -1142,7 +1289,7 @@ offset += actual; if (offset < 8) { -@@ -1657,7 +1580,7 @@ +@@ -1657,7 +1605,7 @@ filesize = little_endianize_int32(filesize); /* read the RIFF file type and make sure it's a WAVE file */ @@ -1151,7 +1298,7 @@ offset += actual; if (offset < 12) { -@@ -1673,9 +1596,9 @@ +@@ -1673,9 +1621,9 @@ /* seek until we find a format tag */ while (true) { @@ -1163,7 +1310,7 @@ offset += actual; length = little_endianize_int32(length); if (memcmp(&buf[0], "fmt ", 4) == 0) -@@ -1691,7 +1614,7 @@ +@@ -1691,7 +1639,7 @@ } /* read the format -- make sure it is PCM */ @@ -1172,7 +1319,7 @@ offset += actual; temp16 = little_endianize_int16(temp16); if (temp16 != 1) -@@ -1701,7 +1624,7 @@ +@@ -1701,7 +1649,7 @@ } /* number of channels -- only stereo is supported */ @@ -1181,7 +1328,7 @@ offset += actual; temp16 = little_endianize_int16(temp16); if (temp16 != 2) -@@ -1711,7 +1634,7 @@ +@@ -1711,7 +1659,7 @@ } /* sample rate */ @@ -1190,7 +1337,7 @@ offset += actual; rate = little_endianize_int32(rate); if (rate != 44100) -@@ -1721,11 +1644,11 @@ +@@ -1721,11 +1669,11 @@ } /* bytes/second and block alignment are ignored */ @@ -1204,7 +1351,7 @@ offset += actual; bits = little_endianize_int16(bits); if (bits != 16) -@@ -1740,9 +1663,9 @@ +@@ -1740,9 +1688,9 @@ /* seek until we find a data tag */ while (true) { @@ -1216,7 +1363,7 @@ offset += actual; length = little_endianize_int32(length); if (memcmp(&buf[0], "data", 4) == 0) -@@ -1764,7 +1687,12 @@ +@@ -1764,7 +1712,12 @@ return 0; } @@ -1230,7 +1377,7 @@ return length; } -@@ -1864,11 +1792,11 @@ +@@ -1864,11 +1817,11 @@ outtoc.numsessions = 1; @@ -1246,7 +1393,7 @@ { osd_printf_error("ERROR: Not a Nero 5.5 or later image!\n"); fclose(infile); -@@ -1888,8 +1816,13 @@ +@@ -1888,8 +1841,13 @@ while (!done) { @@ -1262,7 +1409,7 @@ chunk_size = get_u32be(&buffer[4]); -@@ -1901,15 +1834,20 @@ +@@ -1901,15 +1859,20 @@ // skip second chunk size and UPC code fseek(infile, 20, SEEK_CUR); @@ -1285,7 +1432,7 @@ for (int track = start; track <= end; track++) { uint32_t size, mode; -@@ -1925,7 +1863,15 @@ +@@ -1925,7 +1888,15 @@ // printf("Track %d: sector size %d mode %x index0 %llx index1 %llx track_end %llx (pregap %d sectors, length %d sectors)\n", track, size, mode, index0, index1, track_end, (uint32_t)(index1-index0)/size, (uint32_t)(track_end-index1)/size); outinfo.track[track-1].fname.assign(tocfname); @@ -1302,7 +1449,7 @@ outinfo.track[track-1].idx[0] = outinfo.track[track-1].idx[1] = 0; switch (mode) -@@ -1976,21 +1922,29 @@ +@@ -1976,21 +1947,29 @@ return chd_file::error::UNSUPPORTED_FORMAT; } @@ -1337,7 +1484,7 @@ } } -@@ -2000,7 +1954,15 @@ +@@ -2000,7 +1979,15 @@ } else { @@ -1354,7 +1501,7 @@ } } -@@ -2042,6 +2004,20 @@ +@@ -2042,6 +2029,20 @@ outinfo.reset(); uint64_t size = get_file_size(tocfname); @@ -1375,7 +1522,7 @@ fclose(infile); -@@ -2052,7 +2028,15 @@ +@@ -2052,7 +2053,15 @@ outinfo.track[0].offset = 0; outinfo.track[0].idx[0] = outinfo.track[0].idx[1] = 0; @@ -1392,7 +1539,7 @@ { outtoc.tracks[0].trktype = CD_TRACK_MODE1; outtoc.tracks[0].frames = size / 2048; -@@ -2149,7 +2133,7 @@ +@@ -2149,7 +2158,7 @@ const int numtracks = atoi(token); @@ -1401,7 +1548,7 @@ { osd_printf_error("GDI expects too many tracks. Expected %d tracks but only up to %zu tracks allowed\n", numtracks, std::size(outinfo.track) + 1); return chd_file::error::INVALID_DATA; -@@ -2178,7 +2162,7 @@ +@@ -2178,7 +2187,7 @@ paramcnt++; const int trknum = atoi(token) - 1; @@ -1410,7 +1557,7 @@ { osd_printf_error("Track %d is out of expected range of 1 to %d\n", trknum + 1, numtracks); return chd_file::error::INVALID_DATA; -@@ -2222,7 +2206,7 @@ +@@ -2222,7 +2231,7 @@ outtoc.tracks[trknum].trktype = CD_TRACK_MODE1; outtoc.tracks[trknum].datasize = 2048; } @@ -1419,7 +1566,7 @@ { outtoc.tracks[trknum].trktype = CD_TRACK_AUDIO; outtoc.tracks[trknum].datasize = 2352; -@@ -2252,6 +2236,12 @@ +@@ -2252,6 +2261,12 @@ outinfo.track[trknum].fname.assign(path).append(token); const uint64_t sz = get_file_size(outinfo.track[trknum].fname); @@ -1432,7 +1579,7 @@ outtoc.tracks[trknum].frames = sz / trksize; outtoc.tracks[trknum].padframes = 0; -@@ -2345,6 +2335,19 @@ +@@ -2345,6 +2360,19 @@ enum gdi_area current_area = SINGLE_DENSITY; bool is_multibin = false; int leadin = -1; @@ -1452,7 +1599,7 @@ FILE *infile = fopen(path.c_str(), "rt"); if (!infile) -@@ -2371,6 +2374,14 @@ +@@ -2371,6 +2399,14 @@ char linebuffer[512]; memset(linebuffer, 0, sizeof(linebuffer)); @@ -1467,7 +1614,7 @@ while (!feof(infile)) { /* get the next line */ -@@ -2427,6 +2438,8 @@ +@@ -2427,6 +2463,8 @@ /* get lead-out time */ TOKENIZE int leadout_offset = msf_to_frames(token); @@ -1476,7 +1623,7 @@ outinfo.track[trknum].leadout = leadout_offset; } else if (!strncmp(linebuffer+i, "LEAD-IN", 7)) -@@ -2472,13 +2485,18 @@ +@@ -2472,13 +2510,18 @@ /* get the file type */ TOKENIZE @@ -1497,7 +1644,7 @@ } else if (!strcmp(token, "WAVE")) { -@@ -2489,6 +2507,8 @@ +@@ -2489,6 +2532,8 @@ osd_printf_error("ERROR: couldn't read [%s] or not a valid .WAV\n", lastfname); return chd_file::error::INVALID_DATA; } @@ -1506,7 +1653,7 @@ } else { -@@ -2502,6 +2522,12 @@ +@@ -2502,6 +2547,12 @@ /* get the track number */ TOKENIZE trknum = strtoul(token, nullptr, 10) - 1; @@ -1519,7 +1666,7 @@ /* next token on the line is the track type */ TOKENIZE -@@ -2541,6 +2567,10 @@ +@@ -2541,6 +2592,10 @@ outinfo.track[trknum].offset = wavoffs; wavoffs = wavlen = 0; } @@ -1530,7 +1677,7 @@ outinfo.track[trknum].fname.assign(lastfname); /* default filename to the last one */ -@@ -2586,6 +2616,54 @@ +@@ -2586,6 +2641,54 @@ osd_printf_error("ERROR: encountered invalid index %d\n", idx); return chd_file::error::INVALID_DATA; } @@ -1585,7 +1732,7 @@ outinfo.track[trknum].idx[idx] = frames; -@@ -2610,6 +2688,8 @@ +@@ -2610,6 +2713,8 @@ /* get index */ TOKENIZE frames = msf_to_frames(token); @@ -1594,7 +1741,7 @@ outtoc.tracks[trknum].pregap = frames; } -@@ -2620,11 +2700,15 @@ +@@ -2620,11 +2725,15 @@ /* get index */ TOKENIZE frames = msf_to_frames(token); @@ -1610,7 +1757,7 @@ outtoc.tracks[trknum].control_flags = 0; /* keep looping over remaining tokens in FLAGS line until there's no more to read */ -@@ -2667,18 +2751,51 @@ +@@ -2667,18 +2776,51 @@ } /* this is true for cue/bin and cue/iso, and we need it for cue/wav since .WAV is little-endian */ @@ -1664,7 +1811,7 @@ tlen = get_file_size(outinfo.track[trknum].fname); if (tlen == 0) { -@@ -2700,11 +2817,17 @@ +@@ -2700,11 +2842,17 @@ return chd_file::error::INVALID_DATA; } @@ -1683,7 +1830,7 @@ } else if (outtoc.tracks[trknum].frames == 0) { -@@ -2777,6 +2900,77 @@ +@@ -2777,6 +2925,77 @@ } } @@ -1761,7 +1908,7 @@ if (is_gdrom) { /* -@@ -2787,6 +2981,18 @@ +@@ -2787,6 +3006,18 @@ uint32_t this_pregap = outtoc.tracks[trknum].pregap; uint32_t this_offset = this_pregap * (outtoc.tracks[trknum].datasize + outtoc.tracks[trknum].subsize); @@ -1780,7 +1927,7 @@ outtoc.tracks[trknum-1].frames += this_pregap; outtoc.tracks[trknum-1].splitframes += this_pregap; -@@ -2808,7 +3014,7 @@ +@@ -2808,7 +3039,7 @@ outtoc.tracks[trknum].physframeofs = 45000; int dif=outtoc.tracks[trknum].physframeofs-(outtoc.tracks[trknum-1].frames+outtoc.tracks[trknum-1].physframeofs); outtoc.tracks[trknum-1].frames += dif; @@ -1789,7 +1936,7 @@ } else { -@@ -2965,6 +3171,67 @@ +@@ -2965,6 +3196,67 @@ char linebuffer[512]; memset(linebuffer, 0, sizeof(linebuffer)); @@ -1857,7 +2004,7 @@ while (!feof(infile)) { /* get the next line */ -@@ -2989,6 +3256,8 @@ +@@ -2989,6 +3281,8 @@ */ if (!strcmp(token, "NO")) { @@ -1866,7 +2013,7 @@ TOKENIZE if (!strcmp(token, "COPY")) outtoc.tracks[trknum].control_flags &= ~CD_FLAG_CONTROL_DIGITAL_COPY_PERMITTED; -@@ -2997,100 +3266,210 @@ +@@ -2997,100 +3291,210 @@ } else if (!strcmp(token, "COPY")) { @@ -2120,7 +2267,7 @@ /* next token on the line is the track type */ TOKENIZE -@@ -3117,22 +3496,128 @@ +@@ -3117,22 +3521,128 @@ } else if (!strcmp(token, "START")) { @@ -2324,7 +2471,11 @@ static uint64_t read_uint64(FILE *infile); --- a/src/lib/util/chd.cpp +++ b/src/lib/util/chd.cpp -@@ -28,6 +28,20 @@ +@@ -25,9 +25,24 @@ + #include + #include + #include ++#include #include #include @@ -2345,7 +2496,7 @@ //************************************************************************** // CONSTANTS -@@ -853,6 +867,17 @@ +@@ -853,6 +868,17 @@ void chd_file::close() { @@ -2363,7 +2514,7 @@ // reset file characteristics m_file.reset(); m_allow_reads = false; -@@ -892,6 +917,48 @@ +@@ -892,6 +918,48 @@ m_cachehunk = ~0; } @@ -2412,17 +2563,20 @@ std::error_condition chd_file::codec_process_hunk(uint32_t hunknum) { // punt if no file -@@ -919,6 +986,9 @@ +@@ -919,6 +987,12 @@ case V34_MAP_ENTRY_TYPE_COMPRESSED: { uint32_t const blocklen = get_u16be(&rawmap[12]) | (uint32_t(rawmap[14]) << 16); + // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end + if (UNEXPECTED(blocklen > m_compressed.size())) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: and a hunk compressed in a CHD that says it isn't: 0.289 called a codec it doesn't have ++ if (UNEXPECTED(!m_decompressor[0])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; -@@ -931,7 +1001,14 @@ +@@ -931,11 +1005,21 @@ return std::error_condition(error::UNSUPPORTED_FORMAT); case V34_MAP_ENTRY_TYPE_SELF_HUNK: @@ -2438,17 +2592,27 @@ case V34_MAP_ENTRY_TYPE_PARENT_HUNK: if (UNEXPECTED(m_parent_missing)) -@@ -958,6 +1035,9 @@ + return std::error_condition(error::REQUIRES_PARENT); ++ // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer ++ if (UNEXPECTED(!m_parent)) ++ return std::error_condition(error::INVALID_DATA); + return m_parent->codec_process_hunk(blockoffs); + } + } +@@ -958,6 +1042,12 @@ case COMPRESSION_TYPE_2: case COMPRESSION_TYPE_3: { + // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end + if (UNEXPECTED(blocklen > m_compressed.size())) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: and a codec slot the header leaves empty: 0.289 called a codec it doesn't have ++ if (UNEXPECTED(!m_decompressor[rawmap[0]])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; -@@ -970,7 +1050,14 @@ +@@ -970,11 +1060,24 @@ return std::error_condition(error::UNSUPPORTED_FORMAT); case COMPRESSION_SELF: @@ -2464,17 +2628,30 @@ case COMPRESSION_PARENT: if (UNEXPECTED(m_parent_missing)) -@@ -1045,6 +1132,9 @@ + return std::error_condition(error::REQUIRES_PARENT); ++ // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer ++ if (UNEXPECTED(!m_parent)) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: a parent whose units are larger than its hunks has no hunk for a unit (0.289 divided by zero) ++ if (UNEXPECTED(m_parent->unit_bytes() > m_parent->hunk_bytes())) ++ return std::error_condition(error::INVALID_DATA); + return m_parent->codec_process_hunk(blockoffs / (m_parent->hunk_bytes() / m_parent->unit_bytes())); + } + break; +@@ -1045,6 +1148,12 @@ case V34_MAP_ENTRY_TYPE_COMPRESSED: { uint32_t const blocklen = get_u16be(&rawmap[12]) | (uint32_t(rawmap[14]) << 16); + // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end + if (UNEXPECTED(blocklen > m_compressed.size())) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: and a hunk compressed in a CHD that says it isn't: 0.289 called a codec it doesn't have ++ if (UNEXPECTED(!m_decompressor[0])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; -@@ -1073,7 +1163,14 @@ +@@ -1073,11 +1182,21 @@ return std::error_condition(); case V34_MAP_ENTRY_TYPE_SELF_HUNK: @@ -2490,7 +2667,14 @@ case V34_MAP_ENTRY_TYPE_PARENT_HUNK: if (UNEXPECTED(m_parent_missing)) -@@ -1115,6 +1212,14 @@ + return std::error_condition(error::REQUIRES_PARENT); ++ // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer ++ if (UNEXPECTED(!m_parent)) ++ return std::error_condition(error::INVALID_DATA); + return m_parent->read_hunk(blockoffs, dest); + } + } +@@ -1115,6 +1234,17 @@ case COMPRESSION_TYPE_2: case COMPRESSION_TYPE_3: { @@ -2501,11 +2685,14 @@ +#endif + // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end + if (UNEXPECTED(blocklen > m_compressed.size())) ++ return std::error_condition(error::INVALID_DATA); ++ // Discpress: and a codec slot the header leaves empty: 0.289 called a codec it doesn't have ++ if (UNEXPECTED(!m_decompressor[rawmap[0]])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; -@@ -1130,16 +1235,29 @@ +@@ -1130,20 +1260,36 @@ case COMPRESSION_NONE: { @@ -2536,7 +2723,169 @@ case COMPRESSION_PARENT: if (UNEXPECTED(m_parent_missing)) -@@ -2173,6 +2291,10 @@ + return std::error_condition(error::REQUIRES_PARENT); ++ // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer ++ if (UNEXPECTED(!m_parent)) ++ return std::error_condition(error::INVALID_DATA); + return m_parent->read_bytes(blockoffs * m_parent->unit_bytes(), dest, m_hunkbytes); + } + } +@@ -1307,6 +1453,11 @@ + + std::error_condition chd_file::read_bytes(uint64_t offset, void *buffer, uint32_t bytes) + { ++ // Discpress: nothing to read, as for a CD track without data (a damaged or crafted CHD's): 0.289's ++ // last hunk wrapped around at offset 0, and it read a whole hunk into the buffer ++ if (bytes == 0) ++ return std::error_condition(); ++ + // iterate over hunks + uint32_t const first_hunk = offset / m_hunkbytes; + uint32_t const last_hunk = (offset + bytes - 1) / m_hunkbytes; +@@ -1360,6 +1511,10 @@ + + std::error_condition chd_file::write_bytes(uint64_t offset, const void *buffer, uint32_t bytes) + { ++ // Discpress: nothing to write (as read_bytes) ++ if (bytes == 0) ++ return std::error_condition(); ++ + // iterate over hunks + uint32_t const first_hunk = offset / m_hunkbytes; + uint32_t const last_hunk = (offset + bytes - 1) / m_hunkbytes; +@@ -1453,7 +1608,7 @@ + // read the metadata + try { output.resize(metaentry.length); } + catch (std::bad_alloc const &) { return std::errc::not_enough_memory; } +- return file_read(metaentry.offset + METADATA_HEADER_SIZE, &output[0], metaentry.length); ++ return file_read(metaentry.offset + METADATA_HEADER_SIZE, output.data(), metaentry.length); // Discpress: data(), as metadata can be empty + } + + /** +@@ -1515,7 +1670,7 @@ + // read the metadata + try { output.resize(metaentry.length); } + catch (std::bad_alloc const &) { return std::errc::not_enough_memory; } +- err = file_read(metaentry.offset + METADATA_HEADER_SIZE, &output[0], metaentry.length); ++ err = file_read(metaentry.offset + METADATA_HEADER_SIZE, output.data(), metaentry.length); // Discpress: data(), as metadata can be empty + if (UNEXPECTED(err)) + return err; + resulttag = metaentry.metatag; +@@ -1678,12 +1833,12 @@ + // read the metadata item + try { filedata.resize(metaentry.length); } + catch (std::bad_alloc const &) { return std::errc::not_enough_memory; } +- err = source.file_read(metaentry.offset + METADATA_HEADER_SIZE, &filedata[0], metaentry.length); ++ err = source.file_read(metaentry.offset + METADATA_HEADER_SIZE, filedata.data(), metaentry.length); // Discpress: data(), as metadata can be empty + if (UNEXPECTED(err)) + return err; + + // write it to the destination +- err = write_metadata(metaentry.metatag, (uint32_t)-1, &filedata[0], metaentry.length, metaentry.flags); ++ err = write_metadata(metaentry.metatag, (uint32_t)-1, filedata.data(), metaentry.length, metaentry.flags); + if (UNEXPECTED(err)) + return err; + } +@@ -1725,14 +1880,14 @@ + + // allocate memory and read the data + filedata.resize(metaentry.length); +- err = file_read(metaentry.offset + METADATA_HEADER_SIZE, &filedata[0], metaentry.length); ++ err = file_read(metaentry.offset + METADATA_HEADER_SIZE, filedata.data(), metaentry.length); // Discpress: data(), as metadata can be empty + if (UNEXPECTED(err)) + throw err; + + // create an entry for this metadata and add it + metadata_hash hashentry; + put_u32be(hashentry.tag, metaentry.metatag); +- hashentry.sha1 = util::sha1_creator::simple(&filedata[0], metaentry.length); ++ hashentry.sha1 = util::sha1_creator::simple(filedata.data(), metaentry.length); + hasharray.push_back(hashentry); + } + if (err != error::METADATA_NOT_FOUND) +@@ -1880,6 +2035,41 @@ + } + + /** ++ * @fn bool chd_file::past_end(uint64_t offset, uint64_t length) const ++ * @brief past_end - Discpress: whether length bytes from offset would run past the end of the ++ * file (false when its length is unknown, so that the read reports it) ++ */ ++bool chd_file::past_end(uint64_t offset, uint64_t length) const ++{ ++ uint64_t filelength; ++ if (!m_file || m_file->length(filelength)) ++ return false; ++ return (offset > filelength) || (length > filelength - offset); ++} ++ ++/** ++ * @fn void chd_file::check_sizes(uint64_t hunkcount) const ++ * ++ * @brief ------------------------------------------------- ++ * check_sizes - Discpress: a header whose sizes no data can have (a damaged or ++ * crafted file) is an error. 0.289 divided by a hunk or unit size of 0, and with a ++ * logical size of more hunks than a map can count, it made a map too small for them ++ * -------------------------------------------------. ++ * ++ * @exception CHDERR_INVALID_FILE Thrown when the sizes can't be right. ++ * ++ * @param hunkcount The number of hunks the header gives or implies. ++ */ ++ ++void chd_file::check_sizes(uint64_t hunkcount) const ++{ ++ if (UNEXPECTED(m_hunkbytes == 0 || m_unitbytes == 0)) ++ throw std::error_condition(error::INVALID_FILE); ++ if (UNEXPECTED(hunkcount > std::numeric_limits::max())) ++ throw std::error_condition(error::INVALID_FILE); ++} ++ ++/** + * @fn void chd_file::parse_v3_header(uint8_t *rawheader, sha1_t &parentsha1) + * + * @brief ------------------------------------------------- +@@ -1938,7 +2128,8 @@ + parentsha1 = be_read_sha1(&rawheader[m_parentsha1_offset]); + + // guess at the units based on snooping the metadata +- m_unitbytes = guess_unitbytes(); ++ m_unitbytes = (m_hunkbytes != 0) ? guess_unitbytes() : 0; ++ check_sizes(m_hunkcount); + m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes; + } + +@@ -2001,7 +2192,8 @@ + parentsha1 = be_read_sha1(&rawheader[m_parentsha1_offset]); + + // guess at the units based on snooping the metadata +- m_unitbytes = guess_unitbytes(); ++ m_unitbytes = (m_hunkbytes != 0) ? guess_unitbytes() : 0; ++ check_sizes(m_hunkcount); + m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes; + } + +@@ -2029,8 +2221,9 @@ + m_mapoffset = get_u64be(&rawheader[40]); + m_metaoffset = get_u64be(&rawheader[48]); + m_hunkbytes = get_u32be(&rawheader[56]); +- m_hunkcount = (m_logicalbytes + m_hunkbytes - 1) / m_hunkbytes; + m_unitbytes = get_u32be(&rawheader[60]); ++ check_sizes((m_hunkbytes != 0) ? ((m_logicalbytes / m_hunkbytes) + ((m_logicalbytes % m_hunkbytes) != 0)) : 0); ++ m_hunkcount = (m_logicalbytes + m_hunkbytes - 1) / m_hunkbytes; + m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes; + + // determine compression +@@ -2073,7 +2266,7 @@ + try + { + // first get a CRC-16 of the original rawmap +- util::crc16_t mapcrc = util::crc16_creator::simple(&m_rawmap[0], m_hunkcount * 12); ++ util::crc16_t mapcrc = util::crc16_creator::simple(m_rawmap.data(), m_hunkcount * 12); + + // create a buffer to hold the RLE data + std::vector compression_rle(m_hunkcount); +@@ -2173,6 +2366,10 @@ // for COMPRESSION_PARENT: parentbits // the overall size is clamped later with bitbuf.flush() int nbits_needed = (8*16) + (12 + std::max({lengthbits+16, selfbits, parentbits}))*m_hunkcount; @@ -2547,7 +2896,7 @@ std::vector compressed(nbits_needed / 8 + 1); bitstream_out bitbuf(&compressed[16], compressed.size() - 16); -@@ -2255,7 +2377,8 @@ +@@ -2255,7 +2452,8 @@ // write the map header uint32_t complen = bitbuf.flush(); @@ -2557,7 +2906,75 @@ put_u32be(&compressed[0], complen); put_u48be(&compressed[4], firstoffs); put_u16be(&compressed[10], mapcrc); -@@ -2966,6 +3089,13 @@ +@@ -2298,7 +2496,7 @@ + // if no offset, we haven't written it yet + if (m_mapoffset == 0) + { +- memset(&m_rawmap[0], 0xff, m_rawmap.size()); ++ memset(m_rawmap.data(), 0xff, m_rawmap.size()); + return; + } + +@@ -2314,13 +2512,21 @@ + uint8_t const lengthbits = rawbuf[12]; + uint8_t const selfbits = rawbuf[13]; + uint8_t const parentbits = rawbuf[14]; ++ // Discpress: the bit reader reads at most 32 bits at a time, as many as compress_v5_map writes: ++ // more (a damaged or crafted map) shifted by a negative count, which C++ leaves undefined ++ if (UNEXPECTED((lengthbits > 32) || (selfbits > 32) || (parentbits > 32))) ++ throw std::error_condition(error::DECOMPRESSION_ERROR); + + // now read the map ++ // Discpress: one that would run past the end of the file is the error reading it gives, without ++ // allocating it first (up to 4 GB, for a damaged or crafted header) ++ if (UNEXPECTED(past_end(m_mapoffset + 16, mapbytes))) ++ throw std::error_condition(std::errc::io_error); + std::vector compressed(mapbytes); +- ioerr = file_read(m_mapoffset + 16, &compressed[0], mapbytes); ++ ioerr = file_read(m_mapoffset + 16, compressed.data(), mapbytes); + if (UNEXPECTED(ioerr)) + throw ioerr; +- bitstream_in bitbuf(&compressed[0], compressed.size()); ++ bitstream_in bitbuf(compressed.data(), compressed.size()); + + // first decode the compression types + huffman_decoder<16, 8> decoder; +@@ -2409,7 +2615,7 @@ + } + + // verify the final CRC +- if (UNEXPECTED(util::crc16_creator::simple(&m_rawmap[0], m_hunkcount * 12) != mapcrc)) ++ if (UNEXPECTED(util::crc16_creator::simple(m_rawmap.data(), m_hunkcount * 12) != mapcrc)) + throw std::error_condition(error::DECOMPRESSION_ERROR); + } + +@@ -2636,14 +2842,23 @@ + } + + // read the map; v5+ compressed drives need to read and decompress their map +- m_rawmap.resize(m_hunkcount * m_mapentrybytes); ++ // Discpress: a map larger than 32 bits can count (a damaged or crafted header) is an error: 0.289 ++ // computed its size in 32 bits, and with the size wrapped, read and wrote past the end of the map ++ uint64_t const mapsize = uint64_t(m_hunkcount) * m_mapentrybytes; ++ if (UNEXPECTED(mapsize > std::numeric_limits::max())) ++ throw std::error_condition(error::INVALID_FILE); ++ // Discpress: and a stored map that would run past the end of the file is the error reading it ++ // gives, without allocating it first ++ if (!(m_version >= 5 && compressed()) && UNEXPECTED(past_end(m_mapoffset, mapsize))) ++ throw std::error_condition(std::errc::io_error); ++ m_rawmap.resize(mapsize); + if (m_version >= 5 && compressed()) + { + decompress_v5_map(); + } + else + { +- std::error_condition err = file_read(m_mapoffset, &m_rawmap[0], m_rawmap.size()); ++ std::error_condition err = file_read(m_mapoffset, m_rawmap.data(), m_rawmap.size()); + if (UNEXPECTED(err)) + throw err; + } +@@ -2966,6 +3181,13 @@ chd_file_compressor::~chd_file_compressor() { @@ -2571,7 +2988,7 @@ // free the work queues osd_work_queue_free(m_read_queue); osd_work_queue_free(m_work_queue); -@@ -3022,6 +3152,10 @@ +@@ -3022,6 +3244,10 @@ // reset write state m_write_hunk = 0; @@ -2582,7 +2999,7 @@ } /** -@@ -3123,6 +3257,11 @@ +@@ -3123,6 +3349,11 @@ } else { @@ -2594,7 +3011,7 @@ // otherwise, append it compressed and add to the self map hunk_write_compressed(item.m_hunknum, item.m_compression, item.m_compressed, item.m_complen, item.m_hash[0].m_crc16); m_total_out += item.m_complen; -@@ -3263,12 +3402,44 @@ +@@ -3263,12 +3494,44 @@ // TODO: data race if ((m_current_map.find(item.m_hash[0].m_crc16, item.m_hash[0].m_sha1) == hashmap::NOT_FOUND) && (m_parent_map.find(item.m_hash[0].m_crc16, item.m_hash[0].m_sha1) == hashmap::NOT_FOUND)) @@ -2640,7 +3057,7 @@ /** * @fn void *chd_file_compressor::async_read_static(void *param, int threadid) * -@@ -3345,6 +3516,16 @@ +@@ -3345,6 +3608,16 @@ assert(item.m_status == WS_READING); item.m_status = WS_QUEUED; item.m_hunknum = hunknum; @@ -2657,7 +3074,7 @@ item.m_osd = osd_work_item_queue(m_work_queue, m_walking_parent ? async_walk_parent_static : async_compress_hunk_static, &item, 0); } -@@ -3512,3 +3693,184 @@ +@@ -3512,3 +3785,184 @@ metadata_entry metaentry; return metadata_find(AV_METADATA_TAG, 0, metaentry); } @@ -2825,7 +3242,7 @@ + memcpy(dest, &ahead.data[0], m_hunkbytes); + return true; + case 3: // compressed bytes -+ m_decompressor[type]->decompress(&ahead.data[0], ahead.data.size(), reinterpret_cast(dest), m_hunkbytes); ++ m_decompressor[type]->decompress(ahead.data.data(), ahead.data.size(), reinterpret_cast(dest), m_hunkbytes); + return true; + default: + return false; @@ -2885,7 +3302,16 @@ private: struct metadata_entry; struct metadata_hash; -@@ -391,6 +411,7 @@ +@@ -380,6 +400,8 @@ + + // internal helpers + uint32_t guess_unitbytes(); ++ void check_sizes(uint64_t hunkcount) const; // Discpress ++ bool past_end(uint64_t offset, uint64_t length) const; // Discpress + void parse_v3_header(uint8_t *rawheader, util::sha1_t &parentsha1); + void parse_v4_header(uint8_t *rawheader, util::sha1_t &parentsha1); + void parse_v5_header(uint8_t *rawheader, util::sha1_t &parentsha1); +@@ -391,6 +413,7 @@ std::error_condition verify_proper_compression_append(uint32_t hunknum) const noexcept; void hunk_write_compressed(uint32_t hunknum, int8_t compression, const uint8_t *compressed, uint32_t complength, util::crc16_t crc16); void hunk_copy_from_self(uint32_t hunknum, uint32_t otherhunk); @@ -2893,7 +3319,7 @@ void hunk_copy_from_parent(uint32_t hunknum, uint64_t parentunit); std::error_condition metadata_find(chd_metadata_tag metatag, int32_t metaindex, metadata_entry &metaentry, bool resume = false) const noexcept; std::error_condition metadata_set_previous_next(uint64_t prevoffset, uint64_t nextoffset) noexcept; -@@ -433,6 +454,21 @@ +@@ -433,6 +456,21 @@ // caching std::vector m_cache; // single-hunk cache for partial reads/writes uint32_t m_cachehunk; // which hunk is in the cache? @@ -2915,7 +3341,7 @@ }; -@@ -450,11 +486,28 @@ +@@ -450,11 +488,28 @@ void compress_begin(); std::error_condition compress_continue(double &progress, double &ratio); @@ -2944,7 +3370,7 @@ // hash map for looking up values class hashmap { -@@ -524,6 +577,7 @@ +@@ -524,6 +579,7 @@ , m_complen(0) , m_compression(0) , m_codecs(nullptr) @@ -2952,7 +3378,7 @@ { } osd_work_item * m_osd; // OSD work item running on this block -@@ -537,6 +591,7 @@ +@@ -537,6 +593,7 @@ uint32_t m_complen; // compressed data length int8_t m_compression; // type of compression used chd_compressor_group *m_codecs; // codec instance @@ -2960,7 +3386,7 @@ std::vector m_hash; // array of hashes }; -@@ -549,6 +604,7 @@ +@@ -549,6 +606,7 @@ void async_read(); // current compression status @@ -3035,7 +3461,24 @@ return header_bytes + complen + m_subcode_compressor.compress(&m_buffer[frames * cdrom_file::MAX_SECTOR_DATA], frames * cdrom_file::MAX_SUBCODE_DATA, &dest[header_bytes + complen]); } -@@ -735,17 +751,33 @@ +@@ -426,8 +442,16 @@ + uint32_t ecc_bytes = (frames + 7) / 8; + uint32_t header_bytes = ecc_bytes + complen_bytes; + ++ // Discpress: a header longer than the data, or a base longer than what follows the header (a ++ // damaged or crafted hunk), is an error: 0.289 decoded past the end of the data, and the ++ // subcode's length wrapped ++ if (UNEXPECTED(complen < header_bytes)) ++ throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR); ++ + // extract compressed length of base + uint32_t complen_base = (complen_bytes > 2) ? get_u24be(&src[ecc_bytes]) : get_u16be(&src[ecc_bytes]); ++ if (UNEXPECTED(complen_base > complen - header_bytes)) ++ throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR); + + // reset and decode + m_base_decompressor.decompress(&src[header_bytes], complen_base, &m_buffer[0], frames * cdrom_file::MAX_SECTOR_DATA); +@@ -735,17 +759,33 @@ // compression for this hunk //------------------------------------------------- @@ -3071,7 +3514,7 @@ // if this is the best one, copy the data into the permanent buffer uint32_t compbytes = m_compressor[codecnum]->compress(src, m_hunkbytes, &m_compress_test[0]); #if CHDCODEC_VERIFY_COMPRESSION -@@ -772,7 +804,7 @@ +@@ -772,7 +812,7 @@ } printf(" codec%d=%d bytes \n", codecnum, compbytes); #endif @@ -3080,7 +3523,7 @@ { compression = codecnum; complen = compbytes; -@@ -787,6 +819,7 @@ +@@ -787,6 +827,7 @@ // if the best is none, copy it over if (compression == -1) memcpy(compressed, src, m_hunkbytes); @@ -3088,7 +3531,7 @@ return compression; } -@@ -852,12 +885,12 @@ +@@ -852,12 +893,12 @@ { // set the low bit of the size so we don't match next time *ptr |= 1; @@ -3103,7 +3546,7 @@ for (int scan = 0; scan < MAX_ZLIB_ALLOCS; scan++) if (codec->m_allocptr[scan] == nullptr) { -@@ -867,7 +900,7 @@ +@@ -867,7 +908,7 @@ // set the low bit of the size so we don't match next time *ptr = size | 1; @@ -3112,7 +3555,7 @@ } -@@ -881,7 +914,7 @@ +@@ -881,7 +922,7 @@ auto *codec = reinterpret_cast(opaque); // find the hunk @@ -3121,7 +3564,7 @@ for (int scan = 0; scan < MAX_ZLIB_ALLOCS; scan++) if (ptr == codec->m_allocptr[scan]) { -@@ -915,6 +948,13 @@ +@@ -915,6 +956,13 @@ throw std::bad_alloc(); else if (zerr != Z_OK) throw std::error_condition(chd_file::error::CODEC_ERROR); @@ -3135,7 +3578,7 @@ } -@@ -925,6 +965,8 @@ +@@ -925,6 +973,8 @@ chd_zlib_compressor::~chd_zlib_compressor() { deflateEnd(&m_deflater); @@ -3144,7 +3587,7 @@ } -@@ -934,12 +976,24 @@ +@@ -934,12 +984,24 @@ uint32_t chd_zlib_compressor::compress(const uint8_t *src, uint32_t srclen, uint8_t *dest) { @@ -3170,7 +3613,7 @@ m_deflater.total_out = 0; int zerr = deflateReset(&m_deflater); if (zerr != Z_OK) -@@ -949,7 +1003,7 @@ +@@ -949,7 +1011,7 @@ zerr = deflate(&m_deflater, Z_FINISH); // if we ended up with more data than we started with, return an error @@ -3179,7 +3622,7 @@ throw std::error_condition(chd_file::error::COMPRESSION_ERROR); // otherwise, return the length -@@ -1066,7 +1120,7 @@ +@@ -1066,7 +1128,7 @@ // do it ZSTD_inBuffer input{ src, srclen, 0 }; @@ -3188,7 +3631,7 @@ while (output.pos < output.size) { result = ZSTD_compressStream2(m_stream, &output, &input, ZSTD_e_end); -@@ -1197,12 +1251,12 @@ +@@ -1197,12 +1259,12 @@ { // set the low bit of the size so we don't match next time *ptr |= 1; @@ -3203,7 +3646,7 @@ for (int scan = 0; scan < MAX_LZMA_ALLOCS; scan++) if (codec->m_allocptr[scan] == nullptr) { -@@ -1212,7 +1266,7 @@ +@@ -1212,7 +1274,7 @@ // set the low bit of the size so we don't match next time *ptr = size | 1; @@ -3212,7 +3655,7 @@ } -@@ -1229,7 +1283,7 @@ +@@ -1229,7 +1291,7 @@ auto *const codec = static_cast(const_cast(p)); // find the hunk @@ -3221,7 +3664,7 @@ for (int scan = 0; scan < MAX_LZMA_ALLOCS; scan++) if (ptr == codec->m_allocptr[scan]) { -@@ -1284,8 +1338,8 @@ +@@ -1284,8 +1346,8 @@ if (res != SZ_OK) throw std::error_condition(chd_file::error::COMPRESSION_ERROR); @@ -3232,7 +3675,7 @@ res = LzmaEnc_MemEncode(encoder, dest, &complen, src, srclen, 0, nullptr, &m_allocator, &m_allocator); if (res != SZ_OK) throw std::error_condition(chd_file::error::COMPRESSION_ERROR); -@@ -1419,7 +1473,7 @@ +@@ -1419,7 +1481,7 @@ uint32_t chd_huffman_compressor::compress(const uint8_t *src, uint32_t srclen, uint8_t *dest) { uint32_t complen; @@ -3241,7 +3684,7 @@ throw std::error_condition(chd_file::error::COMPRESSION_ERROR); return complen; } -@@ -1474,6 +1528,7 @@ +@@ -1474,6 +1536,7 @@ m_encoder.set_num_channels(2); m_encoder.set_block_size(blocksize(hunkbytes)); m_encoder.set_strip_metadata(true); @@ -3249,7 +3692,7 @@ } -@@ -1483,32 +1538,47 @@ +@@ -1483,32 +1546,47 @@ uint32_t chd_flac_compressor::compress(const uint8_t *src, uint32_t srclen, uint8_t *dest) { @@ -3310,7 +3753,7 @@ } return complen + 1; } -@@ -1642,14 +1712,16 @@ +@@ -1642,14 +1720,16 @@ memcpy(&m_buffer[frames * cdrom_file::MAX_SECTOR_DATA + framenum * cdrom_file::MAX_SUBCODE_DATA], &src[framenum * cdrom_file::FRAME_SIZE + cdrom_file::MAX_SECTOR_DATA], cdrom_file::MAX_SUBCODE_DATA); } @@ -3330,6 +3773,18 @@ // deflate the subcode data m_deflater.next_in = const_cast(&m_buffer[frames * cdrom_file::MAX_SECTOR_DATA]); +@@ -1950,6 +2030,11 @@ + + void chd_avhuff_decompressor::decompress(const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) + { ++ // Discpress: a frame larger than the hunk (damaged or crafted data) is an error: 0.289 decoded it ++ // past the end of the hunk (its size, as raw_data_size has it: header, metadata, audio, video) ++ if ((complen >= 8) && (12 + uint64_t(src[0]) + 2 * uint64_t(src[1]) * get_u16be(&src[2]) + 2 * uint64_t(get_u16be(&src[4])) * get_u16be(&src[6]) > destlen)) ++ throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR); ++ + // decode the audio and video + avhuff_error averr = m_decoder.decode_data(src, complen, dest); + if (averr != AVHERR_NONE) --- a/src/lib/util/chdcodec.h +++ b/src/lib/util/chdcodec.h @@ -71,6 +71,13 @@ @@ -3708,6 +4163,42 @@ m_accum.m_raw = crc; } +--- a/src/lib/util/huffman.cpp ++++ b/src/lib/util/huffman.cpp +@@ -631,6 +631,10 @@ + uint32_t nextstart = (curstart + bithisto[codelen]) >> 1; + if (codelen != 1 && nextstart * 2 != (curstart + bithisto[codelen])) + return HUFFERR_INTERNAL_INCONSISTENCY; ++ // Discpress: more codes than one bit has room for (a damaged or crafted tree): 0.289 gave codes ++ // more bits than their length, and build_lookup_table wrote past the end of its table ++ if (codelen == 1 && (curstart + bithisto[codelen]) > 2) ++ return HUFFERR_INTERNAL_INCONSISTENCY; + bithisto[codelen] = curstart; + curstart = nextstart; + } +@@ -654,6 +658,7 @@ + void huffman_context_base::build_lookup_table() + { + // iterate over all codes ++ uint32_t covered = 0; + for (int curcode = 0; curcode < m_numcodes; curcode++) + { + // process all nodes which have non-zero bits +@@ -669,8 +674,14 @@ + lookup_value *destend = &m_lookup[((node.m_bits + 1) << shift) - 1]; + while (dest <= destend) + *dest++ = value; ++ covered = std::max(covered, (node.m_bits + 1) << shift); + } + } ++ ++ // Discpress: the entries no code covers decode as code 0 of no bits. Canonical codes cover the table ++ // from 0 up, all of it for a complete tree (what an encoder writes); 0.289 left the rest as it was, ++ // whatever was in that memory, and a damaged tree's data read bit counts from it ++ std::fill(m_lookup + covered, m_lookup + (uint32_t(1) << m_maxbits), lookup_value(0)); + } + + --- a/src/lib/util/unicode.cpp +++ b/src/lib/util/unicode.cpp @@ -16,71 +16,12 @@ diff --git a/engine/mame/3rdparty/flac/src/libFLAC/stream_encoder.c b/engine/mame/3rdparty/flac/src/libFLAC/stream_encoder.c index c1c03e4..c518289 100644 --- a/engine/mame/3rdparty/flac/src/libFLAC/stream_encoder.c +++ b/engine/mame/3rdparty/flac/src/libFLAC/stream_encoder.c @@ -4126,7 +4126,8 @@ uint32_t find_best_partition_order_( residual, #endif abs_residual_partition_sums+sum, - raw_bits_per_partition+sum, + /* Discpress: null without escape coding, which C doesn't let an offset be added to */ + raw_bits_per_partition != NULL ? raw_bits_per_partition+sum : NULL, residual_samples, predictor_order, rice_parameter_limit, diff --git a/engine/mame/src/lib/util/avhuff.cpp b/engine/mame/src/lib/util/avhuff.cpp index 410f8ae..bf27e58 100644 --- a/engine/mame/src/lib/util/avhuff.cpp +++ b/engine/mame/src/lib/util/avhuff.cpp @@ -67,6 +67,7 @@ #include #include #include +#include #include @@ -723,6 +724,11 @@ avhuff_error avhuff_decoder::decode_data(const uint8_t *source, uint32_t complen uint32_t width = get_u16be(&source[4]); uint32_t height = get_u16be(&source[6]); + // Discpress: no more channels than there are places for (the encoder takes up to 16): 0.289 went past + // the end of audiostart and of m_config.audio + if (channels > std::size(m_config.audio)) + return AVHERR_INVALID_DATA; + // validate that the sizes make sense if (complength < 10 + 2 * channels) return AVHERR_INVALID_DATA; @@ -932,6 +938,10 @@ avhuff_error avhuff_decoder::decode_audio(int channels, int samples, const uint8 // if no huffman length, just copy the data if (treesize == 0) { + // Discpress: as much as the samples take (as stored): 0.289 read past the channel's + // data, and past the end of the hunk's + if (size < 2 * samples) + return AVHERR_INVALID_DATA; const uint8_t *cursource = source; for (int sampnum = 0; sampnum < samples; sampnum++) { diff --git a/engine/mame/src/lib/util/bitstream.h b/engine/mame/src/lib/util/bitstream.h index d96c2e5..b9a9d70 100644 --- a/engine/mame/src/lib/util/bitstream.h +++ b/engine/mame/src/lib/util/bitstream.h @@ -146,7 +146,8 @@ inline uint32_t bitstream_in::peek(int numbits) inline void bitstream_in::remove(int numbits) { - m_buffer <<= numbits; + // (all 32 bits, which a crafted v5 map's field width can ask for: the buffer is empty, not shifted by 32) + m_buffer = (numbits < 32) ? (m_buffer << numbits) : 0; m_bits -= numbits; } @@ -226,6 +227,11 @@ inline bitstream_out::bitstream_out(void *dest, uint32_t destlength) inline void bitstream_out::write(uint32_t newbits, int numbits) { + // Discpress: nothing to write, as for a CHD map with no copies of hunks (0.289 went on to shift + // the value by 32 bits, which C++ leaves undefined, and then returned without using it) + if (numbits <= 0) + return; + newbits <<= 32 - numbits; // flush the buffer if we're going to overflow it diff --git a/engine/mame/src/lib/util/cdrom.cpp b/engine/mame/src/lib/util/cdrom.cpp index 96eff6c..13d0bbf 100644 --- a/engine/mame/src/lib/util/cdrom.cpp +++ b/engine/mame/src/lib/util/cdrom.cpp @@ -921,6 +921,13 @@ const char *cdrom_file::get_subtype_string(uint32_t subtype) * @return A std::error_condition. */ +// Discpress: chd.cpp's formats for the track metadata, with a width for each string that fills its 16-byte +// buffer at most (the longest a CD has is 15 characters, VMODE2_FORM_MIX); a longer one, in damaged or +// crafted metadata, doesn't match. 0.289 read them with %s, which wrote past the end of the buffer. +static const char CDROM_TRACK_METADATA_FORMAT_READ[] = "TRACK:%d TYPE:%15s SUBTYPE:%15s FRAMES:%d"; +static const char CDROM_TRACK_METADATA2_FORMAT_READ[] = "TRACK:%d TYPE:%15s SUBTYPE:%15s FRAMES:%d PREGAP:%d PGTYPE:%15s PGSUB:%15s POSTGAP:%d"; +static const char GDROM_TRACK_METADATA_FORMAT_READ[] = "TRACK:%d TYPE:%15s SUBTYPE:%15s FRAMES:%d PAD:%d PREGAP:%d PGTYPE:%15s PGSUB:%15s POSTGAP:%d"; + std::error_condition cdrom_file::parse_metadata(chd_file *chd, toc &toc) { std::string metadata; @@ -948,12 +955,12 @@ std::error_condition cdrom_file::parse_metadata(chd_file *chd, toc &toc) // fetch the metadata for this track if (!chd->read_metadata(CDROM_TRACK_METADATA_TAG, toc.numtrks, metadata)) { - if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA_FORMAT, &tracknum, type, subtype, &frames) != 4) + if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA_FORMAT_READ, &tracknum, type, subtype, &frames) != 4) return chd_file::error::INVALID_DATA; } else if (!chd->read_metadata(CDROM_TRACK_METADATA2_TAG, toc.numtrks, metadata)) { - if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA2_FORMAT, &tracknum, type, subtype, &frames, &pregap, pgtype, pgsub, &postgap) != 8) + if (sscanf(metadata.c_str(), CDROM_TRACK_METADATA2_FORMAT_READ, &tracknum, type, subtype, &frames, &pregap, pgtype, pgsub, &postgap) != 8) return chd_file::error::INVALID_DATA; } else @@ -968,7 +975,7 @@ std::error_condition cdrom_file::parse_metadata(chd_file *chd, toc &toc) if (err) break; - if (sscanf(metadata.c_str(), GDROM_TRACK_METADATA_FORMAT, &tracknum, type, subtype, &frames, &padframes, &pregap, pgtype, pgsub, &postgap) != 9) + if (sscanf(metadata.c_str(), GDROM_TRACK_METADATA_FORMAT_READ, &tracknum, type, subtype, &frames, &padframes, &pregap, pgtype, pgsub, &postgap) != 9) return chd_file::error::INVALID_DATA; toc.flags |= CD_FLAG_GDROM; @@ -1019,7 +1026,14 @@ std::error_condition cdrom_file::parse_metadata(chd_file *chd, toc &toc) /* if we got any tracks this way, we're done */ if (toc.numtrks > 0) + { + // Discpress: with each track once: a track number that repeats or skips one (damaged or crafted + // metadata) left a track of nothing, of 0-byte sectors, which extractcd divided by + for (uint32_t i = 0; i < toc.numtrks; i++) + if (toc.tracks[i].datasize == 0) + return chd_file::error::INVALID_DATA; return std::error_condition(); + } osd_printf_info("toc.numtrks = %u?!\n", toc.numtrks); @@ -1028,6 +1042,9 @@ std::error_condition cdrom_file::parse_metadata(chd_file *chd, toc &toc) err = chd->read_metadata(CDROM_OLD_METADATA_TAG, 0, oldmetadata); if (err) return err; + // Discpress: as long as the table it holds (0.289 read past the end of shorter metadata) + if (oldmetadata.size() < 4 + MAX_TRACKS * 6 * 4) + return chd_file::error::INVALID_DATA; /* reconstruct the TOC from it */ auto *mrp = reinterpret_cast(&oldmetadata[0]); @@ -1068,6 +1085,14 @@ std::error_condition cdrom_file::parse_metadata(chd_file *chd, toc &toc) } } + // Discpress: no more tracks than the table holds, with sectors a CD has: anything else (damaged or + // crafted metadata) made 0.289 read past the end of the table, and of a sector's buffer + if (toc.numtrks > MAX_TRACKS) + return chd_file::error::INVALID_DATA; + for (uint32_t i = 0; i < toc.numtrks; i++) + if (toc.tracks[i].datasize == 0 || toc.tracks[i].datasize > MAX_SECTOR_DATA || toc.tracks[i].subsize > MAX_SUBCODE_DATA) + return chd_file::error::INVALID_DATA; + return std::error_condition(); } diff --git a/engine/mame/src/lib/util/chd.cpp b/engine/mame/src/lib/util/chd.cpp index f9c1c5f..db3abee 100644 --- a/engine/mame/src/lib/util/chd.cpp +++ b/engine/mame/src/lib/util/chd.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -989,6 +990,9 @@ std::error_condition chd_file::codec_process_hunk(uint32_t hunknum) // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end if (UNEXPECTED(blocklen > m_compressed.size())) return std::error_condition(error::INVALID_DATA); + // Discpress: and a hunk compressed in a CHD that says it isn't: 0.289 called a codec it doesn't have + if (UNEXPECTED(!m_decompressor[0])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; @@ -1013,6 +1017,9 @@ std::error_condition chd_file::codec_process_hunk(uint32_t hunknum) case V34_MAP_ENTRY_TYPE_PARENT_HUNK: if (UNEXPECTED(m_parent_missing)) return std::error_condition(error::REQUIRES_PARENT); + // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer + if (UNEXPECTED(!m_parent)) + return std::error_condition(error::INVALID_DATA); return m_parent->codec_process_hunk(blockoffs); } } @@ -1038,6 +1045,9 @@ std::error_condition chd_file::codec_process_hunk(uint32_t hunknum) // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end if (UNEXPECTED(blocklen > m_compressed.size())) return std::error_condition(error::INVALID_DATA); + // Discpress: and a codec slot the header leaves empty: 0.289 called a codec it doesn't have + if (UNEXPECTED(!m_decompressor[rawmap[0]])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; @@ -1062,6 +1072,12 @@ std::error_condition chd_file::codec_process_hunk(uint32_t hunknum) case COMPRESSION_PARENT: if (UNEXPECTED(m_parent_missing)) return std::error_condition(error::REQUIRES_PARENT); + // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer + if (UNEXPECTED(!m_parent)) + return std::error_condition(error::INVALID_DATA); + // Discpress: a parent whose units are larger than its hunks has no hunk for a unit (0.289 divided by zero) + if (UNEXPECTED(m_parent->unit_bytes() > m_parent->hunk_bytes())) + return std::error_condition(error::INVALID_DATA); return m_parent->codec_process_hunk(blockoffs / (m_parent->hunk_bytes() / m_parent->unit_bytes())); } break; @@ -1135,6 +1151,9 @@ std::error_condition chd_file::read_hunk(uint32_t hunknum, void *buffer) // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end if (UNEXPECTED(blocklen > m_compressed.size())) return std::error_condition(error::INVALID_DATA); + // Discpress: and a hunk compressed in a CHD that says it isn't: 0.289 called a codec it doesn't have + if (UNEXPECTED(!m_decompressor[0])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; @@ -1175,6 +1194,9 @@ std::error_condition chd_file::read_hunk(uint32_t hunknum, void *buffer) case V34_MAP_ENTRY_TYPE_PARENT_HUNK: if (UNEXPECTED(m_parent_missing)) return std::error_condition(error::REQUIRES_PARENT); + // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer + if (UNEXPECTED(!m_parent)) + return std::error_condition(error::INVALID_DATA); return m_parent->read_hunk(blockoffs, dest); } } @@ -1220,6 +1242,9 @@ std::error_condition chd_file::read_hunk(uint32_t hunknum, void *buffer) // Discpress: more than the buffer holds (a damaged or crafted map); 0.289 read past its end if (UNEXPECTED(blocklen > m_compressed.size())) return std::error_condition(error::INVALID_DATA); + // Discpress: and a codec slot the header leaves empty: 0.289 called a codec it doesn't have + if (UNEXPECTED(!m_decompressor[rawmap[0]])) + return std::error_condition(error::INVALID_DATA); std::error_condition err = file_read(blockoffs, &m_compressed[0], blocklen); if (UNEXPECTED(err)) return err; @@ -1262,6 +1287,9 @@ std::error_condition chd_file::read_hunk(uint32_t hunknum, void *buffer) case COMPRESSION_PARENT: if (UNEXPECTED(m_parent_missing)) return std::error_condition(error::REQUIRES_PARENT); + // Discpress: and a CHD without a parent whose map says so (damaged or crafted): 0.289 read through a null pointer + if (UNEXPECTED(!m_parent)) + return std::error_condition(error::INVALID_DATA); return m_parent->read_bytes(blockoffs * m_parent->unit_bytes(), dest, m_hunkbytes); } } @@ -1425,6 +1453,11 @@ std::error_condition chd_file::write_units(uint64_t unitnum, const void *buffer, std::error_condition chd_file::read_bytes(uint64_t offset, void *buffer, uint32_t bytes) { + // Discpress: nothing to read, as for a CD track without data (a damaged or crafted CHD's): 0.289's + // last hunk wrapped around at offset 0, and it read a whole hunk into the buffer + if (bytes == 0) + return std::error_condition(); + // iterate over hunks uint32_t const first_hunk = offset / m_hunkbytes; uint32_t const last_hunk = (offset + bytes - 1) / m_hunkbytes; @@ -1478,6 +1511,10 @@ std::error_condition chd_file::read_bytes(uint64_t offset, void *buffer, uint32_ std::error_condition chd_file::write_bytes(uint64_t offset, const void *buffer, uint32_t bytes) { + // Discpress: nothing to write (as read_bytes) + if (bytes == 0) + return std::error_condition(); + // iterate over hunks uint32_t const first_hunk = offset / m_hunkbytes; uint32_t const last_hunk = (offset + bytes - 1) / m_hunkbytes; @@ -1571,7 +1608,7 @@ std::error_condition chd_file::read_metadata(chd_metadata_tag searchtag, uint32_ // read the metadata try { output.resize(metaentry.length); } catch (std::bad_alloc const &) { return std::errc::not_enough_memory; } - return file_read(metaentry.offset + METADATA_HEADER_SIZE, &output[0], metaentry.length); + return file_read(metaentry.offset + METADATA_HEADER_SIZE, output.data(), metaentry.length); // Discpress: data(), as metadata can be empty } /** @@ -1633,7 +1670,7 @@ std::error_condition chd_file::read_metadata(chd_metadata_tag searchtag, uint32_ // read the metadata try { output.resize(metaentry.length); } catch (std::bad_alloc const &) { return std::errc::not_enough_memory; } - err = file_read(metaentry.offset + METADATA_HEADER_SIZE, &output[0], metaentry.length); + err = file_read(metaentry.offset + METADATA_HEADER_SIZE, output.data(), metaentry.length); // Discpress: data(), as metadata can be empty if (UNEXPECTED(err)) return err; resulttag = metaentry.metatag; @@ -1796,12 +1833,12 @@ std::error_condition chd_file::clone_all_metadata(chd_file &source) // read the metadata item try { filedata.resize(metaentry.length); } catch (std::bad_alloc const &) { return std::errc::not_enough_memory; } - err = source.file_read(metaentry.offset + METADATA_HEADER_SIZE, &filedata[0], metaentry.length); + err = source.file_read(metaentry.offset + METADATA_HEADER_SIZE, filedata.data(), metaentry.length); // Discpress: data(), as metadata can be empty if (UNEXPECTED(err)) return err; // write it to the destination - err = write_metadata(metaentry.metatag, (uint32_t)-1, &filedata[0], metaentry.length, metaentry.flags); + err = write_metadata(metaentry.metatag, (uint32_t)-1, filedata.data(), metaentry.length, metaentry.flags); if (UNEXPECTED(err)) return err; } @@ -1843,14 +1880,14 @@ util::sha1_t chd_file::compute_overall_sha1(util::sha1_t rawsha1) // allocate memory and read the data filedata.resize(metaentry.length); - err = file_read(metaentry.offset + METADATA_HEADER_SIZE, &filedata[0], metaentry.length); + err = file_read(metaentry.offset + METADATA_HEADER_SIZE, filedata.data(), metaentry.length); // Discpress: data(), as metadata can be empty if (UNEXPECTED(err)) throw err; // create an entry for this metadata and add it metadata_hash hashentry; put_u32be(hashentry.tag, metaentry.metatag); - hashentry.sha1 = util::sha1_creator::simple(&filedata[0], metaentry.length); + hashentry.sha1 = util::sha1_creator::simple(filedata.data(), metaentry.length); hasharray.push_back(hashentry); } if (err != error::METADATA_NOT_FOUND) @@ -1997,6 +2034,41 @@ uint32_t chd_file::guess_unitbytes() return m_hunkbytes; } +/** + * @fn bool chd_file::past_end(uint64_t offset, uint64_t length) const + * @brief past_end - Discpress: whether length bytes from offset would run past the end of the + * file (false when its length is unknown, so that the read reports it) + */ +bool chd_file::past_end(uint64_t offset, uint64_t length) const +{ + uint64_t filelength; + if (!m_file || m_file->length(filelength)) + return false; + return (offset > filelength) || (length > filelength - offset); +} + +/** + * @fn void chd_file::check_sizes(uint64_t hunkcount) const + * + * @brief ------------------------------------------------- + * check_sizes - Discpress: a header whose sizes no data can have (a damaged or + * crafted file) is an error. 0.289 divided by a hunk or unit size of 0, and with a + * logical size of more hunks than a map can count, it made a map too small for them + * -------------------------------------------------. + * + * @exception CHDERR_INVALID_FILE Thrown when the sizes can't be right. + * + * @param hunkcount The number of hunks the header gives or implies. + */ + +void chd_file::check_sizes(uint64_t hunkcount) const +{ + if (UNEXPECTED(m_hunkbytes == 0 || m_unitbytes == 0)) + throw std::error_condition(error::INVALID_FILE); + if (UNEXPECTED(hunkcount > std::numeric_limits::max())) + throw std::error_condition(error::INVALID_FILE); +} + /** * @fn void chd_file::parse_v3_header(uint8_t *rawheader, sha1_t &parentsha1) * @@ -2056,7 +2128,8 @@ void chd_file::parse_v3_header(uint8_t *rawheader, util::sha1_t &parentsha1) parentsha1 = be_read_sha1(&rawheader[m_parentsha1_offset]); // guess at the units based on snooping the metadata - m_unitbytes = guess_unitbytes(); + m_unitbytes = (m_hunkbytes != 0) ? guess_unitbytes() : 0; + check_sizes(m_hunkcount); m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes; } @@ -2119,7 +2192,8 @@ void chd_file::parse_v4_header(uint8_t *rawheader, util::sha1_t &parentsha1) parentsha1 = be_read_sha1(&rawheader[m_parentsha1_offset]); // guess at the units based on snooping the metadata - m_unitbytes = guess_unitbytes(); + m_unitbytes = (m_hunkbytes != 0) ? guess_unitbytes() : 0; + check_sizes(m_hunkcount); m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes; } @@ -2147,8 +2221,9 @@ void chd_file::parse_v5_header(uint8_t *rawheader, util::sha1_t &parentsha1) m_mapoffset = get_u64be(&rawheader[40]); m_metaoffset = get_u64be(&rawheader[48]); m_hunkbytes = get_u32be(&rawheader[56]); - m_hunkcount = (m_logicalbytes + m_hunkbytes - 1) / m_hunkbytes; m_unitbytes = get_u32be(&rawheader[60]); + check_sizes((m_hunkbytes != 0) ? ((m_logicalbytes / m_hunkbytes) + ((m_logicalbytes % m_hunkbytes) != 0)) : 0); + m_hunkcount = (m_logicalbytes + m_hunkbytes - 1) / m_hunkbytes; m_unitcount = (m_logicalbytes + m_unitbytes - 1) / m_unitbytes; // determine compression @@ -2191,7 +2266,7 @@ std::error_condition chd_file::compress_v5_map() try { // first get a CRC-16 of the original rawmap - util::crc16_t mapcrc = util::crc16_creator::simple(&m_rawmap[0], m_hunkcount * 12); + util::crc16_t mapcrc = util::crc16_creator::simple(m_rawmap.data(), m_hunkcount * 12); // create a buffer to hold the RLE data std::vector compression_rle(m_hunkcount); @@ -2421,7 +2496,7 @@ void chd_file::decompress_v5_map() // if no offset, we haven't written it yet if (m_mapoffset == 0) { - memset(&m_rawmap[0], 0xff, m_rawmap.size()); + memset(m_rawmap.data(), 0xff, m_rawmap.size()); return; } @@ -2437,13 +2512,21 @@ void chd_file::decompress_v5_map() uint8_t const lengthbits = rawbuf[12]; uint8_t const selfbits = rawbuf[13]; uint8_t const parentbits = rawbuf[14]; + // Discpress: the bit reader reads at most 32 bits at a time, as many as compress_v5_map writes: + // more (a damaged or crafted map) shifted by a negative count, which C++ leaves undefined + if (UNEXPECTED((lengthbits > 32) || (selfbits > 32) || (parentbits > 32))) + throw std::error_condition(error::DECOMPRESSION_ERROR); // now read the map + // Discpress: one that would run past the end of the file is the error reading it gives, without + // allocating it first (up to 4 GB, for a damaged or crafted header) + if (UNEXPECTED(past_end(m_mapoffset + 16, mapbytes))) + throw std::error_condition(std::errc::io_error); std::vector compressed(mapbytes); - ioerr = file_read(m_mapoffset + 16, &compressed[0], mapbytes); + ioerr = file_read(m_mapoffset + 16, compressed.data(), mapbytes); if (UNEXPECTED(ioerr)) throw ioerr; - bitstream_in bitbuf(&compressed[0], compressed.size()); + bitstream_in bitbuf(compressed.data(), compressed.size()); // first decode the compression types huffman_decoder<16, 8> decoder; @@ -2532,7 +2615,7 @@ void chd_file::decompress_v5_map() } // verify the final CRC - if (UNEXPECTED(util::crc16_creator::simple(&m_rawmap[0], m_hunkcount * 12) != mapcrc)) + if (UNEXPECTED(util::crc16_creator::simple(m_rawmap.data(), m_hunkcount * 12) != mapcrc)) throw std::error_condition(error::DECOMPRESSION_ERROR); } @@ -2759,14 +2842,23 @@ void chd_file::create_open_common() } // read the map; v5+ compressed drives need to read and decompress their map - m_rawmap.resize(m_hunkcount * m_mapentrybytes); + // Discpress: a map larger than 32 bits can count (a damaged or crafted header) is an error: 0.289 + // computed its size in 32 bits, and with the size wrapped, read and wrote past the end of the map + uint64_t const mapsize = uint64_t(m_hunkcount) * m_mapentrybytes; + if (UNEXPECTED(mapsize > std::numeric_limits::max())) + throw std::error_condition(error::INVALID_FILE); + // Discpress: and a stored map that would run past the end of the file is the error reading it + // gives, without allocating it first + if (!(m_version >= 5 && compressed()) && UNEXPECTED(past_end(m_mapoffset, mapsize))) + throw std::error_condition(std::errc::io_error); + m_rawmap.resize(mapsize); if (m_version >= 5 && compressed()) { decompress_v5_map(); } else { - std::error_condition err = file_read(m_mapoffset, &m_rawmap[0], m_rawmap.size()); + std::error_condition err = file_read(m_mapoffset, m_rawmap.data(), m_rawmap.size()); if (UNEXPECTED(err)) throw err; } @@ -3857,7 +3949,7 @@ bool chd_file::wasm_read_cached(uint32_t hunknum, uint8_t type, void *dest) memcpy(dest, &ahead.data[0], m_hunkbytes); return true; case 3: // compressed bytes - m_decompressor[type]->decompress(&ahead.data[0], ahead.data.size(), reinterpret_cast(dest), m_hunkbytes); + m_decompressor[type]->decompress(ahead.data.data(), ahead.data.size(), reinterpret_cast(dest), m_hunkbytes); return true; default: return false; diff --git a/engine/mame/src/lib/util/chd.h b/engine/mame/src/lib/util/chd.h index 8b6e993..c10ee8d 100644 --- a/engine/mame/src/lib/util/chd.h +++ b/engine/mame/src/lib/util/chd.h @@ -400,6 +400,8 @@ class chd_file // internal helpers uint32_t guess_unitbytes(); + void check_sizes(uint64_t hunkcount) const; // Discpress + bool past_end(uint64_t offset, uint64_t length) const; // Discpress void parse_v3_header(uint8_t *rawheader, util::sha1_t &parentsha1); void parse_v4_header(uint8_t *rawheader, util::sha1_t &parentsha1); void parse_v5_header(uint8_t *rawheader, util::sha1_t &parentsha1); diff --git a/engine/mame/src/lib/util/chdcodec.cpp b/engine/mame/src/lib/util/chdcodec.cpp index 6d58e99..0048b48 100644 --- a/engine/mame/src/lib/util/chdcodec.cpp +++ b/engine/mame/src/lib/util/chdcodec.cpp @@ -442,8 +442,16 @@ class chd_cd_decompressor : public chd_decompressor uint32_t ecc_bytes = (frames + 7) / 8; uint32_t header_bytes = ecc_bytes + complen_bytes; + // Discpress: a header longer than the data, or a base longer than what follows the header (a + // damaged or crafted hunk), is an error: 0.289 decoded past the end of the data, and the + // subcode's length wrapped + if (UNEXPECTED(complen < header_bytes)) + throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR); + // extract compressed length of base uint32_t complen_base = (complen_bytes > 2) ? get_u24be(&src[ecc_bytes]) : get_u16be(&src[ecc_bytes]); + if (UNEXPECTED(complen_base > complen - header_bytes)) + throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR); // reset and decode m_base_decompressor.decompress(&src[header_bytes], complen_base, &m_buffer[0], frames * cdrom_file::MAX_SECTOR_DATA); @@ -2022,6 +2030,11 @@ void chd_avhuff_decompressor::process(const uint8_t *src, uint32_t complen) void chd_avhuff_decompressor::decompress(const uint8_t *src, uint32_t complen, uint8_t *dest, uint32_t destlen) { + // Discpress: a frame larger than the hunk (damaged or crafted data) is an error: 0.289 decoded it + // past the end of the hunk (its size, as raw_data_size has it: header, metadata, audio, video) + if ((complen >= 8) && (12 + uint64_t(src[0]) + 2 * uint64_t(src[1]) * get_u16be(&src[2]) + 2 * uint64_t(get_u16be(&src[4])) * get_u16be(&src[6]) > destlen)) + throw std::error_condition(chd_file::error::DECOMPRESSION_ERROR); + // decode the audio and video avhuff_error averr = m_decoder.decode_data(src, complen, dest); if (averr != AVHERR_NONE) diff --git a/engine/mame/src/lib/util/huffman.cpp b/engine/mame/src/lib/util/huffman.cpp index a1f37b6..0638810 100644 --- a/engine/mame/src/lib/util/huffman.cpp +++ b/engine/mame/src/lib/util/huffman.cpp @@ -631,6 +631,10 @@ huffman_error huffman_context_base::assign_canonical_codes() uint32_t nextstart = (curstart + bithisto[codelen]) >> 1; if (codelen != 1 && nextstart * 2 != (curstart + bithisto[codelen])) return HUFFERR_INTERNAL_INCONSISTENCY; + // Discpress: more codes than one bit has room for (a damaged or crafted tree): 0.289 gave codes + // more bits than their length, and build_lookup_table wrote past the end of its table + if (codelen == 1 && (curstart + bithisto[codelen]) > 2) + return HUFFERR_INTERNAL_INCONSISTENCY; bithisto[codelen] = curstart; curstart = nextstart; } @@ -654,6 +658,7 @@ huffman_error huffman_context_base::assign_canonical_codes() void huffman_context_base::build_lookup_table() { // iterate over all codes + uint32_t covered = 0; for (int curcode = 0; curcode < m_numcodes; curcode++) { // process all nodes which have non-zero bits @@ -669,8 +674,14 @@ void huffman_context_base::build_lookup_table() lookup_value *destend = &m_lookup[((node.m_bits + 1) << shift) - 1]; while (dest <= destend) *dest++ = value; + covered = std::max(covered, (node.m_bits + 1) << shift); } } + + // Discpress: the entries no code covers decode as code 0 of no bits. Canonical codes cover the table + // from 0 up, all of it for a complete tree (what an encoder writes); 0.289 left the rest as it was, + // whatever was in that memory, and a damaged tree's data read bit counts from it + std::fill(m_lookup + covered, m_lookup + (uint32_t(1) << m_maxbits), lookup_value(0)); } diff --git a/scripts/fuzz.sh b/scripts/fuzz.sh new file mode 100755 index 0000000..ab6bece --- /dev/null +++ b/scripts/fuzz.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# Fuzzes the engine with libFuzzer, under AddressSanitizer and UndefinedBehaviorSanitizer +# (tests/fuzz/README.md: what each fuzzer feeds the engine, and how to read a finding). +# +# scripts/fuzz.sh build the fuzzers (build/fuzz/) and their seeds (build/fuzz/seeds/) +# scripts/fuzz.sh run [seconds] fuzz (default 300 s each); new inputs go to build/fuzz/corpus/, +# findings to build/fuzz/findings/; with "all", the four run at once +# scripts/fuzz.sh check every seed, past finding (tests/fuzz/regressions) and corpus input +# once: fails on any finding (CI) +# scripts/fuzz.sh repro one input, with the engine's output +# scripts/fuzz.sh prune the corpus without the inputs that add no coverage +# +# The fuzzers are chd, desc, codec and ecm. The engine is built with __EMSCRIPTEN__ defined, so that its +# browser-only code is in, with tests/fuzz/host.cpp doing what the page's JavaScript does for it. +# Needs clang with its libFuzzer runtime (Debian and Ubuntu: clang and libclang-rt--dev) and make. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)}" +CC="${CC:-clang}" +CXX="${CXX:-clang++}" +OUT="${FUZZ_OUT:-$ROOT/build/fuzz}" +FUZZERS=(chd desc codec ecm) + +# (without clang's check of calls through a pointer to another function type, which gcc's UBSan, as in +# scripts/build-sanitized.sh, doesn't have: the LZMA SDK calls its match finder's functions through +# pointers that take void *, which every ABI makes the same call, WebAssembly's call_indirect included) +SAN="-fsanitize=address,undefined -fno-sanitize=function -fno-sanitize-recover=undefined -fno-omit-frame-pointer" +# -Dmain=...: chdman's main gives way to libFuzzer's (host.cpp runs commands with chdman_begin) +OPT="-O1 -g $SAN -fsanitize=fuzzer-no-link -D__EMSCRIPTEN__ -Dmain=chdman_native_main -I$ROOT/tests/fuzz/shim" +# the native build of every engine source, wasm_helper.cpp and ecm.cpp too +MAKEVARS=(T=native "OPT=$OPT" "CC=$CC" "CXX=$CXX" "O=$OUT/obj" 'CPP_SRC=$(UTIL_SRC) $(OSD_SRC) $(TOOL_SRC)') + +# failing to allocate more than 1 GB at once (as a phone's browser does) must be an error, not a crash; +# leaks that unmodified chdman 0.289 has too are suppressed, as for the sanitized build +export ASAN_OPTIONS="${ASAN_OPTIONS:-allocator_may_return_null=1:max_allocation_size_mb=1024:detect_stack_use_after_return=1:strict_string_checks=1}" +export LSAN_OPTIONS="${LSAN_OPTIONS:-suppressions=$ROOT/tests/support/lsan.supp}" +export UBSAN_OPTIONS="${UBSAN_OPTIONS:-print_stacktrace=1}" +FUZZ_OPTS=(-timeout=25 -rss_limit_mb=2560 -malloc_limit_mb=2048) + +build() { + mkdir -p "$OUT" + # objects built with other flags are built again (the Makefile only tracks sources and headers) + if [ "$(cat "$OUT/obj/flags" 2>/dev/null)" != "$OPT $CC $CXX" ]; then rm -rf "$OUT/obj"; mkdir -p "$OUT/obj"; echo "$OPT $CC $CXX" > "$OUT/obj/flags"; fi + make -s -C "$ROOT/wasm" -j"$JOBS" "${MAKEVARS[@]}" objs + local cxxflags + # (the engine's flags, but for the main rename: seeds.cpp has its own) + cxxflags="$(make -s -C "$ROOT/wasm" "${MAKEVARS[@]}" --eval 'print-cxxflags: ; @echo $(CXXFLAGS)' print-cxxflags)" + cxxflags="${cxxflags//-Dmain=chdman_native_main/}" + local objs + mapfile -t objs < "$OUT/obj/objs.rsp" + local f + for f in host "${FUZZERS[@]/#/fuzz_}" seeds; do + # shellcheck disable=SC2086 + $CXX $cxxflags -c "$ROOT/tests/fuzz/$f.cpp" -o "$OUT/obj/$f.o" & + done + wait + for f in "${FUZZERS[@]}"; do + # shellcheck disable=SC2086 + $CXX $SAN -fsanitize=fuzzer "$OUT/obj/fuzz_$f.o" "$OUT/obj/host.o" "${objs[@]}" -o "$OUT/$f" -lpthread & + done + # shellcheck disable=SC2086 + $CXX $SAN "$OUT/obj/seeds.o" "$OUT/obj/host.o" "${objs[@]}" -o "$OUT/seeds-tool" -lpthread & + wait + rm -rf "$OUT/seeds" + ASAN_OPTIONS="$ASAN_OPTIONS:detect_leaks=0" "$OUT/seeds-tool" "$OUT/seeds" > /dev/null 2> "$OUT/seeds.log" || { cat "$OUT/seeds.log" >&2; exit 1; } + echo "fuzzers: ${FUZZERS[*]/#/$OUT/}; seeds: $(find "$OUT/seeds" -type f | wc -l) in $OUT/seeds" +} + +names() { + if [ "${1:-}" = all ]; then echo "${FUZZERS[@]}"; return; fi + local f + for f in "${FUZZERS[@]}"; do [ "$f" = "${1:-}" ] && { echo "$f"; return; }; done + echo "unknown fuzzer '${1:-}' (${FUZZERS[*]} or all)" >&2 + exit 2 +} + +need_build() { + local f + for f in "$@"; do [ -x "$OUT/$f" ] || { echo "no $OUT/$f: scripts/fuzz.sh build first" >&2; exit 1; }; done +} + +# libFuzzer writes new inputs into the first folder; the seeds are only read +run() { + local secs="${2:-300}" pids=() status=0 f + local list + read -ra list <<< "$(names "$1")" + need_build "${list[@]}" + mkdir -p "$OUT/findings" + for f in "${list[@]}"; do + mkdir -p "$OUT/corpus/$f" + "$OUT/$f" "${FUZZ_OPTS[@]}" -max_total_time="$secs" -close_fd_mask=3 -print_final_stats=1 \ + -artifact_prefix="$OUT/findings/$f-" "$OUT/corpus/$f" "$OUT/seeds/$f" $([ -d "$ROOT/tests/fuzz/regressions/$f" ] && echo "$ROOT/tests/fuzz/regressions/$f") > "$OUT/$f.log" 2>&1 & + pids+=($!) + done + for i in "${!pids[@]}"; do + f="${list[$i]}" + if wait "${pids[$i]}"; then + echo "$f: $(grep -m1 -o 'stat::number_of_executed_units: *[0-9]*' "$OUT/$f.log" | grep -o '[0-9]*$') runs, corpus $(find "$OUT/corpus/$f" -type f | wc -l), no findings" + else + status=1 + echo "$f: FINDING (log: $OUT/$f.log)" + grep -m3 -E 'ERROR:|runtime error:|SUMMARY:|keeps pausing|Test unit written' "$OUT/$f.log" || tail -20 "$OUT/$f.log" + fi + done + return $status +} + +check() { + local status=0 f dirs + local list + read -ra list <<< "$(names "$1")" + need_build "${list[@]}" + mkdir -p "$OUT/findings" + for f in "${list[@]}"; do + dirs=("$OUT/seeds/$f") + [ -d "$ROOT/tests/fuzz/regressions/$f" ] && dirs+=("$ROOT/tests/fuzz/regressions/$f") + [ -d "$OUT/corpus/$f" ] && dirs+=("$OUT/corpus/$f") + if "$OUT/$f" "${FUZZ_OPTS[@]}" -runs=0 -close_fd_mask=3 -artifact_prefix="$OUT/findings/$f-" "${dirs[@]}" > "$OUT/$f-check.log" 2>&1; then + echo "$f: $(find "${dirs[@]}" -type f | wc -l) inputs, no findings" + else + status=1 + echo "$f: FINDING (log: $OUT/$f-check.log)" + grep -m3 -E 'ERROR:|runtime error:|SUMMARY:|keeps pausing' "$OUT/$f-check.log" || tail -20 "$OUT/$f-check.log" + fi + done + return $status +} + +# keeps only the corpus inputs that add coverage (libFuzzer's merge), so the saved corpus doesn't grow forever +prune() { + local f + local list + read -ra list <<< "$(names "$1")" + need_build "${list[@]}" + for f in "${list[@]}"; do + [ -d "$OUT/corpus/$f" ] || continue + rm -rf "$OUT/corpus/$f.new" + mkdir -p "$OUT/corpus/$f.new" + "$OUT/$f" "${FUZZ_OPTS[@]}" -merge=1 -close_fd_mask=3 "$OUT/corpus/$f.new" "$OUT/corpus/$f" > "$OUT/$f-prune.log" 2>&1 \ + || { echo "$f: merge failed (log: $OUT/$f-prune.log)"; return 1; } + echo "$f: corpus $(find "$OUT/corpus/$f" -type f | wc -l) -> $(find "$OUT/corpus/$f.new" -type f | wc -l)" + rm -rf "$OUT/corpus/$f" + mv "$OUT/corpus/$f.new" "$OUT/corpus/$f" + done +} + +case "${1:-}" in + build) build ;; + run) run "${2:-all}" "${3:-300}" ;; + check) check "${2:-all}" ;; + prune) prune "${2:-all}" ;; + repro) + [ $# -eq 3 ] || { echo "usage: scripts/fuzz.sh repro " >&2; exit 2; } + need_build "$(names "$2")" + exec "$OUT/$2" "${FUZZ_OPTS[@]}" "$3" ;; + *) sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; +esac diff --git a/tests/README.md b/tests/README.md index 4b5d64b..5132c1f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -48,6 +48,10 @@ it serves, and the run stops if that isn't the page `DISCPRESS_HTML` picks (stop checks skip. Set it locally too to be sure a run compared everything. The same workflow runs `scripts/check-dist.sh`, `scripts/engine-diff.sh --check` and `scripts/check-licenses.sh`. +`.github/workflows/browsers.yml` runs the suite in Firefox and in WebKit (the `firefox`, `webkit` and +`iphone-webkit` projects, `EXTRA_BROWSERS=1`), with the same references, when the page or the tests +change and weekly. + `.github/workflows/sanitizers.yml` runs `ui/engine.spec.js` on the engine built with AddressSanitizer, LeakSanitizer and UndefinedBehaviorSanitizer (`scripts/build-sanitized.sh`, `build/chdman-san`), named by `CHDMAN_ENGINE`: a sanitizer report in chdman's output fails the test, whatever its exit code; a @@ -57,6 +61,10 @@ when `engine/`, `wasm/`, the spec, the fixtures or the suppressions change, and (`support/lsan.supp`, each with how that was checked). Locally: `CHDMAN_ENGINE=../build/chdman-san LSAN_OPTIONS=suppressions=$PWD/support/lsan.supp npx playwright test ui/engine.spec.js`. +`.github/workflows/fuzz.yml` fuzzes the engine with libFuzzer under the same sanitizers +(`scripts/fuzz.sh`; what each fuzzer feeds it, and the bugs found so far: `fuzz/README.md`). Each +finding, once fixed, is kept as an input in `fuzz/regressions/`, which every run replays. + ## How correctness is checked - **Native chdman is the oracle.** Every CHD the app creates is compared with what native chdman @@ -136,8 +144,14 @@ Times real conversions through the UI (from pressing the button to the finished with the page's own clock) and the same work with native chdman. Options: `--fixtures`, `--quick`, `--ops create,extract`, `--threads 1,2,4`, `--simd on,off`, `--presets default,plan,fast,zstd,none` (`plan` and `fast` use the engine's own options and are checked against `build/chdman-native`), `--repeat N`, `--html PATH`, `--label NAME`, -`--compare FILE|latest`, `--cd-mb`, `--dvd-mb`, `--no-native`, `--no-verify`. +`--compare FILE|latest`, `--cd-mb`, `--dvd-mb`, `--no-native`, `--no-verify`, `--browser chromium|firefox|webkit` +(the other two once Playwright has installed them). Each run writes `.cache/bench/