From 7390803287013f56811a8d6eaab69b09053a46ce Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 04:30:12 +0000 Subject: [PATCH 1/4] Engine: a wider compression window in the browser, byte for byte chdman's CHDs Profiling the job worker (DEBUG.profile, now in the page) showed the helpers idle half the time: chdman keeps 256 hunks in flight, so with 4 helpers a batch was 16 hunks, 1.2 ms of work for zlib,huff, and each costs a message. In the browser the compressor now keeps up to 1024 hunks of 4 KiB or less in flight (m_window; chdman's 256 for larger hunks and with a parent), and the page sizes batches from it, up to 64 hunks. chdman's window is part of its output: a partial last hunk is compressed whole, with what the work buffer held past the end of the data, the hunk 256 before it. So that tail is copied from that hunk, still in the wider buffer (zeros under 256 hunks), and the CHD is chdman 0.289's byte for byte (tests/ui/window.spec.js, each case, 1 and 4 threads). On the 1 GB DVD at 4 threads: Faster to create 12.99 -> 10.36 s, the default 50.42 -> 44.67 s. Also: chdman prints the whole CHD's final ratio (0.289 printed the last progress step's, as little as 0% in the browser). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QUSh6AFtA1CAR66DNvjtty --- app/ui.js | 4 +- app/worker.js | 50 +++++-- dist/discpress.html | 227 +++++++++++++++++++++++++++---- engine/README.md | 2 + engine/mame-0.289.diff | 158 ++++++++++++++++++++- engine/mame/src/lib/util/chd.cpp | 68 ++++++--- engine/mame/src/lib/util/chd.h | 9 ++ tests/fuzz/host.cpp | 2 +- tests/fuzz/seeds.cpp | 7 + tests/ui/window.spec.js | 62 +++++++++ wasm/par_lib.js | 7 +- 11 files changed, 529 insertions(+), 67 deletions(-) create mode 100644 tests/ui/window.spec.js diff --git a/app/ui.js b/app/ui.js index e0f4558..3018d94 100644 --- a/app/ui.js +++ b/app/ui.js @@ -400,7 +400,7 @@ var Engine = { hw.onerror = function (e) { helperFailed(e.message || 'failed to start'); }; // (tests: DEBUG.failHelpers makes helpers fail as a miscompiling WebAssembly engine does; 'lie': the // first one answers for other hunks than it was given) - self.post(hw, { type: 'helper', port: ch.port2, fail: o.debugFail === 'all' || (o.debugFail === 'one' && i === 0), lie: o.debugFail === 'lie' && i === 0 }, [ch.port2]); + self.post(hw, { type: 'helper', port: ch.port2, fail: o.debugFail === 'all' || (o.debugFail === 'one' && i === 0), lie: o.debugFail === 'lie' && i === 0, profile: !!DEBUG.profile }, [ch.port2]); helpers.push(hw); ports.push(ch.port1); } catch (e) { break; } @@ -441,7 +441,7 @@ var Engine = { self.post(worker, { type: 'run', jobId: o.jobId, dirPath: o.dirPath, args: o.args, inputs: o.inputs || [], writable: o.writable || [], slots: o.slots || 0, outMode: o.outMode || 'mem', ports: ports, debugStage: DEBUG.stage, debugSinkMax: DEBUG.sinkMax, debugStorage: DEBUG.failStorage, - debugFreeze: o.onStall ? DEBUG.freeze : 0 + debugFreeze: o.onStall ? DEBUG.freeze : 0, debugProfile: !!DEBUG.profile }, ports); }); return { diff --git a/app/worker.js b/app/worker.js index 3a2a4d5..ebe9055 100644 --- a/app/worker.js +++ b/app/worker.js @@ -49,6 +49,11 @@ function instantiate(extra) { return createChdman(opts); } +/* ---------------- profiling (DEBUG.profile) ---------------- */ +// where the job worker's time goes, in ms by what it did; reported in the job's log when chdman ends +var PROF = null; +function profAdd(name, t0) { if (PROF) PROF[name] = (PROF[name] || 0) + (performance.now() - t0); } + /* ---------------- storage backends ---------------- */ function BlobStore(blob) { @@ -71,12 +76,14 @@ BlobStore.prototype.block = function (bi) { return b; }; BlobStore.prototype.read = function (dst, pos, len) { + var t0 = PROF ? performance.now() : 0; try { return this.read2(dst, pos, len); } catch (e) { if (e && typeof e.errno === 'number') throw e; postMessage({ type: 'notice', level: 'error', message: 'Reading the input failed: ' + (e && e.message || e) }); throw { errno: ERR.EIO }; } + finally { profAdd('read input', t0); } }; BlobStore.prototype.read2 = function (dst, pos, len) { if (pos >= this.sizeV) return 0; @@ -488,12 +495,13 @@ function OpfsStore(slot) { this.tick = 0; } OpfsStore.prototype.put = function (src, pos) { - var n; + var n, t0 = PROF ? performance.now() : 0; try { n = this.h.write(src, { at: pos }); } catch (e) { postMessage({ type: 'notice', level: 'error', message: 'Storage full: the browser refused to write more data (' + (e && e.name || e) + ').' }); throw { errno: ERR.ENOSPC }; } + finally { profAdd('write output', t0); } if (n !== src.length) throw { errno: ERR.ENOSPC }; }; OpfsStore.prototype.flushBuf = function (b) { @@ -1045,6 +1053,11 @@ function setupParallel(M, ports) { if (m.type !== 'result') return; // chdman ended (done, or failed): its work items are gone, so nothing more is written to them if (closed) return; + var tr = PROF ? performance.now() : 0; + try { takeResult(h, m); } finally { profAdd('take results', tr); } + }; + function takeResult(h, m) { + if (PROF && m.ms) { PROF['helpers compressing'] = (PROF['helpers compressing'] || 0) + m.ms[0]; PROF['helpers busy'] = (PROF['helpers busy'] || 0) + m.ms[1]; } // results go into chdman's work items only for the batch this helper was given, item by item: // an answer for another batch, or for other items, would write over hunks it doesn't hold. And // each as the compressor gives it: a codec slot that was tried, shorter than a hunk; -1 (stored) @@ -1073,19 +1086,20 @@ function setupParallel(M, ports) { } dispatch(); maybeWake(); - }; + } }); // flags: 1 = libdeflate for the deflate codec (--libdeflate) - M.parSetup = function (hb, ub, comps, flags) { + M.parSetup = function (hb, ub, comps, flags, window) { if (!helpers.length) return false; // the LaserDisc codec (avhu) reads settings from the CHD's own metadata: keep it single-threaded if (comps.indexOf(0x61766875) >= 0) return false; hunkbytes = hb; - // chdman keeps at most 256 hunks in flight and reads them 128 at a time, so a batch must be - // small enough for those 128 to give every helper two batches; otherwise helpers sit idle - // (with 4 KiB DVD hunks, 512 KiB batches meant only two helpers ever had work) - batchSize = Math.max(1, Math.min(64, Math.floor((512 * 1024) / hb), Math.floor(128 / (2 * helpers.length)))); + // chdman keeps `window` hunks in flight (256, or up to 1024 small ones in the browser) and reads + // half of them at a time, so a batch must be small enough for each read to give every helper two + // batches; otherwise helpers sit idle (with 4 KiB DVD hunks, 512 KiB batches meant only two helpers + // ever had work). And as large as that allows: each batch costs a message both ways + batchSize = Math.max(1, Math.min(64, Math.floor((512 * 1024) / hb), Math.floor((window || 256) / 2 / (2 * helpers.length)))); scratch = M._malloc(hb + 64); if (!scratch) return false; // no memory for it: chdman compresses by itself helpers.forEach(function (h) { h.port.postMessage({ type: 'init', hunkbytes: hb, unitbytes: ub, comps: comps, flags: flags || 0 }); }); @@ -1100,13 +1114,17 @@ function setupParallel(M, ports) { }; // codecs: the codec slots the codec plan tries for this hunk, a bit each (15 = all) M.parSubmit = function (item, ptr, len, codecs) { + var t0 = PROF ? performance.now() : 0; + try { parSubmit(item, ptr, len, codecs); } finally { profAdd('hand out hunks', t0); } + }; + function parSubmit(item, ptr, len, codecs) { if (!open) open = { n: 0, items: new Uint32Array(batchSize), codecs: new Uint8Array(batchSize), buf: new Uint8Array(batchSize * hunkbytes) }; open.buf.set(M.HEAPU8.subarray(ptr, ptr + len), open.n * hunkbytes); open.codecs[open.n] = codecs; open.items[open.n++] = item; fifo.push(item); if (open.n === batchSize) flushOpen(); - }; + } // chdman paused after a compression step: resolves when it can go on (its next hunk is in) M.parWait = function () { return new Promise(function (resolve) { @@ -1254,9 +1272,12 @@ async function runHelper(msg) { if (m.type === 'batch') { var n = m.items.length, data = new Uint8Array(m.data), codecs = m.codecs; var meta = new Int32Array(n * 3), sha1 = new Uint8Array(n * 20), parts = [], total = 0; + var tb = msg.profile ? performance.now() : 0, tc = 0; for (var i = 0; i < n; i++) { M.HEAPU8.set(data.subarray(i * hb, (i + 1) * hb), inbuf); + var t1 = msg.profile ? performance.now() : 0; var c = M._wasm_helper_compress(res, sha, codecs ? codecs[i] : 15); + if (msg.profile) tc += performance.now() - t1; var heap32 = M.HEAPU32, heap8 = M.HEAPU8; var len = heap32[res >>> 2], crc = heap32[(res >>> 2) + 1]; meta[i * 3] = c; meta[i * 3 + 1] = len; meta[i * 3 + 2] = crc; @@ -1267,8 +1288,8 @@ async function runHelper(msg) { var out = new Uint8Array(total), o = 0; parts.forEach(function (p) { out.set(p, o); o += p.length; }); if (msg.lie) m.items[0] += 64; // a test (DEBUG.failHelpers 'lie'): an answer for another hunk - port.postMessage({ type: 'result', id: m.id, items: m.items, meta: meta, sha1: sha1, out: out }, - [m.items.buffer, meta.buffer, sha1.buffer, out.buffer]); + port.postMessage({ type: 'result', id: m.id, items: m.items, meta: meta, sha1: sha1, out: out, + ms: msg.profile ? [tc, performance.now() - tb] : null }, [m.items.buffer, meta.buffer, sha1.buffer, out.buffer]); } } catch (err) { // (a WebAssembly trap here, which some Safari versions' optimizing compiler causes: the job worker @@ -1318,21 +1339,30 @@ async function runChdman(M, args) { ['chdman'].concat(args).forEach(function (a, i) { var p = M.stringToNewUTF8(a); M.HEAPU32[(argv >>> 2) + i] = p; }); M.HEAPU32[(argv >>> 2) + args.length + 1] = 0; try { + var t0 = PROF ? performance.now() : 0; var code = M._chdman_begin(args.length + 1, argv); + profAdd('in the engine', t0); while (code === -1) { + t0 = PROF ? performance.now() : 0; if (sinkFull()) await sinkDrained(); if (M.rdBusy && M.rdBusy()) await M.rdWait(); else if (M.parWait) await M.parWait(); + profAdd('waiting for helpers', t0); + t0 = PROF ? performance.now() : 0; code = M._chdman_resume(); + profAdd('in the engine', t0); } return code; } finally { if (M.parClose) M.parClose(); + // (the engine's time includes its reads, writes and hunks handed out; the wait includes taking results) + if (PROF) postMessage({ type: 'notice', level: 'info', message: 'Profile (ms): ' + Object.keys(PROF).map(function (k) { return k + ' ' + Math.round(PROF[k]); }).join(', ') }); } } async function runJob(msg) { if (msg.debugSinkMax) SINK_MAX = msg.debugSinkMax; // testing: a small limit + if (msg.debugProfile) PROF = {}; await loadModule(msg); var backing = await makeBacking(msg); postMessage({ type: 'storage', mode: backing.mode, reason: memReason }); diff --git a/dist/discpress.html b/dist/discpress.html index 81456be..211e714 100644 --- a/dist/discpress.html +++ b/dist/discpress.html @@ -3312,7 +3312,7 @@

About and licenses

+#include <emscripten.h> +extern "C" { + // provided by the JavaScript host (browser build): offload hunk compression to helper workers -+ int wasm_par_enabled(uint32_t hunkbytes, uint32_t unitbytes, const uint32_t *compression, uint32_t flags); ++ int wasm_par_enabled(uint32_t hunkbytes, uint32_t unitbytes, const uint32_t *compression, uint32_t flags, uint32_t window); + void wasm_par_submit(void *item, const uint8_t *data, uint32_t length, uint32_t codecs); + // and decompression, for commands that read (see chd_file::wasm_read_ahead) + int wasm_rd_enabled(const void *chd, uint32_t hunkbytes, uint32_t unitbytes, const uint32_t *compression); @@ -3817,18 +3817,81 @@

About and licenses

// free the work queues osd_work_queue_free(m_read_queue); osd_work_queue_free(m_work_queue); -@@ -3022,6 +3244,10 @@ +@@ -3000,11 +3222,21 @@ + m_read_done_offset = 0; + m_read_error.clear(); + ++ // Discpress: the window. chdman's 256 hunks natively; in the browser up to 4 MiB of hunks (at most ++ // WORK_BUFFER_HUNKS), as one message to a helper worker costs about as much as compressing a few ++ // small hunks, but chdman's 256 for larger hunks (CDs') and with a parent, whose walk leaves other ++ // bytes in the buffer than the ones a partial last hunk must get (async_read) ++ m_window = CHDMAN_WINDOW; ++#ifdef __EMSCRIPTEN__ ++ if (!m_parent) ++ m_window = std::clamp<uint32_t>(((4 << 20) / hunk_bytes()) & ~1U, CHDMAN_WINDOW, WORK_BUFFER_HUNKS); ++#endif ++ + // reset work item state +- m_work_buffer.resize(hunk_bytes() * (WORK_BUFFER_HUNKS + 1)); ++ m_work_buffer.resize(hunk_bytes() * (m_window + 1)); + memset(&m_work_buffer[0], 0, m_work_buffer.size()); +- m_compressed_buffer.resize(hunk_bytes() * WORK_BUFFER_HUNKS); +- for (int itemnum = 0; itemnum < WORK_BUFFER_HUNKS; itemnum++) ++ m_compressed_buffer.resize(hunk_bytes() * m_window); ++ for (uint32_t itemnum = 0; itemnum < m_window; itemnum++) + { + work_item &item = m_work_item[itemnum]; + item.m_compressor = this; +@@ -3022,6 +3254,10 @@ // reset write state m_write_hunk = 0; + +#ifdef __EMSCRIPTEN__ -+ s_wasm_par = compressed() ? wasm_par_enabled(hunk_bytes(), unit_bytes(), m_compression, m_libdeflate ? 1 : 0) : 0; // flags: 1 = libdeflate ++ s_wasm_par = compressed() ? wasm_par_enabled(hunk_bytes(), unit_bytes(), m_compression, m_libdeflate ? 1 : 0, m_window) : 0; // flags: 1 = libdeflate +#endif } /** -@@ -3123,6 +3349,11 @@ +@@ -3048,9 +3284,9 @@ + { + // see if we have enough free work items to read the next half of a buffer + uint32_t const startitem = m_read_queue_offset / hunk_bytes(); +- uint32_t const enditem = startitem + WORK_BUFFER_HUNKS / 2; ++ uint32_t const enditem = startitem + m_window / 2; + uint32_t curitem = startitem; +- while ((curitem < enditem) && (m_work_item[curitem % WORK_BUFFER_HUNKS].m_status == WS_READY)) ++ while ((curitem < enditem) && (m_work_item[curitem % m_window].m_status == WS_READY)) + ++curitem; + + // if it's not all clear, defer +@@ -3059,20 +3295,20 @@ + + // if we're walking the parent, we want one more item to have cleared so we + // can read an extra hunk there +- if (m_walking_parent && m_work_item[curitem % WORK_BUFFER_HUNKS].m_status != WS_READY) ++ if (m_walking_parent && m_work_item[curitem % m_window].m_status != WS_READY) + break; + + // queue the next read + for (curitem = startitem; curitem < enditem; curitem++) +- m_work_item[curitem % WORK_BUFFER_HUNKS].m_status = WS_READING; ++ m_work_item[curitem % m_window].m_status = WS_READING; + osd_work_item_queue(m_read_queue, async_read_static, this, WORK_ITEM_FLAG_AUTO_RELEASE); +- m_read_queue_offset += WORK_BUFFER_HUNKS * hunk_bytes() / 2; ++ m_read_queue_offset += m_window * hunk_bytes() / 2; + } + + // flush out any finished items +- while (m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_status == WS_COMPLETE) ++ while (m_work_item[m_write_hunk % m_window].m_status == WS_COMPLETE) + { +- work_item &item = m_work_item[m_write_hunk % WORK_BUFFER_HUNKS]; ++ work_item &item = m_work_item[m_write_hunk % m_window]; + + // free any OSD work item + if (item.m_osd) +@@ -3123,6 +3359,11 @@ } else { @@ -3840,7 +3903,35 @@

About and licenses

// otherwise, append it compressed and add to the self map hunk_write_compressed(item.m_hunknum, item.m_compression, item.m_compressed, item.m_complen, item.m_hash[0].m_crc16); m_total_out += item.m_complen; -@@ -3263,12 +3494,44 @@ +@@ -3148,6 +3389,12 @@ + } + else + { ++ // Discpress: the ratio of the whole CHD, which chdman prints as the final one (0.289 left it as ++ // the last step before had it: in the browser, where helper workers' results come in batches, ++ // as little as 0%) ++ progress = 1.0; ++ ratio = (m_total_in == 0) ? 1.0 : double(m_total_out) / double(m_total_in); ++ + // wait for all reads to finish and if we're compressed, write the final SHA1 and map + osd_work_queue_wait(m_read_queue, 30 * osd_ticks_per_second()); + if (!compressed()) +@@ -3169,10 +3416,10 @@ + + // if we're waiting for work, wait + // sometimes code can get here with .m_status == WS_READY and .m_osd != nullptr, TODO find out why this happens +- while ((m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_status != WS_READY) && +- (m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_status != WS_COMPLETE) && +- m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_osd) +- osd_work_item_wait(m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_osd, osd_ticks_per_second()); ++ while ((m_work_item[m_write_hunk % m_window].m_status != WS_READY) && ++ (m_work_item[m_write_hunk % m_window].m_status != WS_COMPLETE) && ++ m_work_item[m_write_hunk % m_window].m_osd) ++ osd_work_item_wait(m_work_item[m_write_hunk % m_window].m_osd, osd_ticks_per_second()); + + return m_walking_parent ? error::WALKING_PARENT : error::COMPRESSING; + } +@@ -3263,12 +3510,44 @@ // TODO: data race if ((m_current_map.find(item.m_hash[0].m_crc16, item.m_hash[0].m_sha1) == hashmap::NOT_FOUND) && (m_parent_map.find(item.m_hash[0].m_crc16, item.m_hash[0].m_sha1) == hashmap::NOT_FOUND)) @@ -3886,7 +3977,43 @@

About and licenses

/** * @fn void *chd_file_compressor::async_read_static(void *param, int threadid) * -@@ -3345,6 +3608,16 @@ +@@ -3301,7 +3580,7 @@ + return; + + // determine parameters for the read +- uint32_t const work_buffer_bytes = WORK_BUFFER_HUNKS * hunk_bytes(); ++ uint32_t const work_buffer_bytes = m_window * hunk_bytes(); + uint32_t numbytes = work_buffer_bytes / 2; + if ((m_read_done_offset + numbytes) > logical_bytes()) + numbytes = logical_bytes() - m_read_done_offset; +@@ -3335,16 +3614,42 @@ + { + // otherwise, call the virtual function + read_data(dest, m_read_done_offset, numbytes); ++#ifdef __EMSCRIPTEN__ ++ // Discpress: a partial last hunk is compressed whole, and past the end of the data it holds ++ // what the work buffer held there before: with a wider window than chdman's, other bytes than ++ // in chdman's buffer. So it gets chdman's: those of the hunk CHDMAN_WINDOW before it (still in ++ // the buffer, as the window is wider), or the zeros the buffer starts with when there is none ++ if ((m_window != CHDMAN_WINDOW) && (end_offset == logical_bytes()) && (logical_bytes() % hunk_bytes())) ++ { ++ uint64_t const last = logical_bytes() / hunk_bytes(); ++ uint32_t const tail = logical_bytes() % hunk_bytes(); ++ uint8_t *const slot = &m_work_buffer[(last % m_window) * hunk_bytes()]; ++ if (last >= CHDMAN_WINDOW) ++ memcpy(slot + tail, &m_work_buffer[((last - CHDMAN_WINDOW) % m_window) * hunk_bytes()] + tail, hunk_bytes() - tail); ++ else ++ memset(slot + tail, 0, hunk_bytes() - tail); ++ } ++#endif + } + + // spawn off work for each hunk + for (uint64_t curoffs = m_read_done_offset; curoffs < end_offset; curoffs += hunk_bytes()) + { + uint32_t hunknum = curoffs / hunk_bytes(); +- work_item &item = m_work_item[hunknum % WORK_BUFFER_HUNKS]; ++ work_item &item = m_work_item[hunknum % m_window]; assert(item.m_status == WS_READING); item.m_status = WS_QUEUED; item.m_hunknum = hunknum; @@ -3903,7 +4030,7 @@

About and licenses

item.m_osd = osd_work_item_queue(m_work_queue, m_walking_parent ? async_walk_parent_static : async_compress_hunk_static, &item, 0); } -@@ -3512,3 +3785,184 @@ +@@ -3512,3 +3817,184 @@ metadata_entry metaentry; return metadata_find(AV_METADATA_TAG, 0, metaentry); } @@ -4223,6 +4350,23 @@

About and licenses

bool m_walking_parent; // are we building the parent map? uint64_t m_total_in; // total bytes in uint64_t m_total_out; // total bytes out +@@ -565,7 +623,16 @@ + std::error_condition m_read_error; // error during reading, if any + + // work item thread ++#ifdef __EMSCRIPTEN__ ++ // Discpress (browser build): room for up to 1024 hunks in flight, so that the batches helper workers ++ // get carry enough work for their messages (see m_window) ++ static constexpr int WORK_BUFFER_HUNKS = 1024; ++#else + static constexpr int WORK_BUFFER_HUNKS = 256; ++#endif ++ // Discpress: chdman's own window, which the bytes past the end of a partial last hunk follow (async_read) ++ static constexpr uint32_t CHDMAN_WINDOW = 256; ++ uint32_t m_window = CHDMAN_WINDOW; // Discpress: hunks in the work buffer, at most WORK_BUFFER_HUNKS + osd_work_queue * m_work_queue; // queue for doing work on other threads + std::vector<uint8_t> m_work_buffer; // buffer containing hunk data to work on + std::vector<uint8_t> m_compressed_buffer;// buffer containing compressed data --- a/src/lib/util/chdcodec.cpp +++ b/src/lib/util/chdcodec.cpp @@ -20,9 +20,12 @@ @@ -7020,12 +7164,13 @@

About and licenses

--- /dev/null +++ par_lib.js (new file, Emscripten JS library) +addToLibrary({ -+ // flags: 1 = the deflate codec encodes with libdeflate (--libdeflate) -+ wasm_par_enabled: function (hunkbytes, unitbytes, comp, flags) { ++ // flags: 1 = the deflate codec encodes with libdeflate (--libdeflate); window: the hunks chdman keeps in ++ // flight, reading half of them at a time (chd_file_compressor::m_window) ++ wasm_par_enabled: function (hunkbytes, unitbytes, comp, flags, window) { + if (!Module['parSetup']) return 0; + var p = comp >>> 2; + var list = [HEAPU32[p], HEAPU32[p + 1], HEAPU32[p + 2], HEAPU32[p + 3]]; -+ return Module['parSetup'](hunkbytes >>> 0, unitbytes >>> 0, list, flags >>> 0) ? 1 : 0; ++ return Module['parSetup'](hunkbytes >>> 0, unitbytes >>> 0, list, flags >>> 0, window >>> 0) ? 1 : 0; + }, + // codecs: the codec slots to try for this hunk, a bit each (the codec plan; 15 = all) + wasm_par_submit: function (item, data, length, codecs) { @@ -7706,7 +7851,7 @@

About and licenses

- - - + + - + diff --git a/engine/README.md b/engine/README.md index 1e6941a..a3b3aa0 100644 --- a/engine/README.md +++ b/engine/README.md @@ -6,6 +6,7 @@ The paths mirror MAME's, so every file can be compared with its original. **[`ma - **Browser build**, under `#ifdef __EMSCRIPTEN__`: - helper-worker hooks in `chd_file_compressor` (`chd.cpp`, `chd.h`); + - a wider window for the compressor (`m_window`): up to 1024 hunks in flight instead of 256, for hunks of 4 KiB or less without a parent, so that the page can give each helper batches of up to 64 hunks (one message costs about as much as compressing a few small hunks). chdman's window is part of its output: a partial last hunk is compressed whole, and past the end of the data it holds what the work buffer held there, the hunk 256 before it in chdman's buffer (or zeros in a CHD of fewer). So with a wider window that tail is copied from that hunk, still in the buffer, and the CHD is chdman's byte for byte (`tests/ui/window.spec.js`). Natively the window stays 256; - `chd_file::wasm_read_ahead`, which reads a window of stored hunks in one go for `verify` and the extract commands, hands the compressed ones to helper workers, and lets `read_hunk` take them from memory (with the usual CRC check); - verify and extract read in 8 MB buffers instead of 32 MB; - compression and `read_ahead` also wait while results streamed into a folder are waiting to be written (`wasm_out_full`, `chdman.cpp`). @@ -63,6 +64,7 @@ The paths mirror MAME's, so every file can be compared with its original. **[`ma - A CD codec's hunk (`cdlz`, `cdzl`, `cdzs`) whose header is longer than the hunk's data, or says its sector data is, is a decompression error (`chdcodec.cpp`): upstream decoded past the end of the data, and the subcode's length wrapped around. So is a v5 map that reads more than 32 bits at a time (`decompress_v5_map`): upstream's bit reader shifted by a negative count. - CD track metadata is read with a width for each string (`%15s`, the longest a CD has being 15 characters): upstream's `%s` wrote a longer one past the end of its 16-byte buffer, on the stack (`parse_metadata`, `cdrom.cpp`). - CD metadata that leaves out a track (a track number repeated or skipped), and old-style binary CD metadata (`CHCD`) shorter than its table, with more than 99 tracks or with sectors no CD has, are invalid data (`parse_metadata`, `cdrom.cpp`): upstream made a track of 0-byte sectors, which `extractcd` divided by, or read past the end of the metadata and of a sector's buffer. + - The final ratio chdman prints is the whole CHD's (`compress_continue`): 0.289 printed the last progress step's, which in the browser, where helpers' results come in batches, could be as little as 0%. Only the line changes, not the CHD. - Reading or writing 0 bytes (`read_bytes`, `write_bytes`) does nothing: upstream's last hunk wrapped around at offset 0, and it read a whole hunk into the caller's buffer. The page's identification reads a CD track that has no metadata (a damaged CHD's) that way. - LaserDisc (`avhu`) hunks: a frame larger than the hunk, more than 16 audio channels, or raw audio shorter than its samples is a decompression error (`chdcodec.cpp`, `avhuff.cpp`). Upstream decoded past the end of the hunk, of its tables of channels, and of the data. - Any other unexpected exception is an error instead of a crash. diff --git a/engine/mame-0.289.diff b/engine/mame-0.289.diff index d910f70..b7da3d8 100644 --- a/engine/mame-0.289.diff +++ b/engine/mame-0.289.diff @@ -2473,7 +2473,7 @@ +#include +extern "C" { + // provided by the JavaScript host (browser build): offload hunk compression to helper workers -+ int wasm_par_enabled(uint32_t hunkbytes, uint32_t unitbytes, const uint32_t *compression, uint32_t flags); ++ int wasm_par_enabled(uint32_t hunkbytes, uint32_t unitbytes, const uint32_t *compression, uint32_t flags, uint32_t window); + void wasm_par_submit(void *item, const uint8_t *data, uint32_t length, uint32_t codecs); + // and decompression, for commands that read (see chd_file::wasm_read_ahead) + int wasm_rd_enabled(const void *chd, uint32_t hunkbytes, uint32_t unitbytes, const uint32_t *compression); @@ -2978,18 +2978,81 @@ // free the work queues osd_work_queue_free(m_read_queue); osd_work_queue_free(m_work_queue); -@@ -3022,6 +3244,10 @@ +@@ -3000,11 +3222,21 @@ + m_read_done_offset = 0; + m_read_error.clear(); + ++ // Discpress: the window. chdman's 256 hunks natively; in the browser up to 4 MiB of hunks (at most ++ // WORK_BUFFER_HUNKS), as one message to a helper worker costs about as much as compressing a few ++ // small hunks, but chdman's 256 for larger hunks (CDs') and with a parent, whose walk leaves other ++ // bytes in the buffer than the ones a partial last hunk must get (async_read) ++ m_window = CHDMAN_WINDOW; ++#ifdef __EMSCRIPTEN__ ++ if (!m_parent) ++ m_window = std::clamp(((4 << 20) / hunk_bytes()) & ~1U, CHDMAN_WINDOW, WORK_BUFFER_HUNKS); ++#endif ++ + // reset work item state +- m_work_buffer.resize(hunk_bytes() * (WORK_BUFFER_HUNKS + 1)); ++ m_work_buffer.resize(hunk_bytes() * (m_window + 1)); + memset(&m_work_buffer[0], 0, m_work_buffer.size()); +- m_compressed_buffer.resize(hunk_bytes() * WORK_BUFFER_HUNKS); +- for (int itemnum = 0; itemnum < WORK_BUFFER_HUNKS; itemnum++) ++ m_compressed_buffer.resize(hunk_bytes() * m_window); ++ for (uint32_t itemnum = 0; itemnum < m_window; itemnum++) + { + work_item &item = m_work_item[itemnum]; + item.m_compressor = this; +@@ -3022,6 +3254,10 @@ // reset write state m_write_hunk = 0; + +#ifdef __EMSCRIPTEN__ -+ s_wasm_par = compressed() ? wasm_par_enabled(hunk_bytes(), unit_bytes(), m_compression, m_libdeflate ? 1 : 0) : 0; // flags: 1 = libdeflate ++ s_wasm_par = compressed() ? wasm_par_enabled(hunk_bytes(), unit_bytes(), m_compression, m_libdeflate ? 1 : 0, m_window) : 0; // flags: 1 = libdeflate +#endif } /** -@@ -3123,6 +3349,11 @@ +@@ -3048,9 +3284,9 @@ + { + // see if we have enough free work items to read the next half of a buffer + uint32_t const startitem = m_read_queue_offset / hunk_bytes(); +- uint32_t const enditem = startitem + WORK_BUFFER_HUNKS / 2; ++ uint32_t const enditem = startitem + m_window / 2; + uint32_t curitem = startitem; +- while ((curitem < enditem) && (m_work_item[curitem % WORK_BUFFER_HUNKS].m_status == WS_READY)) ++ while ((curitem < enditem) && (m_work_item[curitem % m_window].m_status == WS_READY)) + ++curitem; + + // if it's not all clear, defer +@@ -3059,20 +3295,20 @@ + + // if we're walking the parent, we want one more item to have cleared so we + // can read an extra hunk there +- if (m_walking_parent && m_work_item[curitem % WORK_BUFFER_HUNKS].m_status != WS_READY) ++ if (m_walking_parent && m_work_item[curitem % m_window].m_status != WS_READY) + break; + + // queue the next read + for (curitem = startitem; curitem < enditem; curitem++) +- m_work_item[curitem % WORK_BUFFER_HUNKS].m_status = WS_READING; ++ m_work_item[curitem % m_window].m_status = WS_READING; + osd_work_item_queue(m_read_queue, async_read_static, this, WORK_ITEM_FLAG_AUTO_RELEASE); +- m_read_queue_offset += WORK_BUFFER_HUNKS * hunk_bytes() / 2; ++ m_read_queue_offset += m_window * hunk_bytes() / 2; + } + + // flush out any finished items +- while (m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_status == WS_COMPLETE) ++ while (m_work_item[m_write_hunk % m_window].m_status == WS_COMPLETE) + { +- work_item &item = m_work_item[m_write_hunk % WORK_BUFFER_HUNKS]; ++ work_item &item = m_work_item[m_write_hunk % m_window]; + + // free any OSD work item + if (item.m_osd) +@@ -3123,6 +3359,11 @@ } else { @@ -3001,7 +3064,35 @@ // otherwise, append it compressed and add to the self map hunk_write_compressed(item.m_hunknum, item.m_compression, item.m_compressed, item.m_complen, item.m_hash[0].m_crc16); m_total_out += item.m_complen; -@@ -3263,12 +3494,44 @@ +@@ -3148,6 +3389,12 @@ + } + else + { ++ // Discpress: the ratio of the whole CHD, which chdman prints as the final one (0.289 left it as ++ // the last step before had it: in the browser, where helper workers' results come in batches, ++ // as little as 0%) ++ progress = 1.0; ++ ratio = (m_total_in == 0) ? 1.0 : double(m_total_out) / double(m_total_in); ++ + // wait for all reads to finish and if we're compressed, write the final SHA1 and map + osd_work_queue_wait(m_read_queue, 30 * osd_ticks_per_second()); + if (!compressed()) +@@ -3169,10 +3416,10 @@ + + // if we're waiting for work, wait + // sometimes code can get here with .m_status == WS_READY and .m_osd != nullptr, TODO find out why this happens +- while ((m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_status != WS_READY) && +- (m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_status != WS_COMPLETE) && +- m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_osd) +- osd_work_item_wait(m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_osd, osd_ticks_per_second()); ++ while ((m_work_item[m_write_hunk % m_window].m_status != WS_READY) && ++ (m_work_item[m_write_hunk % m_window].m_status != WS_COMPLETE) && ++ m_work_item[m_write_hunk % m_window].m_osd) ++ osd_work_item_wait(m_work_item[m_write_hunk % m_window].m_osd, osd_ticks_per_second()); + + return m_walking_parent ? error::WALKING_PARENT : error::COMPRESSING; + } +@@ -3263,12 +3510,44 @@ // TODO: data race if ((m_current_map.find(item.m_hash[0].m_crc16, item.m_hash[0].m_sha1) == hashmap::NOT_FOUND) && (m_parent_map.find(item.m_hash[0].m_crc16, item.m_hash[0].m_sha1) == hashmap::NOT_FOUND)) @@ -3047,7 +3138,43 @@ /** * @fn void *chd_file_compressor::async_read_static(void *param, int threadid) * -@@ -3345,6 +3608,16 @@ +@@ -3301,7 +3580,7 @@ + return; + + // determine parameters for the read +- uint32_t const work_buffer_bytes = WORK_BUFFER_HUNKS * hunk_bytes(); ++ uint32_t const work_buffer_bytes = m_window * hunk_bytes(); + uint32_t numbytes = work_buffer_bytes / 2; + if ((m_read_done_offset + numbytes) > logical_bytes()) + numbytes = logical_bytes() - m_read_done_offset; +@@ -3335,16 +3614,42 @@ + { + // otherwise, call the virtual function + read_data(dest, m_read_done_offset, numbytes); ++#ifdef __EMSCRIPTEN__ ++ // Discpress: a partial last hunk is compressed whole, and past the end of the data it holds ++ // what the work buffer held there before: with a wider window than chdman's, other bytes than ++ // in chdman's buffer. So it gets chdman's: those of the hunk CHDMAN_WINDOW before it (still in ++ // the buffer, as the window is wider), or the zeros the buffer starts with when there is none ++ if ((m_window != CHDMAN_WINDOW) && (end_offset == logical_bytes()) && (logical_bytes() % hunk_bytes())) ++ { ++ uint64_t const last = logical_bytes() / hunk_bytes(); ++ uint32_t const tail = logical_bytes() % hunk_bytes(); ++ uint8_t *const slot = &m_work_buffer[(last % m_window) * hunk_bytes()]; ++ if (last >= CHDMAN_WINDOW) ++ memcpy(slot + tail, &m_work_buffer[((last - CHDMAN_WINDOW) % m_window) * hunk_bytes()] + tail, hunk_bytes() - tail); ++ else ++ memset(slot + tail, 0, hunk_bytes() - tail); ++ } ++#endif + } + + // spawn off work for each hunk + for (uint64_t curoffs = m_read_done_offset; curoffs < end_offset; curoffs += hunk_bytes()) + { + uint32_t hunknum = curoffs / hunk_bytes(); +- work_item &item = m_work_item[hunknum % WORK_BUFFER_HUNKS]; ++ work_item &item = m_work_item[hunknum % m_window]; assert(item.m_status == WS_READING); item.m_status = WS_QUEUED; item.m_hunknum = hunknum; @@ -3064,7 +3191,7 @@ item.m_osd = osd_work_item_queue(m_work_queue, m_walking_parent ? async_walk_parent_static : async_compress_hunk_static, &item, 0); } -@@ -3512,3 +3785,184 @@ +@@ -3512,3 +3817,184 @@ metadata_entry metaentry; return metadata_find(AV_METADATA_TAG, 0, metaentry); } @@ -3384,6 +3511,23 @@ bool m_walking_parent; // are we building the parent map? uint64_t m_total_in; // total bytes in uint64_t m_total_out; // total bytes out +@@ -565,7 +623,16 @@ + std::error_condition m_read_error; // error during reading, if any + + // work item thread ++#ifdef __EMSCRIPTEN__ ++ // Discpress (browser build): room for up to 1024 hunks in flight, so that the batches helper workers ++ // get carry enough work for their messages (see m_window) ++ static constexpr int WORK_BUFFER_HUNKS = 1024; ++#else + static constexpr int WORK_BUFFER_HUNKS = 256; ++#endif ++ // Discpress: chdman's own window, which the bytes past the end of a partial last hunk follow (async_read) ++ static constexpr uint32_t CHDMAN_WINDOW = 256; ++ uint32_t m_window = CHDMAN_WINDOW; // Discpress: hunks in the work buffer, at most WORK_BUFFER_HUNKS + osd_work_queue * m_work_queue; // queue for doing work on other threads + std::vector m_work_buffer; // buffer containing hunk data to work on + std::vector m_compressed_buffer;// buffer containing compressed data --- a/src/lib/util/chdcodec.cpp +++ b/src/lib/util/chdcodec.cpp @@ -20,9 +20,12 @@ diff --git a/engine/mame/src/lib/util/chd.cpp b/engine/mame/src/lib/util/chd.cpp index db3abee..837324e 100644 --- a/engine/mame/src/lib/util/chd.cpp +++ b/engine/mame/src/lib/util/chd.cpp @@ -33,7 +33,7 @@ #include extern "C" { // provided by the JavaScript host (browser build): offload hunk compression to helper workers - int wasm_par_enabled(uint32_t hunkbytes, uint32_t unitbytes, const uint32_t *compression, uint32_t flags); + int wasm_par_enabled(uint32_t hunkbytes, uint32_t unitbytes, const uint32_t *compression, uint32_t flags, uint32_t window); void wasm_par_submit(void *item, const uint8_t *data, uint32_t length, uint32_t codecs); // and decompression, for commands that read (see chd_file::wasm_read_ahead) int wasm_rd_enabled(const void *chd, uint32_t hunkbytes, uint32_t unitbytes, const uint32_t *compression); @@ -3222,11 +3222,21 @@ void chd_file_compressor::compress_begin() m_read_done_offset = 0; m_read_error.clear(); + // Discpress: the window. chdman's 256 hunks natively; in the browser up to 4 MiB of hunks (at most + // WORK_BUFFER_HUNKS), as one message to a helper worker costs about as much as compressing a few + // small hunks, but chdman's 256 for larger hunks (CDs') and with a parent, whose walk leaves other + // bytes in the buffer than the ones a partial last hunk must get (async_read) + m_window = CHDMAN_WINDOW; +#ifdef __EMSCRIPTEN__ + if (!m_parent) + m_window = std::clamp(((4 << 20) / hunk_bytes()) & ~1U, CHDMAN_WINDOW, WORK_BUFFER_HUNKS); +#endif + // reset work item state - m_work_buffer.resize(hunk_bytes() * (WORK_BUFFER_HUNKS + 1)); + m_work_buffer.resize(hunk_bytes() * (m_window + 1)); memset(&m_work_buffer[0], 0, m_work_buffer.size()); - m_compressed_buffer.resize(hunk_bytes() * WORK_BUFFER_HUNKS); - for (int itemnum = 0; itemnum < WORK_BUFFER_HUNKS; itemnum++) + m_compressed_buffer.resize(hunk_bytes() * m_window); + for (uint32_t itemnum = 0; itemnum < m_window; itemnum++) { work_item &item = m_work_item[itemnum]; item.m_compressor = this; @@ -3246,7 +3256,7 @@ void chd_file_compressor::compress_begin() m_write_hunk = 0; #ifdef __EMSCRIPTEN__ - s_wasm_par = compressed() ? wasm_par_enabled(hunk_bytes(), unit_bytes(), m_compression, m_libdeflate ? 1 : 0) : 0; // flags: 1 = libdeflate + s_wasm_par = compressed() ? wasm_par_enabled(hunk_bytes(), unit_bytes(), m_compression, m_libdeflate ? 1 : 0, m_window) : 0; // flags: 1 = libdeflate #endif } @@ -3274,9 +3284,9 @@ std::error_condition chd_file_compressor::compress_continue(double &progress, do { // see if we have enough free work items to read the next half of a buffer uint32_t const startitem = m_read_queue_offset / hunk_bytes(); - uint32_t const enditem = startitem + WORK_BUFFER_HUNKS / 2; + uint32_t const enditem = startitem + m_window / 2; uint32_t curitem = startitem; - while ((curitem < enditem) && (m_work_item[curitem % WORK_BUFFER_HUNKS].m_status == WS_READY)) + while ((curitem < enditem) && (m_work_item[curitem % m_window].m_status == WS_READY)) ++curitem; // if it's not all clear, defer @@ -3285,20 +3295,20 @@ std::error_condition chd_file_compressor::compress_continue(double &progress, do // if we're walking the parent, we want one more item to have cleared so we // can read an extra hunk there - if (m_walking_parent && m_work_item[curitem % WORK_BUFFER_HUNKS].m_status != WS_READY) + if (m_walking_parent && m_work_item[curitem % m_window].m_status != WS_READY) break; // queue the next read for (curitem = startitem; curitem < enditem; curitem++) - m_work_item[curitem % WORK_BUFFER_HUNKS].m_status = WS_READING; + m_work_item[curitem % m_window].m_status = WS_READING; osd_work_item_queue(m_read_queue, async_read_static, this, WORK_ITEM_FLAG_AUTO_RELEASE); - m_read_queue_offset += WORK_BUFFER_HUNKS * hunk_bytes() / 2; + m_read_queue_offset += m_window * hunk_bytes() / 2; } // flush out any finished items - while (m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_status == WS_COMPLETE) + while (m_work_item[m_write_hunk % m_window].m_status == WS_COMPLETE) { - work_item &item = m_work_item[m_write_hunk % WORK_BUFFER_HUNKS]; + work_item &item = m_work_item[m_write_hunk % m_window]; // free any OSD work item if (item.m_osd) @@ -3379,6 +3389,12 @@ std::error_condition chd_file_compressor::compress_continue(double &progress, do } else { + // Discpress: the ratio of the whole CHD, which chdman prints as the final one (0.289 left it as + // the last step before had it: in the browser, where helper workers' results come in batches, + // as little as 0%) + progress = 1.0; + ratio = (m_total_in == 0) ? 1.0 : double(m_total_out) / double(m_total_in); + // wait for all reads to finish and if we're compressed, write the final SHA1 and map osd_work_queue_wait(m_read_queue, 30 * osd_ticks_per_second()); if (!compressed()) @@ -3400,10 +3416,10 @@ std::error_condition chd_file_compressor::compress_continue(double &progress, do // if we're waiting for work, wait // sometimes code can get here with .m_status == WS_READY and .m_osd != nullptr, TODO find out why this happens - while ((m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_status != WS_READY) && - (m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_status != WS_COMPLETE) && - m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_osd) - osd_work_item_wait(m_work_item[m_write_hunk % WORK_BUFFER_HUNKS].m_osd, osd_ticks_per_second()); + while ((m_work_item[m_write_hunk % m_window].m_status != WS_READY) && + (m_work_item[m_write_hunk % m_window].m_status != WS_COMPLETE) && + m_work_item[m_write_hunk % m_window].m_osd) + osd_work_item_wait(m_work_item[m_write_hunk % m_window].m_osd, osd_ticks_per_second()); return m_walking_parent ? error::WALKING_PARENT : error::COMPRESSING; } @@ -3564,7 +3580,7 @@ void chd_file_compressor::async_read() return; // determine parameters for the read - uint32_t const work_buffer_bytes = WORK_BUFFER_HUNKS * hunk_bytes(); + uint32_t const work_buffer_bytes = m_window * hunk_bytes(); uint32_t numbytes = work_buffer_bytes / 2; if ((m_read_done_offset + numbytes) > logical_bytes()) numbytes = logical_bytes() - m_read_done_offset; @@ -3598,13 +3614,29 @@ void chd_file_compressor::async_read() { // otherwise, call the virtual function read_data(dest, m_read_done_offset, numbytes); +#ifdef __EMSCRIPTEN__ + // Discpress: a partial last hunk is compressed whole, and past the end of the data it holds + // what the work buffer held there before: with a wider window than chdman's, other bytes than + // in chdman's buffer. So it gets chdman's: those of the hunk CHDMAN_WINDOW before it (still in + // the buffer, as the window is wider), or the zeros the buffer starts with when there is none + if ((m_window != CHDMAN_WINDOW) && (end_offset == logical_bytes()) && (logical_bytes() % hunk_bytes())) + { + uint64_t const last = logical_bytes() / hunk_bytes(); + uint32_t const tail = logical_bytes() % hunk_bytes(); + uint8_t *const slot = &m_work_buffer[(last % m_window) * hunk_bytes()]; + if (last >= CHDMAN_WINDOW) + memcpy(slot + tail, &m_work_buffer[((last - CHDMAN_WINDOW) % m_window) * hunk_bytes()] + tail, hunk_bytes() - tail); + else + memset(slot + tail, 0, hunk_bytes() - tail); + } +#endif } // spawn off work for each hunk for (uint64_t curoffs = m_read_done_offset; curoffs < end_offset; curoffs += hunk_bytes()) { uint32_t hunknum = curoffs / hunk_bytes(); - work_item &item = m_work_item[hunknum % WORK_BUFFER_HUNKS]; + work_item &item = m_work_item[hunknum % m_window]; assert(item.m_status == WS_READING); item.m_status = WS_QUEUED; item.m_hunknum = hunknum; diff --git a/engine/mame/src/lib/util/chd.h b/engine/mame/src/lib/util/chd.h index c10ee8d..50d0c64 100644 --- a/engine/mame/src/lib/util/chd.h +++ b/engine/mame/src/lib/util/chd.h @@ -623,7 +623,16 @@ class chd_file_compressor : public chd_file std::error_condition m_read_error; // error during reading, if any // work item thread +#ifdef __EMSCRIPTEN__ + // Discpress (browser build): room for up to 1024 hunks in flight, so that the batches helper workers + // get carry enough work for their messages (see m_window) + static constexpr int WORK_BUFFER_HUNKS = 1024; +#else static constexpr int WORK_BUFFER_HUNKS = 256; +#endif + // Discpress: chdman's own window, which the bytes past the end of a partial last hunk follow (async_read) + static constexpr uint32_t CHDMAN_WINDOW = 256; + uint32_t m_window = CHDMAN_WINDOW; // Discpress: hunks in the work buffer, at most WORK_BUFFER_HUNKS osd_work_queue * m_work_queue; // queue for doing work on other threads std::vector m_work_buffer; // buffer containing hunk data to work on std::vector m_compressed_buffer;// buffer containing compressed data diff --git a/tests/fuzz/host.cpp b/tests/fuzz/host.cpp index 8f9516b..3c8649f 100644 --- a/tests/fuzz/host.cpp +++ b/tests/fuzz/host.cpp @@ -78,7 +78,7 @@ bool deliver() // wasm/par_lib.js's functions, as the page answers them extern "C" { -int wasm_par_enabled(uint32_t, uint32_t, const uint32_t *, uint32_t) { return 0; } +int wasm_par_enabled(uint32_t, uint32_t, const uint32_t *, uint32_t, uint32_t) { return 0; } void wasm_par_submit(void *, const uint8_t *, uint32_t, uint32_t) { diff --git a/tests/fuzz/seeds.cpp b/tests/fuzz/seeds.cpp index 00bd3f4..d94a8c1 100644 --- a/tests/fuzz/seeds.cpp +++ b/tests/fuzz/seeds.cpp @@ -219,6 +219,13 @@ void chd_seeds() make_chd(name, { "createraw", "-i", s_tmp + "/raw.bin", "-hs", "4096", "-us", "512", "-np", "1", "-c", codec }); } make_chd("hd.chd", { "createhd", "-i", s_tmp + "/raw.bin", "-np", "1", "-c", "zlib" }); + // 300 and a half hunks: the browser build's wider window gives the partial last hunk's tail the bytes of + // the hunk 256 before it (chd_file_compressor::async_read), which AddressSanitizer watches here + bytes window; + for (int h = 0; h < 300; h++) + window = cat({ window, (h % 3) ? noise(4096, 100 + h) : prose(4096) }); + fuzzhost::write_file(s_tmp + "/window.bin", cat({ window, prose(2048) })); + make_chd("raw-window.chd", { "createraw", "-i", s_tmp + "/window.bin", "-hs", "4096", "-us", "512", "-np", "1", "-c", "zlib" }); fuzzhost::write_file(s_tmp + "/dvd.iso", iso(40)); make_chd("dvd.chd", { "createdvd", "-i", s_tmp + "/dvd.iso", "-np", "1" }); make_chd("dvd-zstd.chd", { "createdvd", "-i", s_tmp + "/dvd.iso", "-np", "1", "-c", "zstd" }); diff --git a/tests/ui/window.spec.js b/tests/ui/window.spec.js new file mode 100644 index 0000000..c272997 --- /dev/null +++ b/tests/ui/window.spec.js @@ -0,0 +1,62 @@ +// In the browser, chdman keeps up to 1024 small hunks in flight instead of 256 (chd_file_compressor::m_window), +// so that helper workers get batches large enough for their messages. A partial last hunk is compressed whole, and +// past the end of the data it holds what chdman's 256-hunk buffer would: the bytes of the hunk 256 before it, or +// zeros. Each case must give chdman 0.289's CHD byte for byte: fewer than 256 hunks, 256 to 1024 (a slot of the +// wider buffer not used yet), more than 1024 (one used before). +import fs from 'node:fs'; +import path from 'node:path'; +import { test, expect } from '../support/app.js'; +import { FIXTURES } from '../support/paths.js'; +import { nativeChdman, reference, sameVersion, sha1File } from '../support/native.js'; + +const dir = path.join(FIXTURES, 'gen', 'window'); + +// an image of 2048-byte sectors, an odd number of them, each different (text, noise and zeros by turns) +function image(name, sectors) { + const file = path.join(dir, name); + if (fs.existsSync(file) && fs.statSync(file).size === sectors * 2048) return 'gen/window/' + name; + fs.mkdirSync(dir, { recursive: true }); + const b = Buffer.alloc(sectors * 2048); + let x = sectors * 2654435761 >>> 0; + for (let s = 0; s < sectors; s++) { + const kind = s % 3, at = s * 2048; + for (let i = 0; i < 2048; i++) { + x = (Math.imul(x, 1103515245) + 12345) >>> 0; + b[at + i] = kind === 0 ? 97 + ((x >>> 16) % 7) : kind === 1 ? x >>> 24 : (i < 64 ? s & 255 : 0); + } + } + fs.writeFileSync(file, b); + return 'gen/window/' + name; +} + +const CASES = [ + ['under 256 hunks', 'w100.iso', 2 * 100 + 1], + ['256 to 1024 hunks', 'w600.iso', 2 * 600 + 1], + ['over 1024 hunks', 'w1500.iso', 2 * 1500 + 1], +]; + +for (const threads of [1, 4]) { + test(`a partial last hunk gives chdman's CHD, with any window (${threads} thread${threads > 1 ? 's' : ''})`, async ({ app, page }) => { + test.skip(!nativeChdman() || !sameVersion(), 'needs native chdman 0.289'); + test.setTimeout(300_000); + const files = CASES.map(([, name, sectors]) => image(name, sectors)); + await app.open({ settings: { threads } }); + await page.click('.tab[data-tab="cli"]'); + await app.add(files, { via: '#cliAdd' }); + await page.click('#cliEditToggle'); + for (const [what, name] of CASES) { + for (const codecs of [[], ['-c', 'zlib,huff']]) { + const out = `${name.replace('.iso', '')}${codecs.length ? '-fast' : ''}.chd`; + await page.locator('#cliText').fill(`chdman createdvd -i ${name} -o ${out} ${codecs.join(' ')}`.trim()); + await page.click('#cliRun'); + await expect(page.locator('#cliConsole')).toContainText(/\[exit code \d+/, { timeout: 120_000 }); + await expect(page.locator('#cliConsole')).toContainText('[exit code 0]'); + await expect(page.locator('#cliOuts')).toContainText(out); + const [dl] = await Promise.all([page.waitForEvent('download'), page.locator('#cliOuts button').first().click()]); + const p = test.info().outputPath(out); + await dl.saveAs(p); + expect(sha1File(p), `${what}${codecs.length ? ', ' + codecs[1] : ''}`).toBe(sha1File(reference('createdvd', `gen/window/${name}`, codecs))); + } + } + }); +} diff --git a/wasm/par_lib.js b/wasm/par_lib.js index 6f03f07..ad2f77f 100644 --- a/wasm/par_lib.js +++ b/wasm/par_lib.js @@ -1,10 +1,11 @@ addToLibrary({ - // flags: 1 = the deflate codec encodes with libdeflate (--libdeflate) - wasm_par_enabled: function (hunkbytes, unitbytes, comp, flags) { + // flags: 1 = the deflate codec encodes with libdeflate (--libdeflate); window: the hunks chdman keeps in + // flight, reading half of them at a time (chd_file_compressor::m_window) + wasm_par_enabled: function (hunkbytes, unitbytes, comp, flags, window) { if (!Module['parSetup']) return 0; var p = comp >>> 2; var list = [HEAPU32[p], HEAPU32[p + 1], HEAPU32[p + 2], HEAPU32[p + 3]]; - return Module['parSetup'](hunkbytes >>> 0, unitbytes >>> 0, list, flags >>> 0) ? 1 : 0; + return Module['parSetup'](hunkbytes >>> 0, unitbytes >>> 0, list, flags >>> 0, window >>> 0) ? 1 : 0; }, // codecs: the codec slots to try for this hunk, a bit each (the codec plan; 15 = all) wasm_par_submit: function (item, data, length, codecs) { From 210f414b65895bdd0451594935c209527a5628c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 04:30:12 +0000 Subject: [PATCH 2/4] Docs and a test: the window, the profile counters, step 3's measurements Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QUSh6AFtA1CAR66DNvjtty --- CLAUDE.md | 4 ++-- docs/chd/fork-plan.md | 22 +++++++++++++++++----- tests/ui/edge-cases.spec.js | 13 +++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2497a95..6bd593c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,7 +57,7 @@ When changing `app/`: `npm run dev-page` (in `tests/`) re-assembles `build/discp The build is reproducible: a clean build must produce a byte-identical `dist/discpress.html` (gzip uses `mtime=0`; `build.sh` fails if the SIMD and non-SIMD JS glue differ, and refuses any Emscripten but 6.0.10, commit `d6c521a7f…`, unless `EMSCRIPTEN_ANY=1`, for experiments only: such a build records no sources hash, so `check-dist.sh` and a release refuse its page). `.github/workflows/sanitizers.yml` runs `engine.spec.js` on the sanitized build (`scripts/build-sanitized.sh`) when `engine/`, `wasm/`, the spec, the fixtures or the suppressions change, and weekly; any sanitizer report fails a test, whatever chdman's exit code, except leaks unmodified chdman 0.289 has too (`tests/support/lsan.supp`, each with how that was checked; only MAME's work queue, `osdsync.cpp`, so far). `.github/workflows/fuzz.yml` fuzzes the engine (`scripts/fuzz.sh`, `tests/fuzz/`): CHD files, descriptors, compressed hunks and ECM data, with the browser-only code built in (`__EMSCRIPTEN__` defined, `tests/fuzz/host.cpp` doing the page's part), weekly for 30 minutes per fuzzer and for 5 when `engine/`, `wasm/` or the fuzzers change; the corpus is kept in the Actions cache, and each finding, once fixed, becomes an input in `tests/fuzz/regressions/` and, where it can be made from a fixture, a test in `engine.spec.js`. `.github/workflows/rebuild.yml` checks it on another machine: it rebuilds with emsdk 6.0.10 and requires the committed page byte for byte, when `engine/`, `wasm/`, `build.sh`, `dist/` or the scripts that hash and assemble the page change, weekly and on demand (a clean build takes about a minute on 4 cores; machine paths don't reach the wasm). `dist/discpress.html` is committed and is what releases ship, so **regenerate and commit it whenever `app/`, `wasm/`, `engine/` or `db/` change**: after `app/` or `db/`, `assemble.py` is enough; after `engine/`, `wasm/` or `build.sh`, only `./build.sh` will do. The page records the sources its wasm was built from: `build.sh` hashes them (`scripts/engine-sources.py`: the contents of `build.sh` and of `engine/` and `wasm/`'s files that git tracks or would (an untracked file it doesn't ignore counts too), without `*.md`, `engine/FILES` and the diff; paths sorted, no dates) into `build/engine-sources.sha256`, which `assemble.py` writes after `` and `extract-build.py` recovers, and `scripts/check-dist.sh` refuses a page whose hash isn't the current sources'. -To try the app, open `dist/discpress.html` directly in a browser (`file://` works). Setting `localStorage['chdman-web-debug'] = '{"stage":1}'` makes the worker copy every input into storage with async reads before running, and uses the main-thread CRC; `"stage":2` also makes those reads fail, so the page streams inputs to the worker (the iOS web view path, see below). For tests, `"identDelay"` and `"crcDelay"` (ms) slow identification and its checksum step, `"sinkMax"` (bytes) lowers that 64 MB limit, `"memMax"` (bytes) lowers the size above which a job whose results are kept in memory asks first, and `"failHelpers"` (`"one"`, `"all"`, `"always"`) makes helper workers trap as a miscompiling WebAssembly engine does (`"lie"`: the first one answers for other hunks than it was given). `"crcSlow"` (ms) delays the worker's checksum, `"shareMax"` (bytes) lowers `SHARE_MAX`, `"failStorage"` (`"lock"`, or anything else) makes the worker's storage fail as a locked or missing OPFS does, `"unzipFull"` fills the storage while a zip is unpacked, `"savePickerMin"` (bytes) lowers `SAVE_PICKER_MIN`, `"freeze"` (ms) keeps a conversion's job worker silent before chdman starts and `"stallMs"` lowers the no-progress watchdog's `STALL_MS`, and `"hosted"` makes the page act as the online version (service worker, manifest) without https. +To try the app, open `dist/discpress.html` directly in a browser (`file://` works). Setting `localStorage['chdman-web-debug'] = '{"stage":1}'` makes the worker copy every input into storage with async reads before running, and uses the main-thread CRC; `"stage":2` also makes those reads fail, so the page streams inputs to the worker (the iOS web view path, see below). For tests, `"identDelay"` and `"crcDelay"` (ms) slow identification and its checksum step, `"sinkMax"` (bytes) lowers that 64 MB limit, `"memMax"` (bytes) lowers the size above which a job whose results are kept in memory asks first, and `"failHelpers"` (`"one"`, `"all"`, `"always"`) makes helper workers trap as a miscompiling WebAssembly engine does (`"lie"`: the first one answers for other hunks than it was given). `"crcSlow"` (ms) delays the worker's checksum, `"shareMax"` (bytes) lowers `SHARE_MAX`, `"failStorage"` (`"lock"`, or anything else) makes the worker's storage fail as a locked or missing OPFS does, `"unzipFull"` fills the storage while a zip is unpacked, `"savePickerMin"` (bytes) lowers `SAVE_PICKER_MIN`, `"profile"` makes the job worker report in the job's log where its time went (reading, the engine, handing out hunks, taking results, waiting for helpers, and the helpers' own time), `"freeze"` (ms) keeps a conversion's job worker silent before chdman starts and `"stallMs"` lowers the no-progress watchdog's `STALL_MS`, and `"hosted"` makes the page act as the online version (service worker, manifest) without https. ## How the single file is assembled @@ -77,7 +77,7 @@ Consequences: app JS is plain browser script (no modules/imports, ES5-style `var - **`app/ui.js`** (main thread, one IIFE): job model (files grouped into jobs by descriptor `.cue`/`.gdi`/etc.; a CloneCD `.ccd` is turned into a cue sheet for its `.img` by `ccdToCue`; a `.cso`/`.zso` becomes an input flagged `ciso`, given to chdman as `.iso`; an ECM image `.ecm` stands for ``, the track file a descriptor names or a lone `.bin` (`trackFile`), as an input flagged `ecm`; a PS1 `EBOOT.PBP` makes a job per disc, `sniffPbp` reading each disc's TOC into a cue sheet and the disc's image an input flagged `pbp`; a `.zip`'s entries join the list as picked files, `zipEntries`, stored ones as slices of it and deflated ones unpacked by `Unzip` into OPFS `chdman-unzip/` (else memory up to `UNZIP_MEM_MAX`) and checked against its CRC-32; formats it can't read (`UNSUPPORTED`: `.cdi`, `.mds`/`.mdf`, `.isz`, `.7z`, `.rar`) get a card saying why), building chdman argument lists per job (`buildJob`; the type and hunk size an identified console needs, the Zstd and MiSTer warnings (`noZstd`: its readers that can't decode Zstandard, for every preset that uses it; `noMister`: MiSTer has no core for it, for the MiSTer preset) and the notes for consoles whose emulators or players don't load CHDs (`keep`: the system doesn't set the CHD type) and the DOSBox Pure hint come from `PROFILES`, applied by `applyIdent`; Settings → Compatibility mode for older emulators (`settings.compat`, which a stored `ps2dvd: 'cd'` of 1.8 and earlier turns on; `COMPAT_NAME`) makes a PS2 game on DVD a CD CHD, the only kind AetherSX2, NetherSX2 and PCSX2 builds from before April 2023 read, and makes the Zstd presets warn for every console (`ZSTD_OLD`); rules for particular consoles and discs are `QUIRKS` in `app/quirks.js` (each with a stable id, shown as `data-quirk` on its note; `act: 'nokeepcue'` makes `buildJob` leave out `--keepcue`, as for Saturn and Sega CD, whose Kronos, Yabause and jgenesis reject it; `act: 'norename'` keeps the file's name, as for NAOMI romsets, through `renameWanted`; `warn` shows a note in the warning colour, and more than two other notes are grouped), looking at `quirkCtx` (the identification, `discModel`: the cue sheet's or TOC's layout from `cueModel`/`tocModel` in `ident.js`: tracks, sessions, INDEX 00, PREGAP and POSTGAP lines); a manifest entry's `quirks` lists the rule ids its card must show (`convert.spec.js`); compression never changes by itself; a PAL PS1 disc's `.sbi` (or DuckStation's `.lsd`) is paired with its job by name, `sbiJob`, and saved next to the CHD under its name, or made from a CloneCD `.sub`'s Q subchannel (`scanSub`, `sbiFromSub`), and is left out again when the disc turns out to be another console's (`detachSbi`); a LibCrypt game (`GameFacts`, from `db/facts/libcrypt.tsv`) without one warns (`libcrypt-missing`); a finished disc of a set says how many discs the set has (`GameDB.discCount`; `data-quirk="set-incomplete"`, set by `renderResult`, not a `QUIRKS` rule) until all are converted; two or more "(Disc N)" results of one game get an `.m3u` on the first disc's card, `discSet`), the Advanced tab CLI builder (`CMDS`/`OPT`), OPFS bookkeeping (`Store`), settings, and the `Engine` that runs commands. `Engine.run` starts one **job worker** per chdman command plus N **helper workers** connected to it by `MessageChannel`s. N comes from the thread setting: `'auto'` (default) uses `Tuning`, a one-off per-device speed test (helpers compress synthetic DVD hunks at 1, 2, 3, 4, 6, 8… threads until a step adds under 8%, twice below the reported core count or once above it; cached in `localStorage['chdman-web-tuning']` keyed by core count + user agent; up to 8 threads on touch devices (4 on iPhones), 16 elsewhere, regardless of the reported core count), run before the first compressing job; `threadCount()` resolves the setting. A finished CHD can be read back (`verifyResult`: chdman `verify` on the result, the card's Verify button), after every conversion with the `verifyAfter` setting and by itself after one during which a helper failed or the job ran again. A job whose results are kept in memory (no OPFS) asks before it runs when they could be over 2 GB (1 GB on touch devices; `memoryNeed`/`confirmMemory`, in the Advanced tab too). Where the browser has `showSaveFilePicker` (Chrome, Edge), Save asks where to put a result of 1 GB or more (`savePicked`, `SAVE_PICKER_MIN`), so a save canceled there keeps it unsaved; a plain download counts as saved once started. A started job can be canceled or removed before chdman runs (loading, identification, the speed test: `abortStart`), and a helper that fails to start fails the job instead of leaving it waiting. A running conversion the page hears nothing from for 90 s while visible gets a note saying it may be stuck, until it goes on (`Engine.run`'s watchdog, `STALL_MS`, `onStall`; a line in the Advanced tab). - **Input guard rails** (`app/ui.js`): chdman 0.289 reads descriptors byte for byte with case-sensitive keywords, and loops forever when it finds no tracks. So `descriptorJob` decodes a descriptor itself (`decodeText`: BOM, UTF-16, Windows-1252), and `fixDescriptor` either gives chdman a corrected copy (LF line ends, cue keywords upper-cased) or refuses the job (`job.invalid`: no tracks, track numbers outside 1–99, CD+G tracks, which 0.289 rejects). The engine reads TOC files with cdrdao's grammar (a missing length is the rest of the file) and Nero images' stored pregaps; `probePlan` in `ident.js` finds data tracks the same way. File names in descriptors are split into words as chdman does (`chdWords`: single and double quotes); typed Advanced tab commands as a shell does (`shellWords`). Binary `.toc` files are skipped, raw-sector `.iso` files get an `autoCue`, and a `nan% complete` progress line stops the run (`STALLED`), in the Advanced tab too. A descriptor takes each track from its own folder, the subfolder its reference names, or a file picked on its own, never a same-named file from another folder or one already claimed (`pickTrack`: every GDI has `track01.bin`); names are compared in NFC (`nameKey`: a macOS name's é is two characters), through one index per batch of added files (`nameIndex`), so a folder of thousands of games is grouped at once; the duplicate check keys files by path, size and date (`fileKey`). - **`app/worker.js`** (Web Worker, appended to the Emscripten glue): message roles `run` (run a chdman command line with `chdman_begin`/`chdman_resume`), `helper` (compress or decompress hunks for the job worker), `reader` (open a CHD and serve sectors for identification) and `crc`. It mounts a custom Emscripten filesystem (`makeFS`) over storage backends: `BlobStore` (read inputs in place via `FileReaderSync`), `OpfsStore` (outputs to Origin Private File System), `MemStore` (fallback), `CisoStore` (a CSO v1/v2 or ZSO compressed ISO read as the ISO inside, over any of the others: deflate blocks through the wasm's `wasm_inflate_raw`, LZ4 in JS; also used by the `reader` and `crc` roles to identify it), `PbpStore` (a PS1 PBP's disc read as its raw image: 16-sector blocks, deflated through `wasm_inflate_raw` or stored, from the disc's block table), `EcmStore` (an ECM image read as the CD image inside: a scan of its chunk headers places a checkpoint every 1 MiB of the image, and each window is rebuilt, sync, headers, EDC and ECC, by `wasm_ecm_decode` in `wasm/ecm.cpp`; read in order to its end, the image's EDC must match the file's, else the job stops; the scan reports how far it got, `scan` messages, which the card shows and which keep identification's reader from timing out; the `reader` role serves ECM images' bytes for identification, the `crc` role their checksum), `CrcStore` (outMode `crc`: outputs only checksummed, which `redumpCheck` in `ui.js` uses after Verify to compare a CHD with Redump via `extractcd --redump`), and `StreamStore` (outputs streamed to a user-chosen folder: `s-*` messages handled by `makeSink` in `ui.js` via the File System Access API; a file already in the folder is replaced only after `confirmReplace` asks, and is never deleted on cancel or failure; the page acknowledges each write (`s-ack`), and while more than 64 MB is unwritten the worker pauses chdman at its pause points, via the browser-only engine hook `wasm_out_full`). When a worker can't read picked files (some iOS file-viewer web views), it sends `stage-request` and the page streams chunks (`stage-chunk`/`stage-ack`/`stage-end`). -- **Multi-core compression** doesn't use pthreads/SharedArrayBuffer (so no COOP/COEP headers are needed). The engine has hooks in `chd_file_compressor` (`engine/mame/src/lib/util/chd.cpp`); `wasm/par_lib.js` (Emscripten JS library) hands hunks to `Module.parSubmit`, each with the codec slots to try (all of them, unless `--codecplan`, the opt-in "Nearly as small, faster" preset, narrows them by what the hunk holds: `chd_file_compressor::codec_plan_codecs`; with `--libdeflate`, in both faster presets, the deflate codec uses `engine/libdeflate` and `wasm_par_enabled` tells the helpers), and helper workers run a second wasm instance whose `wasm_helper_*` exports (`wasm/wasm_helper.cpp`) compress hunks exactly as `chd_file_compressor` would. The job worker keeps each batch until a helper answers it, and takes an answer only for that batch, item by item, each no longer than a hunk (else the helper is dropped): a helper that fails (a WebAssembly trap; some WebKit versions' optimizing compiler miscompiles, `docs/ios/README.md`) is dropped and the others do its batches; with none left the page runs the job once more (`engineFailure`), then says to update the browser (an extract or verify job decompresses by itself instead). The commands that compress are C++20 coroutines (`chdman_task` in `chdman.cpp`, no Asyncify): the worker starts a command with `chdman_begin` and, while it returns -1 (paused after a compression step), awaits `Module.parWait()` and calls `chdman_resume`; natively `main` runs them straight through. New exported C functions must be added to `EXPORTED_FUNCTIONS` in `wasm/link.sh`. +- **Multi-core compression** doesn't use pthreads/SharedArrayBuffer (so no COOP/COEP headers are needed). The engine has hooks in `chd_file_compressor` (`engine/mame/src/lib/util/chd.cpp`); `wasm/par_lib.js` (Emscripten JS library) hands hunks to `Module.parSubmit`, each with the codec slots to try (all of them, unless `--codecplan`, the opt-in "Nearly as small, faster" preset, narrows them by what the hunk holds: `chd_file_compressor::codec_plan_codecs`; with `--libdeflate`, in both faster presets, the deflate codec uses `engine/libdeflate` and `wasm_par_enabled` tells the helpers), and helper workers run a second wasm instance whose `wasm_helper_*` exports (`wasm/wasm_helper.cpp`) compress hunks exactly as `chd_file_compressor` would. In the browser the compressor keeps up to 1024 hunks of 4 KiB or less in flight (`m_window`, chdman's 256 for larger hunks and with a parent), and the page sizes its batches from that window (`wasm_par_enabled`'s `window`, up to 64 hunks); chdman's window is part of its output, since a partial last hunk is compressed with what the buffer held past the end of the data, so the engine gives that tail the bytes chdman's 256-hunk buffer would (`tests/ui/window.spec.js`). The job worker keeps each batch until a helper answers it, and takes an answer only for that batch, item by item, each no longer than a hunk (else the helper is dropped): a helper that fails (a WebAssembly trap; some WebKit versions' optimizing compiler miscompiles, `docs/ios/README.md`) is dropped and the others do its batches; with none left the page runs the job once more (`engineFailure`), then says to update the browser (an extract or verify job decompresses by itself instead). The commands that compress are C++20 coroutines (`chdman_task` in `chdman.cpp`, no Asyncify): the worker starts a command with `chdman_begin` and, while it returns -1 (paused after a compression step), awaits `Module.parWait()` and calls `chdman_resume`; natively `main` runs them straight through. New exported C functions must be added to `EXPORTED_FUNCTIONS` in `wasm/link.sh`. - **Multi-core extract and verify** work the same way in reverse. `verify` and the extract commands are coroutines too, and before each buffer's worth of reading they `co_await read_ahead(...)`. `chd_file::wasm_read_ahead` (`chd.cpp`, browser only) reads the stored data of that window and the next in one file read (small reads are expensive in the browser). It hands codec-compressed hunks to helpers (`wasm_rd_submit` → `Module.rdSubmit`; results come back through `wasm_rd_slot`/`wasm_rd_done`), and pauses the command until the window is in (`Module.rdWait()`). `read_hunk` then serves hunks from memory with chdman's usual CRC check, so errors are unchanged. With one thread it keeps the window's compressed bytes and decompresses them when read. Verify and extract jobs get `readHelpers()`: the thread setting, but at most one fewer than the cores, since the job worker stays busy. - **Game identification** (`app/ident.js`, runs on the main thread): reads boot data (IP.BIN, SYSTEM.CNF, PARAM.SFO, …) via ISO 9660 from raw images, or from inside CHDs via the worker `reader` role backed by `wasm_probe_*` in `wasm_helper.cpp`; matches serial, or size + CRC-32 (computed in a worker with zlib's `crc32` exported from the wasm, JS fallbacks in the worker and on the page), against the database. `detectTrack` checks, in order: Sega headers, 3DO, GameCube/Wii, PC Engine and PC-FX boot sectors, CD-i's own volume descriptor, then the ISO 9660 volume (PSP and UMD Video, PS1/PS2 with `hasUdf` for a PS2 DVD, PS3, Video CD, CD-i Bridge, Neo Geo CD, CDTV/CD32, Xbox, video discs, else a weak `pc`), and without one an Xbox partition; PC-98 has no detector and is found by size + CRC-32 only; a cue sheet of audio tracks only is checked for a Jaguar CD header (`detectJaguar`, `plan.audio`). Serials are keyed per system by `canonKey` (Sega's `MK-51064` is Redump's `51064`), a disc is looked up under `serialCandidates` (its serial, without a version suffix, a PlayStation boot file's name), and `bySerial` in `identifyJob` ranks the releases found: the disc of a set Sega headers name (`det.disc`), then size, how the serial matched, and region (`det.area`, `regionFits`); a release found only under a trimmed or base serial, without a matching size, names nothing when its region doesn't fit, and when a Sega header's title doesn't fit it either it is only offered (`serial-related`, with Use this name: Saturn's T-25406H is Hexen, T-25406H-50 Doom); the ident note's `data-method`, `data-how` (the serial that found it: `serial`, `trim`, `boot` or `base`), `data-disc` and `data-region` say what it found. Removing a card stops its checksum (`abortChecksums`), so the next card's identification goes on. Identification is two-phase: once console/serial/sizes are known, `onProvisional` lets a conversion start (`job.identKnown`); the checksum then confirms the exact release and `renameOutputs` renames finished results if it differs (not when results were already saved, named by the user, or written straight into a folder, which waits for the checksum). `db/db.json.gz` is `{version, systems: {key: "name\tserial\tsize\tcrc\ttrack\text" rows}}`, a release listed under several serials once, with them joined by `|` (`entry.serials`; `entry.serial` is the first); system keys in `db/mkdb.py` must be keys of `SYSTEMS` in `ident.js` (which also has systems without database rows: `gc`, `wii`, `xbox`, `ps3`, `vcd`, `video`, `pc`). The user's own DAT files (Logiqx XML or clrmamepro, Redump or No-Intro) are parsed by `parseDat` into `Dats` and kept in OPFS `chdman-dats/` (`DatStore` in `ui.js`); `datCheck` matches every track of a job by size and CRC-32 (`datMatch`), a CHD's after Verify from `extractcd --redump`'s checksums, and a full match names the result (`datApply`). The rules in `quirks.js` add what common emulators can't load (PCSX2: PS2 CDs with audio; Flycast 2.7: CD-based Dreamcast discs with pregaps). - **`engine/`** (see `engine/README.md`): the chdman fork. `engine/mame` holds MAME 0.289's chdman and exactly the MAME sources it links (`engine/FILES`), at MAME's paths. Change them in place, then run `scripts/engine-diff.sh`: `engine/mame-0.289.diff` lists every change and is shown in Help → About. New code goes outside `engine/mame`. `scripts/fetch-mame.sh` fetches the unmodified release for the diff and for `build-upstream.sh`. diff --git a/docs/chd/fork-plan.md b/docs/chd/fork-plan.md index e867c6d..0fd4069 100644 --- a/docs/chd/fork-plan.md +++ b/docs/chd/fork-plan.md @@ -95,6 +95,8 @@ These rules come from the emulator survey (D1 §8, D2 §15, G §1). A Discpress | Deflate | raw deflate | any conforming encoder | | zstd | one frame per hunk, window ≤ 2²⁷ | level | +**chdman's window is part of its output.** A partial last hunk (a disc whose size isn't a whole number of hunks, as most CDs and many ISOs) is compressed whole, and past the end of the data it holds whatever chdman's work buffer of 256 hunks held at that place: the bytes of the hunk 256 before it, or zeros. The SHA-1s don't cover them, but the compressed bytes do. So a writer that keeps another number of hunks in flight must give that tail the same bytes, as the engine's wider browser window does (`m_window`, `tests/ui/window.spec.js`); the thread count doesn't matter. + **Identity.** Codec and hunk choices never change the CHD SHA-1. Metadata changes always do. Extra *non-checksummed* metadata doesn't change the SHA-1s, and most readers ignore it (C §2), but not all: Kronos and Yabause (Saturn) and jgenesis (Sega CD) refuse a CHD with any tag that isn't a track's (D2 §0). So the app leaves `--keepcue`, whose `CUES` tag is such metadata, out for those two consoles (`keepcue-strict` in `app/quirks.js`). ### Output tiers @@ -127,7 +129,7 @@ Effort estimates assume one developer working with the existing test harness. Th | 5. What users ask for | Done (1.5.0) | | 6. Detection that knows the game | Done (1.7.0) | | 7. Layouts each emulator reads right | Planned; nothing built (no new conversion engine, but steps 2 and 5 change the fork) | -| 8. Faster, more efficient, more reliable | Step 1 done (fuzzing and crafted headers found over 20 faults that damaged files reach, all but one also chdman 0.289's); step 2's workflows in place; steps 3 and 4 planned | +| 8. Faster, more efficient, more reliable | Step 1 done (fuzzing and crafted headers found over 20 faults that damaged files reach, all but one also chdman 0.289's); step 2's workflows in place, with first results; step 3 in progress (a wider compression window: DVDs 11–20% faster); step 4 planned | The two releases after milestone 6 are outside the milestones, and their CHDs are the same: - **1.8.0** fixed what a full review of the app found: discs that took another game's tracks or name, results that could be lost, slow adding of big folders. @@ -451,10 +453,20 @@ Where the time goes, measured on the benchmark discs (300 MB CD, 1 GB DVD; 4 cor 2. **Measure on real browsers** (workflows in place). `.github/workflows/bench.yml` times the page in Chromium and Firefox (Ubuntu) and WebKit (an Apple-silicon Mac). Whether Firefox's or Safari's numbers come before step 3 is decided from its results. -3. **Speed with the same bytes** (planned). - - The job worker's serial path first: what "Faster to create" waits on (11.2 s against native's 4.5 on a - DVD), and extract's and verify's, profiled with counters the page reads in a debug build. It reads the - input, keeps the running SHA-1 of all of it, hands hunks to the helpers and writes their results. +3. **Speed with the same bytes** (in progress). + - The job worker's serial path, profiled (`DEBUG.profile`): of 13.2 s for "Faster to create" on the 1 GB + DVD at 4 threads, the job worker spent 3.8 s reading the input, 2.5 s in the engine itself (mostly the + running SHA-1), 0.6 s handing out hunks and 0.8 s taking results, and waited the rest; the helpers were + busy 21 s of the 46 they had. Reading faster didn't help: reading ahead asynchronously, or in 4 MB + blocks, cut the read time to 1.6 s but made extracting 7–26% slower and creating no faster. The helpers + waited on messages: 16 hunks a batch (chdman's 256 in flight, read 128 at a time) are 1.2 ms of work + for `zlib,huff`. Done: in the browser the compressor keeps up to 1024 hunks of 4 KiB or less in + flight, and batches grow to 64 hunks (`m_window`, engine/README.md). Measured, median of three: + "Faster to create" on the DVD 12.99 → 10.36 s, the default 50.42 → 44.67 s, CDs (whose hunks keep 256) + the same; byte for byte chdman's, which took giving a partial last hunk's tail the bytes chdman's own + buffer holds there (§3, "chdman's window is part of its output"). Left: the running SHA-1 and the + copies, about 3 s of the job worker's time per GB, and Firefox, whose 4 threads gain less (CI's + benchmark: 14.6 → 11.6 s, against Chromium's 13.7 → 8.7). - Exact SIMD for FLAC's loops: Emscripten already vectorizes the two largest, autocorrelation (373 SIMD instructions in the build) and the residual from the LPC coefficients (349), without changing a result (each lag's sum keeps its order; WebAssembly SIMD has no fused multiply-add). What is left is mostly diff --git a/tests/ui/edge-cases.spec.js b/tests/ui/edge-cases.spec.js index b47e4be..e73c926 100644 --- a/tests/ui/edge-cases.spec.js +++ b/tests/ui/edge-cases.spec.js @@ -802,3 +802,16 @@ test('the no-progress watchdog doesn\u2019t count time the page is hidden', asyn await expect(card.locator('[data-note="no-progress"]')).toHaveCount(0); await app.waitState(card, 'done', 60_000); }); + +// DEBUG.profile: the job worker says in the job's log where its time went, the helpers' too +test('with the profile debug setting, the log says where the job worker’s time went', async ({ app }) => { + await app.open({ settings: { threads: 2 }, debug: { profile: 1 } }); + await app.add(fixture('ps2-dvd').add); + const card = app.jobs().first(); + await app.settled(card); + await app.run(card); + const log = await card.locator('pre.logtext').textContent(); + const line = /Profile \(ms\): (.*)/.exec(log); + expect(line, log.slice(-2000)).not.toBeNull(); + for (const what of ['read input', 'in the engine', 'hand out hunks', 'take results', 'waiting for helpers', 'helpers compressing', 'helpers busy']) expect(line[1]).toMatch(new RegExp(`${what} \\d+`)); +}); From 15dd585f114cb9473d98142215e5b076d057527b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 04:48:53 +0000 Subject: [PATCH 3/4] Build: WASM_EH=exnref, and a CI job that runs the conversions on that build WebAssembly's new exception format (exnref, -sWASM_LEGACY_EXCEPTIONS=0), with its own objects; like EMSCRIPTEN_ANY's, its page records no sources, so check-dist.sh and a release refuse it. rebuild.yml's second job builds it and runs the conversion tests on it: chdman's CHDs byte for byte, at the same speed. The default build is unchanged (the same wasm and glue from a clean build; the page's hash moves because build.sh is a hashed source). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QUSh6AFtA1CAR66DNvjtty --- .github/workflows/rebuild.yml | 61 ++++++++++++++++++++++++++++++++++- CLAUDE.md | 4 +-- build.sh | 23 +++++++++---- dist/discpress.html | 2 +- docs/chd/fork-plan.md | 11 ++++--- 5 files changed, 87 insertions(+), 14 deletions(-) diff --git a/.github/workflows/rebuild.yml b/.github/workflows/rebuild.yml index f99ca72..c89c1b3 100644 --- a/.github/workflows/rebuild.yml +++ b/.github/workflows/rebuild.yml @@ -2,7 +2,8 @@ # page comes out byte for byte the dist/discpress.html in the repository, the file releases ship. # scripts/check-dist.sh (run by CI and by every release) checks that the page records the sources it # was built from, a hash build.sh writes itself; this checks the build itself, on another machine. -# Runs when the engine, its build or the committed page changes, weekly, and on demand. +# Runs when the engine, its build or the committed page changes, weekly, and on demand. A second job builds +# the same sources with WebAssembly's new exception format (exnref) and runs the conversion tests on it. name: Rebuild on: @@ -63,3 +64,61 @@ jobs: build/chdman-nosimd.wasm build/chdman.js retention-days: 14 + + # The same sources with WebAssembly's new exception format (exnref, WASM_EH=exnref in build.sh), so that + # switching to it, once every browser the page supports has it, is a known quantity: the conversions + # must still give native chdman's CHDs byte for byte. Its page is never released (it records no sources). + exnref: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - name: Cache Emscripten 6.0.10 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/emsdk + key: emsdk-6.0.10-${{ runner.os }} + - name: Emscripten 6.0.10 (emsdk) + run: | + [ -d ~/emsdk/.git ] || git clone --depth 1 https://github.com/emscripten-core/emsdk.git ~/emsdk + cd ~/emsdk + ./emsdk install 6.0.10 + ./emsdk activate 6.0.10 + - name: Build with exnref exceptions + run: | + source ~/emsdk/emsdk_env.sh + WASM_EH=exnref ./build.sh + ~/emsdk/upstream/bin/llvm-objdump -d build/chdman.wasm | grep -q try_table + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: tests/package-lock.json + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + - name: numpy, for the synthetic test discs + run: pip install numpy==2.4.6 + - name: Cache MAME 0.289 and its chdman + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + third_party/mame + build/chdman-0.289 + key: mame-0.289-${{ runner.os }}-${{ hashFiles('scripts/fetch-mame.sh', 'scripts/build-upstream.sh', 'wasm/**') }} + - name: Native chdman (0.289 and the engine) + run: | + [ -x build/chdman-0.289 ] || ./scripts/build-upstream.sh + ./scripts/build-native.sh + - name: Playwright and Chromium + working-directory: tests + run: | + npm ci --no-audit --no-fund + npx playwright install --with-deps chromium + - name: Synthetic test discs + run: python3 tests/fixtures/make_fixtures.py + - name: Conversions on the exnref page + working-directory: tests + env: + REQUIRE_NATIVE: '1' + run: npx playwright test ui/smoke.spec.js ui/convert.spec.js ui/window.spec.js ui/fallbacks.spec.js --project=desktop diff --git a/CLAUDE.md b/CLAUDE.md index b1faa50..5a614c0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,7 @@ Research for the chdman fork (CHD format, chdman internals, disc layouts, emulat # one-time: Emscripten 6.0.10 (emsdk); the engine source is in engine/ (no MAME checkout needed) # full build: compiles SIMD and non-SIMD wasm from engine/, links, then assembles source ~/emsdk/emsdk_env.sh -./build.sh # -> build/ (gitignored) and dist/discpress.html; JOBS= optional +./build.sh # -> build/ (gitignored) and dist/discpress.html; JOBS= optional; WASM_EH=exnref: the new exception format, for trying (never released) # after changing only app/ or db/: no Emscripten needed python3 scripts/extract-build.py # once: recovers build/chdman.js + both .wasm (and the engine sources' hash) from dist/discpress.html @@ -55,7 +55,7 @@ Compiler experiments: `make -C wasm ... EXTRA=""` adds flags to every com When changing `app/`: `npm run dev-page` (in `tests/`) re-assembles `build/discpress-dev.html` in under a second (first taking the wasm from `dist/discpress.html` when that is newer than `build/`'s and differs: `extract-build.py --if-stale`, which leaves a fresher `./build.sh` output alone), `npm run test:dev` runs the suite on it, and `npm run bench:dev -- --compare latest` checks performance. Tests fail on any console error or network request. Review UI changes with the screenshots the layout spec writes to `tests/.cache/screens//-.png`. Once satisfied, `python3 scripts/assemble.py` updates `dist/discpress.html`, which must be committed with the source change. `.github/workflows/browsers.yml` runs the suite in Firefox and WebKit too (`EXTRA_BROWSERS=1`, the `firefox`, `webkit` and `iphone-webkit` projects) when the page or the tests change, and weekly. CI (`.github/workflows/ci.yml`) runs `check-dist.sh`, `check-licenses.sh` (only BSD-3-Clause MAME files in `engine/`, `md5.h` public domain) `engine-diff.sh --check` and the whole suite with `REQUIRE_NATIVE=1` on every pull request and push to `main`; locally, set `REQUIRE_NATIVE=1` too, since without `build/chdman-0.289` and a `build/chdman-native` newer than `engine/` the byte-for-byte checks and `engine.spec.js` skip. Bugs found but not fixed yet are pinned with `test.fail()` in the relevant spec, or in the `ACCEPTED` list of `tests/ui/a11y.spec.js`; remove the marker when fixing one. Fixtures are synthetic (`tests/fixtures/`); never add real game data to the repo. -The build is reproducible: a clean build must produce a byte-identical `dist/discpress.html` (gzip uses `mtime=0`; `build.sh` fails if the SIMD and non-SIMD JS glue differ, and refuses any Emscripten but 6.0.10, commit `d6c521a7f…`, unless `EMSCRIPTEN_ANY=1`, for experiments only: such a build records no sources hash, so `check-dist.sh` and a release refuse its page). `.github/workflows/sanitizers.yml` runs `engine.spec.js` on the sanitized build (`scripts/build-sanitized.sh`) when `engine/`, `wasm/`, the spec, the fixtures or the suppressions change, and weekly; any sanitizer report fails a test, whatever chdman's exit code, except leaks unmodified chdman 0.289 has too (`tests/support/lsan.supp`, each with how that was checked; only MAME's work queue, `osdsync.cpp`, so far). `.github/workflows/fuzz.yml` fuzzes the engine (`scripts/fuzz.sh`, `tests/fuzz/`): CHD files, descriptors, compressed hunks and ECM data, with the browser-only code built in (`__EMSCRIPTEN__` defined, `tests/fuzz/host.cpp` doing the page's part), weekly for 30 minutes per fuzzer and for 5 when `engine/`, `wasm/` or the fuzzers change; the corpus is kept in the Actions cache, and each finding, once fixed, becomes an input in `tests/fuzz/regressions/` and, where it can be made from a fixture, a test in `engine.spec.js`. `.github/workflows/rebuild.yml` checks it on another machine: it rebuilds with emsdk 6.0.10 and requires the committed page byte for byte, when `engine/`, `wasm/`, `build.sh`, `dist/` or the scripts that hash and assemble the page change, weekly and on demand (a clean build takes about a minute on 4 cores; machine paths don't reach the wasm). `dist/discpress.html` is committed and is what releases ship, so **regenerate and commit it whenever `app/`, `wasm/`, `engine/` or `db/` change**: after `app/` or `db/`, `assemble.py` is enough; after `engine/`, `wasm/` or `build.sh`, only `./build.sh` will do. The page records the sources its wasm was built from: `build.sh` hashes them (`scripts/engine-sources.py`: the contents of `build.sh` and of `engine/` and `wasm/`'s files that git tracks or would (an untracked file it doesn't ignore counts too), without `*.md`, `engine/FILES` and the diff; paths sorted, no dates) into `build/engine-sources.sha256`, which `assemble.py` writes after `` and `extract-build.py` recovers, and `scripts/check-dist.sh` refuses a page whose hash isn't the current sources'. +The build is reproducible: a clean build must produce a byte-identical `dist/discpress.html` (gzip uses `mtime=0`; `build.sh` fails if the SIMD and non-SIMD JS glue differ, and refuses any Emscripten but 6.0.10, commit `d6c521a7f…`, unless `EMSCRIPTEN_ANY=1`, for experiments only: such a build records no sources hash, so `check-dist.sh` and a release refuse its page). `.github/workflows/sanitizers.yml` runs `engine.spec.js` on the sanitized build (`scripts/build-sanitized.sh`) when `engine/`, `wasm/`, the spec, the fixtures or the suppressions change, and weekly; any sanitizer report fails a test, whatever chdman's exit code, except leaks unmodified chdman 0.289 has too (`tests/support/lsan.supp`, each with how that was checked; only MAME's work queue, `osdsync.cpp`, so far). `.github/workflows/fuzz.yml` fuzzes the engine (`scripts/fuzz.sh`, `tests/fuzz/`): CHD files, descriptors, compressed hunks and ECM data, with the browser-only code built in (`__EMSCRIPTEN__` defined, `tests/fuzz/host.cpp` doing the page's part), weekly for 30 minutes per fuzzer and for 5 when `engine/`, `wasm/` or the fuzzers change; the corpus is kept in the Actions cache, and each finding, once fixed, becomes an input in `tests/fuzz/regressions/` and, where it can be made from a fixture, a test in `engine.spec.js`. `.github/workflows/rebuild.yml` checks it on another machine: it rebuilds with emsdk 6.0.10 and requires the committed page byte for byte, when `engine/`, `wasm/`, `build.sh`, `dist/` or the scripts that hash and assemble the page change, weekly and on demand (a clean build takes about a minute on 4 cores; machine paths don't reach the wasm). Its second job builds with WebAssembly's new exception format (`WASM_EH=exnref`, `-sWASM_LEGACY_EXCEPTIONS=0`; such a page records no sources) and runs the conversion tests on it, so that switching is known to give the same CHDs (measured: the same speed; a switch also needs `EH_TEST` in `app/ui.js` to test for exnref). `dist/discpress.html` is committed and is what releases ship, so **regenerate and commit it whenever `app/`, `wasm/`, `engine/` or `db/` change**: after `app/` or `db/`, `assemble.py` is enough; after `engine/`, `wasm/` or `build.sh`, only `./build.sh` will do. The page records the sources its wasm was built from: `build.sh` hashes them (`scripts/engine-sources.py`: the contents of `build.sh` and of `engine/` and `wasm/`'s files that git tracks or would (an untracked file it doesn't ignore counts too), without `*.md`, `engine/FILES` and the diff; paths sorted, no dates) into `build/engine-sources.sha256`, which `assemble.py` writes after `` and `extract-build.py` recovers, and `scripts/check-dist.sh` refuses a page whose hash isn't the current sources'. To try the app, open `dist/discpress.html` directly in a browser (`file://` works). Setting `localStorage['chdman-web-debug'] = '{"stage":1}'` makes the worker copy every input into storage with async reads before running, and uses the main-thread CRC; `"stage":2` also makes those reads fail, so the page streams inputs to the worker (the iOS web view path, see below). For tests, `"identDelay"` and `"crcDelay"` (ms) slow identification and its checksum step, `"sinkMax"` (bytes) lowers that 64 MB limit, `"memMax"` (bytes) lowers the size above which a job whose results are kept in memory asks first, and `"failHelpers"` (`"one"`, `"all"`, `"always"`) makes helper workers trap as a miscompiling WebAssembly engine does (`"lie"`: the first one answers for other hunks than it was given). `"crcSlow"` (ms) delays the worker's checksum, `"shareMax"` (bytes) lowers `SHARE_MAX`, `"failStorage"` (`"lock"`, or anything else) makes the worker's storage fail as a locked or missing OPFS does, `"unzipFull"` fills the storage while a zip is unpacked, `"savePickerMin"` (bytes) lowers `SAVE_PICKER_MIN`, `"profile"` makes the job worker report in the job's log where its time went (reading, the engine, handing out hunks, taking results, waiting for helpers, and the helpers' own time), `"storeDelay"` (ms) slows the page's check for private storage (`Store.ready`, which jobs wait for), `"freeze"` (ms) keeps a conversion's job worker silent before chdman starts and `"stallMs"` lowers the no-progress watchdog's `STALL_MS`, and `"hosted"` makes the page act as the online version (service worker, manifest) without https. diff --git a/build.sh b/build.sh index 8ef8942..8249725 100755 --- a/build.sh +++ b/build.sh @@ -6,7 +6,10 @@ # can give other WebAssembly bytes, so the build refuses it (EMSCRIPTEN_ANY=1 allows it, for experiments: # the page it makes is not a release build). # -# Environment: JOBS (default: CPU count) +# Environment: JOBS (default: CPU count); WASM_EH=exnref builds with WebAssembly's new exception format +# (exnref: Chrome 137+, Firefox 131+, Safari 18.4+) instead of the legacy one, to try it before switching +# (which also needs app/ui.js's EH_TEST to test for it). Like EMSCRIPTEN_ANY's, that page records no +# sources, so scripts/check-dist.sh and a release refuse it. set -euo pipefail ROOT="$(cd "$(dirname "$0")" && pwd)" OUT="$ROOT/build" @@ -28,6 +31,12 @@ if [ -z "$pinned" ] && [ -z "${EMSCRIPTEN_ANY:-}" ]; then exit 1 fi JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)}" +# (its own objects: the Makefile doesn't rebuild what other flags compiled) +case "${WASM_EH:-legacy}" in + legacy) EH=; EHO= ;; + exnref) EH=-sWASM_LEGACY_EXCEPTIONS=0; EHO=-exnref ;; + *) echo "WASM_EH: legacy or exnref, not $WASM_EH" >&2; exit 1 ;; +esac mkdir -p "$OUT" # The sources this wasm is built from, which the page records for scripts/check-dist.sh: hashed before # compiling, written to build/ only once the build succeeded. @@ -35,15 +44,15 @@ SOURCES="$(python3 "$ROOT/scripts/engine-sources.py")" rm -f "$OUT/engine-sources.sha256" echo "==> Compiling (SIMD)" -make -s -C "$ROOT/wasm" -j"$JOBS" T=wasm O="$OUT/obj-wasm" objs +make -s -C "$ROOT/wasm" -j"$JOBS" T=wasm O="$OUT/obj-wasm$EHO" EXTRA="${EXTRA:-} $EH" objs echo "==> Compiling (no SIMD, for older devices)" -make -s -C "$ROOT/wasm" -j"$JOBS" T=wasm SIMD= O="$OUT/obj-wasm-nosimd" objs +make -s -C "$ROOT/wasm" -j"$JOBS" T=wasm SIMD= O="$OUT/obj-wasm-nosimd$EHO" EXTRA="${EXTRA:-} $EH" objs echo "==> Linking" W="-sMODULARIZE=1 -sEXPORT_NAME=createChdman -sENVIRONMENT=web,worker -sINVOKE_RUN=0 -sFORCE_FILESYSTEM=1" cd "$OUT" -OBJDIR=obj-wasm "$ROOT/wasm/link.sh" chdman.js $W -OBJDIR=obj-wasm-nosimd "$ROOT/wasm/link.sh" nosimd-tmp.js $W +OBJDIR=obj-wasm$EHO "$ROOT/wasm/link.sh" chdman.js $W $EH +OBJDIR=obj-wasm-nosimd$EHO "$ROOT/wasm/link.sh" nosimd-tmp.js $W $EH mv nosimd-tmp.wasm chdman-nosimd.wasm # Both variants share one JS glue file, so it must be identical. if ! diff <(sed 's/nosimd-tmp/chdman/g' nosimd-tmp.js) chdman.js >/dev/null; then @@ -52,8 +61,10 @@ if ! diff <(sed 's/nosimd-tmp/chdman/g' nosimd-tmp.js) chdman.js >/dev/null; the fi rm -f nosimd-tmp.js # Another Emscripten's page records no sources: scripts/check-dist.sh and a release refuse it -if [ -n "$pinned" ]; then +if [ -n "$pinned" ] && [ -z "$EH" ]; then echo "$SOURCES" > engine-sources.sha256 +elif [ -n "$EH" ]; then + echo "warning: built with WebAssembly's exnref exceptions (WASM_EH=exnref): the page is for experiments, and check-dist.sh refuses it" >&2 else echo "warning: built with $have, not Emscripten $EMSCRIPTEN_VERSION: the page is for experiments, and check-dist.sh refuses it" >&2 fi diff --git a/dist/discpress.html b/dist/discpress.html index 1304864..3cd5078 100644 --- a/dist/discpress.html +++ b/dist/discpress.html @@ -15175,4 +15175,4 @@

About and licenses

- + diff --git a/docs/chd/fork-plan.md b/docs/chd/fork-plan.md index 0fd4069..4324799 100644 --- a/docs/chd/fork-plan.md +++ b/docs/chd/fork-plan.md @@ -129,7 +129,7 @@ Effort estimates assume one developer working with the existing test harness. Th | 5. What users ask for | Done (1.5.0) | | 6. Detection that knows the game | Done (1.7.0) | | 7. Layouts each emulator reads right | Planned; nothing built (no new conversion engine, but steps 2 and 5 change the fork) | -| 8. Faster, more efficient, more reliable | Step 1 done (fuzzing and crafted headers found over 20 faults that damaged files reach, all but one also chdman 0.289's); step 2's workflows in place, with first results; step 3 in progress (a wider compression window: DVDs 11–20% faster); step 4 planned | +| 8. Faster, more efficient, more reliable | Step 1 done (fuzzing and crafted headers found over 20 faults that damaged files reach, all but one also chdman 0.289's); step 2's workflows in place, with first results; step 3 in progress (a wider compression window: DVDs 11–20% faster); step 4 done (an exnref build in CI) | The two releases after milestone 6 are outside the milestones, and their CHDs are the same: - **1.8.0** fixed what a full review of the app found: discs that took another game's tracks or name, results that could be lost, slow adding of big folders. @@ -473,9 +473,12 @@ Where the time goes, measured on the benchmark discs (300 MB CD, 1 GB DVD; 4 cor scalar: the partition sums (15) and the fixed predictor (56), integer loops that would give the same results by construction. Only worth writing if a measurement shows them in the page's time. - Whether FLAC can stop after the first channel when it already can't win: only if that is exact. -4. **Later** (as needed): a CI build with WebAssembly's new exception format (`-sWASM_LEGACY_EXCEPTIONS=0`, - exnref), so that switching once every browser has it is one line. A battery saver isn't needed: the - thread setting already is one. +4. **The new exception format** (done): `WASM_EH=exnref ./build.sh` builds with exnref + (`-sWASM_LEGACY_EXCEPTIONS=0`: 1,704 `try_table`, no legacy `try`), and `.github/workflows/rebuild.yml` + runs the conversion tests on that page, which gives chdman's CHDs byte for byte at the same speed (the + DVD's default 44.40 s against 44.67, the CD's 11.65 against 11.66). Switching, once every browser the + page supports has exnref (Chrome 137, Firefox 131, Safari 18.4), is that flag and `EH_TEST` in + `app/ui.js`. A battery saver isn't needed: the thread setting already is one. ## 5. Test and verification strategy From 982522d84e44692e702bc64a119fb0c7bdcb4fa3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 04:50:00 +0000 Subject: [PATCH 4/4] Plan: FLAC's remaining loops and an early stop after a channel, measured and not done Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QUSh6AFtA1CAR66DNvjtty --- docs/chd/fork-plan.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/chd/fork-plan.md b/docs/chd/fork-plan.md index 4324799..572a2b5 100644 --- a/docs/chd/fork-plan.md +++ b/docs/chd/fork-plan.md @@ -467,12 +467,17 @@ Where the time goes, measured on the benchmark discs (300 MB CD, 1 GB DVD; 4 cor buffer holds there (§3, "chdman's window is part of its output"). Left: the running SHA-1 and the copies, about 3 s of the job worker's time per GB, and Firefox, whose 4 threads gain less (CI's benchmark: 14.6 → 11.6 s, against Chromium's 13.7 → 8.7). - - Exact SIMD for FLAC's loops: Emscripten already vectorizes the two largest, autocorrelation (373 SIMD - instructions in the build) and the residual from the LPC coefficients (349), without changing a result - (each lag's sum keeps its order; WebAssembly SIMD has no fused multiply-add). What is left is mostly - scalar: the partition sums (15) and the fixed predictor (56), integer loops that would give the same - results by construction. Only worth writing if a measurement shows them in the page's time. - - Whether FLAC can stop after the first channel when it already can't win: only if that is exact. + - Exact SIMD for FLAC's loops: not written. Emscripten already vectorizes the two largest, + autocorrelation (373 SIMD instructions in the build) and the residual from the LPC coefficients (349), + without changing a result (each lag's sum keeps its order; WebAssembly SIMD has no fused multiply-add). + What is left mostly scalar, the partition sums (15) and the fixed predictor (56), is 21% of FLAC's + instructions; FLAC is about 39% of a DVD hunk's default compression (it encodes each hunk twice, in + both byte orders, a block each), so making them twice as fast would save about 4%. + - FLAC stopping after the first channel when it already can't win: not exact, so not done. chdman's + FLAC (level 8) searches every stereo pairing (left/right, left/side, right/side, mid/side) and keeps the + smallest; after the first channels, the size of the pairings still to come is unknown and can be far + smaller, so nothing counted so far bounds the frame from below. (Within a channel the encoder already + stops after a block that passes the limit, but a DVD hunk is one block.) 4. **The new exception format** (done): `WASM_EH=exnref ./build.sh` builds with exnref (`-sWASM_LEGACY_EXCEPTIONS=0`: 1,704 `try_table`, no legacy `try`), and `.github/workflows/rebuild.yml` runs the conversion tests on that page, which gives chdman's CHDs byte for byte at the same speed (the