Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 60 additions & 1 deletion .github/workflows/rebuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
# page comes out byte for byte the dist/discpress.html in the repository, the file releases ship.
# scripts/check-dist.sh (run by CI and by every release) checks that the page records the sources it
# was built from, a hash build.sh writes itself; this checks the build itself, on another machine.
# Runs when the engine, its build or the committed page changes, weekly, and on demand.
# Runs when the engine, its build or the committed page changes, weekly, and on demand. A second job builds
# the same sources with WebAssembly's new exception format (exnref) and runs the conversion tests on it.
name: Rebuild

on:
Expand Down Expand Up @@ -63,3 +64,61 @@ jobs:
build/chdman-nosimd.wasm
build/chdman.js
retention-days: 14

# The same sources with WebAssembly's new exception format (exnref, WASM_EH=exnref in build.sh), so that
# switching to it, once every browser the page supports has it, is a known quantity: the conversions
# must still give native chdman's CHDs byte for byte. Its page is never released (it records no sources).
exnref:
runs-on: ubuntu-24.04
timeout-minutes: 90
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- name: Cache Emscripten 6.0.10
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ~/emsdk
key: emsdk-6.0.10-${{ runner.os }}
- name: Emscripten 6.0.10 (emsdk)
run: |
[ -d ~/emsdk/.git ] || git clone --depth 1 https://github.com/emscripten-core/emsdk.git ~/emsdk
cd ~/emsdk
./emsdk install 6.0.10
./emsdk activate 6.0.10
- name: Build with exnref exceptions
run: |
source ~/emsdk/emsdk_env.sh
WASM_EH=exnref ./build.sh
~/emsdk/upstream/bin/llvm-objdump -d build/chdman.wasm | grep -q try_table
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 22
cache: npm
cache-dependency-path: tests/package-lock.json
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.12'
- name: numpy, for the synthetic test discs
run: pip install numpy==2.4.6
- name: Cache MAME 0.289 and its chdman
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
third_party/mame
build/chdman-0.289
key: mame-0.289-${{ runner.os }}-${{ hashFiles('scripts/fetch-mame.sh', 'scripts/build-upstream.sh', 'wasm/**') }}
- name: Native chdman (0.289 and the engine)
run: |
[ -x build/chdman-0.289 ] || ./scripts/build-upstream.sh
./scripts/build-native.sh
- name: Playwright and Chromium
working-directory: tests
run: |
npm ci --no-audit --no-fund
npx playwright install --with-deps chromium
- name: Synthetic test discs
run: python3 tests/fixtures/make_fixtures.py
- name: Conversions on the exnref page
working-directory: tests
env:
REQUIRE_NATIVE: '1'
run: npx playwright test ui/smoke.spec.js ui/convert.spec.js ui/window.spec.js ui/fallbacks.spec.js --project=desktop
8 changes: 4 additions & 4 deletions CLAUDE.md

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions app/ui.js
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ var Engine = {
hw.onerror = function (e) { helperFailed(e.message || 'failed to start'); };
// (tests: DEBUG.failHelpers makes helpers fail as a miscompiling WebAssembly engine does; 'lie': the
// first one answers for other hunks than it was given)
self.post(hw, { type: 'helper', port: ch.port2, fail: o.debugFail === 'all' || (o.debugFail === 'one' && i === 0), lie: o.debugFail === 'lie' && i === 0 }, [ch.port2]);
self.post(hw, { type: 'helper', port: ch.port2, fail: o.debugFail === 'all' || (o.debugFail === 'one' && i === 0), lie: o.debugFail === 'lie' && i === 0, profile: !!DEBUG.profile }, [ch.port2]);
helpers.push(hw);
ports.push(ch.port1);
} catch (e) { break; }
Expand Down Expand Up @@ -460,7 +460,7 @@ var Engine = {
self.post(worker, {
type: 'run', jobId: o.jobId, dirPath: o.dirPath, args: o.args, inputs: o.inputs || [],
writable: o.writable || [], slots: o.slots || 0, outMode: o.outMode || 'mem', ports: ports, debugStage: DEBUG.stage, debugSinkMax: DEBUG.sinkMax, debugStorage: DEBUG.failStorage,
debugFreeze: o.onStall ? DEBUG.freeze : 0
debugFreeze: o.onStall ? DEBUG.freeze : 0, debugProfile: !!DEBUG.profile
}, ports);
});
return {
Expand Down
50 changes: 40 additions & 10 deletions app/worker.js
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ function instantiate(extra) {
return createChdman(opts);
}

/* ---------------- profiling (DEBUG.profile) ---------------- */
// where the job worker's time goes, in ms by what it did; reported in the job's log when chdman ends
var PROF = null;
function profAdd(name, t0) { if (PROF) PROF[name] = (PROF[name] || 0) + (performance.now() - t0); }

/* ---------------- storage backends ---------------- */

function BlobStore(blob) {
Expand All @@ -72,12 +77,14 @@ BlobStore.prototype.block = function (bi) {
return b;
};
BlobStore.prototype.read = function (dst, pos, len) {
var t0 = PROF ? performance.now() : 0;
try { return this.read2(dst, pos, len); }
catch (e) {
if (e && typeof e.errno === 'number') throw e;
postMessage({ type: 'notice', level: 'error', message: 'Reading the input failed: ' + (e && e.message || e) });
throw { errno: ERR.EIO };
}
finally { profAdd('read input', t0); }
};
BlobStore.prototype.read2 = function (dst, pos, len) {
if (pos >= this.sizeV) return 0;
Expand Down Expand Up @@ -489,12 +496,13 @@ function OpfsStore(slot) {
this.tick = 0;
}
OpfsStore.prototype.put = function (src, pos) {
var n;
var n, t0 = PROF ? performance.now() : 0;
try { n = this.h.write(src, { at: pos }); }
catch (e) {
postMessage({ type: 'notice', level: 'error', message: 'Storage full: the browser refused to write more data (' + (e && e.name || e) + ').' });
throw { errno: ERR.ENOSPC };
}
finally { profAdd('write output', t0); }
if (n !== src.length) throw { errno: ERR.ENOSPC };
};
OpfsStore.prototype.flushBuf = function (b) {
Expand Down Expand Up @@ -1046,6 +1054,11 @@ function setupParallel(M, ports) {
if (m.type !== 'result') return;
// chdman ended (done, or failed): its work items are gone, so nothing more is written to them
if (closed) return;
var tr = PROF ? performance.now() : 0;
try { takeResult(h, m); } finally { profAdd('take results', tr); }
};
function takeResult(h, m) {
if (PROF && m.ms) { PROF['helpers compressing'] = (PROF['helpers compressing'] || 0) + m.ms[0]; PROF['helpers busy'] = (PROF['helpers busy'] || 0) + m.ms[1]; }
// results go into chdman's work items only for the batch this helper was given, item by item:
// an answer for another batch, or for other items, would write over hunks it doesn't hold. And
// each as the compressor gives it: a codec slot that was tried, shorter than a hunk; -1 (stored)
Expand Down Expand Up @@ -1074,19 +1087,20 @@ function setupParallel(M, ports) {
}
dispatch();
maybeWake();
};
}
});

// flags: 1 = libdeflate for the deflate codec (--libdeflate)
M.parSetup = function (hb, ub, comps, flags) {
M.parSetup = function (hb, ub, comps, flags, window) {
if (!helpers.length) return false;
// the LaserDisc codec (avhu) reads settings from the CHD's own metadata: keep it single-threaded
if (comps.indexOf(0x61766875) >= 0) return false;
hunkbytes = hb;
// chdman keeps at most 256 hunks in flight and reads them 128 at a time, so a batch must be
// small enough for those 128 to give every helper two batches; otherwise helpers sit idle
// (with 4 KiB DVD hunks, 512 KiB batches meant only two helpers ever had work)
batchSize = Math.max(1, Math.min(64, Math.floor((512 * 1024) / hb), Math.floor(128 / (2 * helpers.length))));
// chdman keeps `window` hunks in flight (256, or up to 1024 small ones in the browser) and reads
// half of them at a time, so a batch must be small enough for each read to give every helper two
// batches; otherwise helpers sit idle (with 4 KiB DVD hunks, 512 KiB batches meant only two helpers
// ever had work). And as large as that allows: each batch costs a message both ways
batchSize = Math.max(1, Math.min(64, Math.floor((512 * 1024) / hb), Math.floor((window || 256) / 2 / (2 * helpers.length))));
scratch = M._malloc(hb + 64);
if (!scratch) return false; // no memory for it: chdman compresses by itself
helpers.forEach(function (h) { h.port.postMessage({ type: 'init', hunkbytes: hb, unitbytes: ub, comps: comps, flags: flags || 0 }); });
Expand All @@ -1101,13 +1115,17 @@ function setupParallel(M, ports) {
};
// codecs: the codec slots the codec plan tries for this hunk, a bit each (15 = all)
M.parSubmit = function (item, ptr, len, codecs) {
var t0 = PROF ? performance.now() : 0;
try { parSubmit(item, ptr, len, codecs); } finally { profAdd('hand out hunks', t0); }
};
function parSubmit(item, ptr, len, codecs) {
if (!open) open = { n: 0, items: new Uint32Array(batchSize), codecs: new Uint8Array(batchSize), buf: new Uint8Array(batchSize * hunkbytes) };
open.buf.set(M.HEAPU8.subarray(ptr, ptr + len), open.n * hunkbytes);
open.codecs[open.n] = codecs;
open.items[open.n++] = item;
fifo.push(item);
if (open.n === batchSize) flushOpen();
};
}
// chdman paused after a compression step: resolves when it can go on (its next hunk is in)
M.parWait = function () {
return new Promise(function (resolve) {
Expand Down Expand Up @@ -1255,9 +1273,12 @@ async function runHelper(msg) {
if (m.type === 'batch') {
var n = m.items.length, data = new Uint8Array(m.data), codecs = m.codecs;
var meta = new Int32Array(n * 3), sha1 = new Uint8Array(n * 20), parts = [], total = 0;
var tb = msg.profile ? performance.now() : 0, tc = 0;
for (var i = 0; i < n; i++) {
M.HEAPU8.set(data.subarray(i * hb, (i + 1) * hb), inbuf);
var t1 = msg.profile ? performance.now() : 0;
var c = M._wasm_helper_compress(res, sha, codecs ? codecs[i] : 15);
if (msg.profile) tc += performance.now() - t1;
var heap32 = M.HEAPU32, heap8 = M.HEAPU8;
var len = heap32[res >>> 2], crc = heap32[(res >>> 2) + 1];
meta[i * 3] = c; meta[i * 3 + 1] = len; meta[i * 3 + 2] = crc;
Expand All @@ -1268,8 +1289,8 @@ async function runHelper(msg) {
var out = new Uint8Array(total), o = 0;
parts.forEach(function (p) { out.set(p, o); o += p.length; });
if (msg.lie) m.items[0] += 64; // a test (DEBUG.failHelpers 'lie'): an answer for another hunk
port.postMessage({ type: 'result', id: m.id, items: m.items, meta: meta, sha1: sha1, out: out },
[m.items.buffer, meta.buffer, sha1.buffer, out.buffer]);
port.postMessage({ type: 'result', id: m.id, items: m.items, meta: meta, sha1: sha1, out: out,
ms: msg.profile ? [tc, performance.now() - tb] : null }, [m.items.buffer, meta.buffer, sha1.buffer, out.buffer]);
}
} catch (err) {
// (a WebAssembly trap here, which some Safari versions' optimizing compiler causes: the job worker
Expand Down Expand Up @@ -1319,21 +1340,30 @@ async function runChdman(M, args) {
['chdman'].concat(args).forEach(function (a, i) { var p = M.stringToNewUTF8(a); M.HEAPU32[(argv >>> 2) + i] = p; });
M.HEAPU32[(argv >>> 2) + args.length + 1] = 0;
try {
var t0 = PROF ? performance.now() : 0;
var code = M._chdman_begin(args.length + 1, argv);
profAdd('in the engine', t0);
while (code === -1) {
t0 = PROF ? performance.now() : 0;
if (sinkFull()) await sinkDrained();
if (M.rdBusy && M.rdBusy()) await M.rdWait();
else if (M.parWait) await M.parWait();
profAdd('waiting for helpers', t0);
t0 = PROF ? performance.now() : 0;
code = M._chdman_resume();
profAdd('in the engine', t0);
}
return code;
} finally {
if (M.parClose) M.parClose();
// (the engine's time includes its reads, writes and hunks handed out; the wait includes taking results)
if (PROF) postMessage({ type: 'notice', level: 'info', message: 'Profile (ms): ' + Object.keys(PROF).map(function (k) { return k + ' ' + Math.round(PROF[k]); }).join(', ') });
}
}

async function runJob(msg) {
if (msg.debugSinkMax) SINK_MAX = msg.debugSinkMax; // testing: a small limit
if (msg.debugProfile) PROF = {};
await loadModule(msg);
// (the engine first: until it is instantiated the page holds off terminating this worker, Engine.stop)
var out = makeLineSink(1), err = makeLineSink(2), aborted = null;
Expand Down
23 changes: 17 additions & 6 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
# can give other WebAssembly bytes, so the build refuses it (EMSCRIPTEN_ANY=1 allows it, for experiments:
# the page it makes is not a release build).
#
# Environment: JOBS (default: CPU count)
# Environment: JOBS (default: CPU count); WASM_EH=exnref builds with WebAssembly's new exception format
# (exnref: Chrome 137+, Firefox 131+, Safari 18.4+) instead of the legacy one, to try it before switching
# (which also needs app/ui.js's EH_TEST to test for it). Like EMSCRIPTEN_ANY's, that page records no
# sources, so scripts/check-dist.sh and a release refuse it.
set -euo pipefail
ROOT="$(cd "$(dirname "$0")" && pwd)"
OUT="$ROOT/build"
Expand All @@ -28,22 +31,28 @@ if [ -z "$pinned" ] && [ -z "${EMSCRIPTEN_ANY:-}" ]; then
exit 1
fi
JOBS="${JOBS:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)}"
# (its own objects: the Makefile doesn't rebuild what other flags compiled)
case "${WASM_EH:-legacy}" in
legacy) EH=; EHO= ;;
exnref) EH=-sWASM_LEGACY_EXCEPTIONS=0; EHO=-exnref ;;
*) echo "WASM_EH: legacy or exnref, not $WASM_EH" >&2; exit 1 ;;
esac
mkdir -p "$OUT"
# The sources this wasm is built from, which the page records for scripts/check-dist.sh: hashed before
# compiling, written to build/ only once the build succeeded.
SOURCES="$(python3 "$ROOT/scripts/engine-sources.py")"
rm -f "$OUT/engine-sources.sha256"

echo "==> Compiling (SIMD)"
make -s -C "$ROOT/wasm" -j"$JOBS" T=wasm O="$OUT/obj-wasm" objs
make -s -C "$ROOT/wasm" -j"$JOBS" T=wasm O="$OUT/obj-wasm$EHO" EXTRA="${EXTRA:-} $EH" objs
echo "==> Compiling (no SIMD, for older devices)"
make -s -C "$ROOT/wasm" -j"$JOBS" T=wasm SIMD= O="$OUT/obj-wasm-nosimd" objs
make -s -C "$ROOT/wasm" -j"$JOBS" T=wasm SIMD= O="$OUT/obj-wasm-nosimd$EHO" EXTRA="${EXTRA:-} $EH" objs

echo "==> Linking"
W="-sMODULARIZE=1 -sEXPORT_NAME=createChdman -sENVIRONMENT=web,worker -sINVOKE_RUN=0 -sFORCE_FILESYSTEM=1"
cd "$OUT"
OBJDIR=obj-wasm "$ROOT/wasm/link.sh" chdman.js $W
OBJDIR=obj-wasm-nosimd "$ROOT/wasm/link.sh" nosimd-tmp.js $W
OBJDIR=obj-wasm$EHO "$ROOT/wasm/link.sh" chdman.js $W $EH
OBJDIR=obj-wasm-nosimd$EHO "$ROOT/wasm/link.sh" nosimd-tmp.js $W $EH
mv nosimd-tmp.wasm chdman-nosimd.wasm
# Both variants share one JS glue file, so it must be identical.
if ! diff <(sed 's/nosimd-tmp/chdman/g' nosimd-tmp.js) chdman.js >/dev/null; then
Expand All @@ -52,8 +61,10 @@ if ! diff <(sed 's/nosimd-tmp/chdman/g' nosimd-tmp.js) chdman.js >/dev/null; the
fi
rm -f nosimd-tmp.js
# Another Emscripten's page records no sources: scripts/check-dist.sh and a release refuse it
if [ -n "$pinned" ]; then
if [ -n "$pinned" ] && [ -z "$EH" ]; then
echo "$SOURCES" > engine-sources.sha256
elif [ -n "$EH" ]; then
echo "warning: built with WebAssembly's exnref exceptions (WASM_EH=exnref): the page is for experiments, and check-dist.sh refuses it" >&2
else
echo "warning: built with $have, not Emscripten $EMSCRIPTEN_VERSION: the page is for experiments, and check-dist.sh refuses it" >&2
fi
Expand Down
Loading
Loading