From fddcd397bc795662d53893e1b1a43a42ac2663e6 Mon Sep 17 00:00:00 2001 From: technomancer Date: Sat, 7 Nov 2026 08:38:10 -0800 Subject: [PATCH] wgpu: VPro raster unit on the host GPU, and a wgpu window A common wgpu layer (src/wgpu) for host-GPU rasterisation, and the VPro (Odyssey) raster unit built on it, word for word with the CPU pipeline. - The board draws on the GPU raster unit with board memory mirrored on the CPU; dirty maps both ways, region readbacks, no waits on texture and image uploads (blit records, mip levels and scratch images as regions). - The texture unit runs on the GPU. - Batches submit as the board idles or after 4 ms; display swaps wait on stream positions, so a flip shows only drawing that has completed. - Window-id bytes written by GPU blits are also stored in the mirror, since the display front end reads them without waiting. - The VPro display composes on the GPU. - New window renderer: renderer = "wgpu" (or --renderer wgpu). VPro frames stay on the device, nothing is read back unless a screenshot or VNC asks. Newport's compositor runs as a WGSL kernel. GL stays the default; iris-gui is unchanged. - Testing: the CPU and GPU units agree on recorded sessions (new test), and on the guest GL suites, visual sweeps, piglit and seven X visual modes. New guest suite drivers in test/gltest (glsuite.sh, xsweep.sh, glsuite-compare.sh) and replay knobs (VPRO_REPLAY_WRITES, VPRO_REPLAY_TICK). Squashed from the wgpu-raster branch. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 24 + Cargo.toml | 4 + FEATURES.md | 10 +- HACKING.md | 73 + HELP.md | 5 + TESTING.md | 37 + clanker/CLANKER.md | 2 +- clanker/INDEX.md | 17 + .../gaps-found-by-the-2026-11-07-sweeps.md | 43 + .../monitor-reads-quiesce-the-board.md | 9 +- clanker/plans/wgpu-raster.md | 533 +++++++ clanker/plans/wgpu-ui.md | 76 + clanker/wgpu/gpu-floats-are-not-the-hosts.md | 49 + clanker/wgpu/read-storage-in-place.md | 43 + ...aps-wait-on-stream-positions-not-fences.md | 38 + clanker/wgpu/the-mirror-and-its-dirty-maps.md | 69 + clanker/wgpu/vram-accesses-are-atomic.md | 20 + ...bytes-are-written-through-to-the-mirror.md | 34 + debug.md | 1 + src/config.rs | 58 + src/debug_overlay.rs | 45 +- src/dev/ng1/mod.rs | 3 + src/dev/ng1/rex3.rs | 23 + src/dev/ng1/wgpu/mod.rs | 151 ++ src/dev/ng1/wgpu/shaders/scanout.wgsl | 162 ++ src/dev/ng1/wgpu/tests.rs | 126 ++ src/dev/odyssey/board.rs | 42 + src/dev/odyssey/copy.rs | 30 +- src/dev/odyssey/debug.rs | 51 +- src/dev/odyssey/display.rs | 59 +- src/dev/odyssey/dma.rs | 85 +- src/dev/odyssey/draw.rs | 192 ++- src/dev/odyssey/fifo.rs | 24 +- src/dev/odyssey/frame.rs | 10 +- src/dev/odyssey/gl.rs | 24 +- src/dev/odyssey/golden/mod.rs | 2 +- src/dev/odyssey/golden/replay.rs | 181 ++- src/dev/odyssey/golden/scenes.rs | 2 +- src/dev/odyssey/gpu_unit.rs | 162 ++ src/dev/odyssey/interface.rs | 32 +- src/dev/odyssey/mmio.rs | 9 +- src/dev/odyssey/mod.rs | 36 +- src/dev/odyssey/pipe/interp.rs | 7 + src/dev/odyssey/pipe/mod.rs | 71 +- src/dev/odyssey/pipe/raster.rs | 27 +- src/dev/odyssey/pipe/setup.rs | 16 +- src/dev/odyssey/pipe/texture.rs | 5 +- src/dev/odyssey/raster_bench.rs | 4 +- src/dev/odyssey/raster_unit.rs | 61 +- src/dev/odyssey/reader.rs | 68 +- src/dev/odyssey/testdata/README.md | 2 +- src/dev/odyssey/tests/catalogue.rs | 2 +- src/dev/odyssey/tests/display.rs | 36 + src/dev/odyssey/tests/dma.rs | 2 +- src/dev/odyssey/tests/index.rs | 2 +- src/dev/odyssey/tests/mod.rs | 17 +- src/dev/odyssey/tests/monitor.rs | 2 +- src/dev/odyssey/tests/pixels.rs | 10 +- src/dev/odyssey/tests/raster.rs | 54 +- src/dev/odyssey/tests/stop.rs | 2 +- src/dev/odyssey/view.rs | 8 + src/dev/odyssey/vram.rs | 130 +- src/dev/odyssey/wgpu/mod.rs | 1303 +++++++++++++++++ src/dev/odyssey/wgpu/scanout.rs | 152 ++ src/dev/odyssey/wgpu/shaders/backend.wgsl | 509 +++++++ src/dev/odyssey/wgpu/shaders/scanout.wgsl | 111 ++ src/dev/odyssey/wgpu/shaders/texture.wgsl | 635 ++++++++ src/dev/odyssey/wgpu/shaders/views.wgsl | 235 +++ src/dev/odyssey/wgpu/shaders/vpro.wgsl | 388 +++++ src/dev/odyssey/wgpu/shaders/vram.wgsl | 144 ++ src/dev/odyssey/wgpu/tests.rs | 892 +++++++++++ src/dev/odyssey/worker.rs | 4 +- src/disp.rs | 11 +- src/gfx_display.rs | 6 + src/ip30_machine.rs | 2 +- src/lib.rs | 4 +- src/main.rs | 3 +- src/ui.rs | 222 ++- src/wgpu/batch.rs | 354 +++++ src/wgpu/device.rs | 247 ++++ src/wgpu/memory.rs | 125 ++ src/wgpu/mod.rs | 63 + src/wgpu/present.rs | 447 ++++++ src/wgpu/scanout.rs | 182 +++ src/wgpu/shader.rs | 198 +++ src/wgpu/shaders/bin.wgsl | 47 + src/wgpu/shaders/bin_accept_all.wgsl | 4 + src/wgpu/shaders/common.wgsl | 36 + src/wgpu/shaders/float.wgsl | 55 + src/wgpu/shaders/i64.wgsl | 93 ++ src/wgpu/shaders/present.wgsl | 37 + src/wgpu/shaders/raster.wgsl | 57 + src/wgpu/shaders/scanout.wgsl | 23 + src/wgpu/shaders/unpack.wgsl | 16 + src/wgpu/tests/flat.wgsl | 65 + src/wgpu/tests/mod.rs | 312 ++++ src/wgpu/tiles.rs | 139 ++ test/gltest/glsuite-compare.sh | 40 + test/gltest/glsuite.sh | 68 + test/gltest/xsweep.sh | 47 + 100 files changed, 10271 insertions(+), 127 deletions(-) create mode 100644 clanker/odyssey/gaps-found-by-the-2026-11-07-sweeps.md create mode 100644 clanker/plans/wgpu-raster.md create mode 100644 clanker/plans/wgpu-ui.md create mode 100644 clanker/wgpu/gpu-floats-are-not-the-hosts.md create mode 100644 clanker/wgpu/read-storage-in-place.md create mode 100644 clanker/wgpu/swaps-wait-on-stream-positions-not-fences.md create mode 100644 clanker/wgpu/the-mirror-and-its-dirty-maps.md create mode 100644 clanker/wgpu/vram-accesses-are-atomic.md create mode 100644 clanker/wgpu/window-id-bytes-are-written-through-to-the-mirror.md create mode 100644 src/dev/ng1/wgpu/mod.rs create mode 100644 src/dev/ng1/wgpu/shaders/scanout.wgsl create mode 100644 src/dev/ng1/wgpu/tests.rs create mode 100644 src/dev/odyssey/gpu_unit.rs create mode 100644 src/dev/odyssey/wgpu/mod.rs create mode 100644 src/dev/odyssey/wgpu/scanout.rs create mode 100644 src/dev/odyssey/wgpu/shaders/backend.wgsl create mode 100644 src/dev/odyssey/wgpu/shaders/scanout.wgsl create mode 100644 src/dev/odyssey/wgpu/shaders/texture.wgsl create mode 100644 src/dev/odyssey/wgpu/shaders/views.wgsl create mode 100644 src/dev/odyssey/wgpu/shaders/vpro.wgsl create mode 100644 src/dev/odyssey/wgpu/shaders/vram.wgsl create mode 100644 src/dev/odyssey/wgpu/tests.rs create mode 100644 src/wgpu/batch.rs create mode 100644 src/wgpu/device.rs create mode 100644 src/wgpu/memory.rs create mode 100644 src/wgpu/mod.rs create mode 100644 src/wgpu/present.rs create mode 100644 src/wgpu/scanout.rs create mode 100644 src/wgpu/shader.rs create mode 100644 src/wgpu/shaders/bin.wgsl create mode 100644 src/wgpu/shaders/bin_accept_all.wgsl create mode 100644 src/wgpu/shaders/common.wgsl create mode 100644 src/wgpu/shaders/float.wgsl create mode 100644 src/wgpu/shaders/i64.wgsl create mode 100644 src/wgpu/shaders/present.wgsl create mode 100644 src/wgpu/shaders/raster.wgsl create mode 100644 src/wgpu/shaders/scanout.wgsl create mode 100644 src/wgpu/shaders/unpack.wgsl create mode 100644 src/wgpu/tests/flat.wgsl create mode 100644 src/wgpu/tests/mod.rs create mode 100644 src/wgpu/tiles.rs create mode 100755 test/gltest/glsuite-compare.sh create mode 100644 test/gltest/glsuite.sh create mode 100644 test/gltest/xsweep.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index de9797b3d..291035328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,30 @@ hashes are given where a change is easiest to understand by reading the commit. `vpro fbdump`'s files, to diff against a live board's dump, and `vpro fbdump` adds `objects.bin`. +### VPro on the host GPU (`wgpu` feature, in progress) + +- `[graphics] raster = "wgpu"` moves the VPro's raster unit to the host GPU + through wgpu compute (`src/wgpu`, `src/dev/odyssey/wgpu`): filled + triangles, textured or not, clears and the whole back end run as + kernels over board memory in a GPU buffer, word for word with the CPU + pipeline (the golden scenes, the parity tests and the quake1 replay's + checkpoints pass on it); fogged and user-clipped primitives, lines and + points are still shaded on the CPU and stored by the GPU in order. The + front end's own pixel paths work on a mirror the unit keeps current, and + the display composes on the GPU too, read back into the window until the + GUI shares the device. Batches are submitted as the board goes idle, so + the PROM and X painting reach the screen at once, and a swap latches only + after the drawing armed before it is submitted. `vpro gpu` shows the + unit's counters. Window-id writes are kept in step with the front end's + own reads, which fixed GPU-only failures in the visual sweep; the GL + test suites, the visual and line sweeps and the scenes now read back the + same on both raster units. +- `renderer = "wgpu"` (`--renderer wgpu`) draws the window through wgpu; + with the VPro on the GPU unit the frame is presented from the board's + memory without a readback, and Newport's display is composed on the GPU + (the GL compositor's work in WGSL). Branch `wgpu-raster`, plan in + `clanker/plans/wgpu-raster.md`. + ### VPro drawing - Copies from a window into a texture (`glCopyTexSubImage`) read the diff --git a/Cargo.toml b/Cargo.toml index 8961bb8f6..743a09b7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -179,6 +179,10 @@ rex-jit = ["cranelift-codegen", "cranelift-frontend", "cranelift-jit", "cranelif # engine selected explicitly with REX3_REPLAY_ENGINE=gpu. It never changes the # live CPU renderer by default. rex-gpu = ["gputrace", "rex-dirty-upload", "dep:bytemuck", "dep:pollster", "dep:wgpu"] +# Host-GPU rasterisation on wgpu: board memory on the GPU, primitives binned +# into 8x8 tiles and rasterised by compute kernels (src/wgpu, and +# src/dev//wgpu for each board). See clanker/plans/wgpu-raster.md. +wgpu = ["dep:bytemuck", "dep:pollster", "dep:wgpu"] # Cranelift-based JIT for IMPACT (GR4) raster shaders: RSS/TE1 pixel # pipelines monomorphised on a 64-bit key (src/dev/mgras/rss_jit). gr4-jit = ["cranelift-codegen", "cranelift-frontend", "cranelift-jit", "cranelift-module", "cranelift-native"] diff --git a/FEATURES.md b/FEATURES.md index 5b570b1c4..c9174b22d 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -41,6 +41,7 @@ Select with `cargo build --release --features `; add | `default` | Default bundle: `tlbvmap` and `rexdiag`. | `tlbvmap`, `rexdiag` | | `rex-dirty-upload` | Track conservative Newport framebuffer damage and copy/upload only dirty regions. Opt-in; the default retains the original REX3 and presentation paths. | — | | `rex-jit` | Cranelift compiler for Newport draw shaders; complements precompiled shaders. | `cranelift-codegen`, `cranelift-frontend`, `cranelift-jit`, `cranelift-module`, `cranelift-native` | +| `wgpu` | Host-GPU rasterisation through wgpu compute: the common batch layer in `src/wgpu` and, as they land, the boards' GPU raster units under `src/dev//wgpu`. Work in progress on branch `wgpu-raster`; see `clanker/plans/wgpu-raster.md`. | `dep:bytemuck`, `dep:pollster`, `dep:wgpu` | | `rex-gpu` | Experimental, replay-only Newport wgpu/WGSL engine. It does not replace the live CPU renderer and is not part of `gpujit` or release builds. | `gputrace`, `rex-dirty-upload`, `dep:bytemuck`, `dep:pollster`, `dep:wgpu` | | `gr4-jit` | Cranelift compiler for IMPACT (GR4) raster pipelines (RSS/TE1 shaders). | `cranelift-codegen`, `cranelift-frontend`, `cranelift-jit`, `cranelift-module`, `cranelift-native` | | `gr2-jit` | Cranelift compiler for GR2 (XZ / Extreme) RE3 span pipelines (SHADED / FLAT shaders). | `cranelift-codegen`, `cranelift-frontend`, `cranelift-jit`, `cranelift-module`, `cranelift-native` | @@ -164,6 +165,7 @@ the first `[section]`, because TOML tables continue until the next header. | `ci_display` | `false` | Keep host graphics window visible in CI mode. | | `mouse_scroll_pixels_per_line` | `40.0` | Host scrolling distance per guest PS/2 detent. | | `lock_aspect_ratio` | `true` | Constrain standalone window aspect ratio; false allows letterboxing. | +| `renderer` | `gl` | What draws the window: `gl`, or `wgpu` (needs a build with the `wgpu` feature). With `wgpu` the window is presented through wgpu on the same GPU as a VPro whose raster unit is on the GPU, and its frames are shown without being read back. The iris-gui front end ignores it. | | `serial_log` | `unset` | Append guest channel-B output (the O2's first serial port); used by CI and GUI serial capture. `-` writes to standard output (for a container's log collector). | | `serial_log_a`, `serial_log_b` | `unset` | Log what the guest sends on channel A or B when that channel is not the console (A on the Indy and Indigo2, B on the Octane and O2; `serial_log` is the console's). Truncated at start on the Octane and O2, appended to on the Indy and Indigo2, like `serial_log`. | | `serial_log_timestamps` | `off` | `off` (raw transcript), `wall` (prefix each completed line with a UTC ISO-8601 stamp) or `ecs` (one Elastic Common Schema JSON document per non-empty line, logger `irix.guest`, control characters dropped). A partial line is held until its newline and written on stop. | @@ -186,6 +188,8 @@ the first `[section]`, because TOML tables continue until the next header. | `graphics.resolution` | `guest` | guest, 1024x768, 1280x960, 1280x1024; presets require Newport. | | `graphics.threads` | `2` | Worker threads the board's fragment pipeline draws on, at least 1. VPro: the screen tiles' owners, besides its front-end and raster threads; more help on a host with cores to spare. | | `graphics.tile` | `32` | VPro: the fragment pipeline's screen tile size in pixels, a power of two from 8 to 256. Each tile's work runs in order on one worker at a time; with many threads, 64 can be faster. | +| `graphics.raster` | `cpu` | VPro: where the raster unit runs. `cpu` draws on the threads above; `wgpu` draws on the host GPU through wgpu compute (needs a build with the `wgpu` feature; work in progress, see `clanker/plans/wgpu-raster.md`). | +| `graphics.gpu_backend` | `auto` | The wgpu backend for `raster = "wgpu"`: auto, vulkan, metal, dx12 or gl. | ### [scsi.] @@ -433,6 +437,8 @@ iris-gui does not parse this standalone option set; use its named-machine UI. | `--no-scsi-deferred-int` | Disable deferred SCSI status interrupts (default: enabled for OpenBSD/NetBSD compatibility). | | `--cpu` | Emulated CPU: `r4400` (default), `r5000`, or `r10000`. Overrides `[machine] cpu`. | | `--graphics` | Graphics board: `newport` (default), `xz`, `extreme`, `solidimpact`, `highimpact`, `maximpact` (or `impact:solid`, `impact:high`, `impact:max`), `v6`, `v8`, `v10`, `v12` (`vpro:v12` and so on, Octane only). | +| `--raster` | VPro: `cpu` (default) or `wgpu`, the raster unit on the host GPU (`[graphics] raster`; needs a build with the `wgpu` feature). | +| `--renderer` | What draws the window: `gl` (default) or `wgpu` (`renderer`; needs a build with the `wgpu` feature). | | `--gdb-port` | Enable GDB stub on the given TCP port (e.g. --gdb-port 1234). Connect with: target remote localhost: | | `--monitor-port` | Monitor console port on 127.0.0.1 (default 8888). Give each iris running at once its own. | | `--vnc [ADDR]` | Serve the screen, keyboard and mouse over VNC on ADDR (`host:port` or a port; default 127.0.0.1:5900). Sets `vnc.listen`. | @@ -566,7 +572,9 @@ and offline-tool controls are included so they are not mistaken for GUI settings | `MGRAS_REPLAY_TRACE` | IMPACT replay tracing selector (`gputrace` builds). | [src/dev/mgras/record.rs](src/dev/mgras/record.rs) | | `MGRAS_SIZE_PROBE` | Input for the opt-in IMPACT size/probe test. | [src/dev/mgras/mgras_tests.rs](src/dev/mgras/mgras_tests.rs) | | `VPRO_GOLDEN` | `tolerance` lets a VPro golden frame whose hash changed pass inside its stored limits. | [src/dev/odyssey/golden/mod.rs](src/dev/odyssey/golden/mod.rs) | -| `VPRO_REPLAY` | Replay input path for one VPro recording; `VPRO_REPLAY_DIR` picks the directory that is replayed otherwise, `VPRO_REPLAY_THREADS=1` replays with the raster and back-end threads running, `VPRO_REPLAY_LIMIT=` stops after n records (with `VPRO_REPLAY_DUMP` to see the board there). | [src/dev/odyssey/golden/replay.rs](src/dev/odyssey/golden/replay.rs) | +| `IRIS_WGPU_DUMP` | With the `wgpu` feature: a directory the composed WGSL of every GPU kernel is written to as it is compiled, for `naga` and `spirv-val`. | [src/wgpu/shader.rs](src/wgpu/shader.rs) | +| `VPRO_GPU_TRACE_WAITS` | With the `wgpu` feature: the VPro GPU unit counts its readbacks by the board function that caused them; the replay prints the histogram. | [src/dev/odyssey/wgpu/mod.rs](src/dev/odyssey/wgpu/mod.rs) | +| `VPRO_REPLAY` | Replay input path for one VPro recording; `VPRO_REPLAY_DIR` picks the directory that is replayed otherwise, `VPRO_REPLAY_THREADS=1` replays with the raster and back-end threads running, `VPRO_REPLAY_GPU=1` replays on the GPU raster unit (`wgpu` feature; with `VPRO_REPLAY_LAZY=1` the unit skips the readbacks between checkpoints, to time the GPU path alone), `VPRO_REPLAY_LIMIT=` stops after n records and checks only the checkpoints reached (with `VPRO_REPLAY_DUMP` to see the board there), `VPRO_REPLAY_TICK=` ends a board slice every n stores as a running machine does (on the GPU unit each slice end can submit a batch), `VPRO_REPLAY_WRITES=` logs every write the board makes to guest memory (pixel reads, fences, query results); the report line gives their hash and the final memory hash, to compare two replays. `VPRO_GPU=1` puts every VPro test board (the golden scenes among them) on the GPU raster unit. | [src/dev/odyssey/golden/replay.rs](src/dev/odyssey/golden/replay.rs) | `RUST_LOG` configures the host Rust logger; `IRIS_DEBUG_LOG` configures device logging. Standard host variables (`HOME`, `XDG_CACHE_HOME`, `LOCALAPPDATA`, diff --git a/HACKING.md b/HACKING.md index 99f6b0e58..dfb584948 100644 --- a/HACKING.md +++ b/HACKING.md @@ -476,6 +476,7 @@ How each board uses it: | Newport (`src/dev/ng1/`) | REX3 register writes, GO commands, HOSTRW data | `REX3-Processor` executes REX3 drawing; `REX3-Refresh` composites VC2/XMAP/CMAP scanout and owns the GL context | | GR2 (`src/dev/gr2/`) | HQ2 command FIFO (`hq_fifo`) | `GR2-HQ2` interprets HQ2/GE commands at the GL level and feeds `re3_fifo`; `GR2-RE3` rasterises; `GR2-Display` scans out | | IMPACT (`src/dev/mgras/`) | HQ3 command FIFO and direct raster writes (`hq_fifo`) | `MGRAS-HQ3` runs the command and geometry front end and feeds `rss_fifo`; one RSS thread per raster engine draws (Maximum IMPACT: two, each owning alternate rows, fed by the broadcast FIFO); a refresh thread scans out | +| VPro (`src/dev/odyssey/`) | the command FIFO (`fifo.rs`) | `odyssey-work` decodes, transforms and sets primitives up and hands them to the raster queue; `odyssey-raster` and its `odyssey-pool` tile workers rasterise, texture and store; the display thread scans out. With `[graphics] raster = "wgpu"` the raster and pool threads do not start: the front end records into batches the host GPU draws (below) | GR2 and IMPACT are emulated at a high level: their microcode is uploaded but not executed; the command and GL token stream is interpreted directly. All @@ -496,6 +497,78 @@ without locks and composes it through the board's colour maps and DAC. Everything that touches the host GL context (including teardown) runs on the thread that owns it. +### The raster unit on the host GPU (`wgpu` feature) + +With `[graphics] raster = "wgpu"` the VPro's raster unit runs as compute +kernels on the host GPU (`src/wgpu` is the common layer, `src/dev/odyssey/ +wgpu` the board's kernels; the design is `clanker/plans/wgpu-raster.md`). +The front end runs every draw inside its submit and appends one *record* +per primitive (a triangle's setup, a quad it shaded itself, a clear's +rectangle, a row of pixels of an image put) to a *batch*, with the state +each was issued under in a *slot*; a batch is two dispatches, a bin pass +that gives each 8x8 tile a bitmask over the batch's primitives and a raster +pass with one workgroup per busy tile that walks its mask in order and +stores pixels itself. Order per pixel is the record order, so the kernels +need no atomics on pixels and no synchronisation between workgroups. + +Board memory lives in a GPU buffer; nothing on the CPU maps it. The front +end's own pixel paths (image puts, the integer primitives and their logical +operations, the X server's window-id writes, plane clears, DMA puts, depth +and stencil image writes) store through the unit as *blit* records, so they +keep their place in the order and nothing waits. The board's `Vram` is a +*mirror* for what still runs on the CPU (copies, DMA gets, the hashes and +the monitor), kept in step by two dirty maps: CPU stores mark 16 KB blocks +that go up before the next batch and at the end of each run of stores, and +records mark the blocks their boxes touch; a reader brings back only the +marked blocks under the box it reads (`Board::planes_in`), `dest_mem` and +`gpu_sync` bring everything back, and `Board::raster_wait` on this unit is a +flush and a GPU wait with no readback. A batch is submitted when it fills, +when something waits, and at the end of a run of stores once the board is +going idle or the batch has been open a few milliseconds +(`GpuRaster::tick`), so the screen follows the guest as it would on the +board without a submission per primitive. Swaps follow the same order +without a wait: a swap arm notes its position in the command stream +(`CommandFifo::pushed`), the unit publishes the position its submissions +cover (`DisplayShared::covered`), and the retrace latches a flip only once +the drawing before its arm is submitted; the scanout's own submission +queues behind that drawing on the same GPU queue. The invariant for the mirror is the +one the raster thread already imposed: the front end waits before it touches +memory itself; paths that must not wait (texture and scratch uploads) store +through an untracked pointer and upload exactly the tiles they wrote. The +display thread composes on the GPU as well (`src/wgpu/scanout.rs`), from +the same buffer, and reads the frame back into `Rex3Screen`; `Frame::compose` +on the CPU remains for screenshots and the replay harness. + +The front end reads the window-id buffer's bytes itself, without waiting +for the raster unit (the drawable format for a read, the ownership test). +So when it sends a store into those byte fields to the GPU it makes the +same store in the mirror, untracked: the mirror's id bytes stay current +with nothing uploaded twice. A new path that writes them on the GPU must do +the same. + +With `renderer = "wgpu"` the window is drawn by `src/wgpu/present.rs` +(`Presenter`: the surface, a pipeline of textured quads for the frame, the +debug overlay and the status bar, and a small kernel that unpacks a frame +already on the device). The surface comes from the instance the board's +`Gpu` was opened on (`GfxDisplay::gpu`). The VPro's display thread then +composes on the GPU and hands the scan-out buffer to the renderer +(`Renderer::present_gpu_frame`) unless something wants `screen.rgba` filled +(a screenshot, a frame request, a capturing VNC client), in which case it +reads the frame back as before. Newport's planes are composed on the +GPU too, by `src/dev/ng1/wgpu` (the GL and software compositors' rules as a +kernel on the common scan-out driver, checked pixel for pixel against +`SwCompositor`), and presented from there. The other boards' frames are +uploaded from the CPU. + +The kernels are held to the CPU model word for word by parity tests +(`cargo test --features wgpu --lib odyssey::wgpu`), the golden scenes with +`VPRO_GPU=1` and the replay checkpoints with `VPRO_REPLAY_GPU=1`. Two rules +came out of that and live in `clanker/wgpu/`: a GPU's division is not +correctly rounded and its compiler fuses multiply-adds, so the kernels +divide through `div_exact` and keep products apart from sums; and a large +struct copied by value into a lane's private memory gave silently wrong +results, so records, slots and tables are read from their buffers in place. + The status bar under the frame is fed by `StatusSource` (`src/disp.rs`), which the machine builds once and hands to every graphics board: the heartbeat word (front-panel LEDs and SCSI/network activity bits), the CPU diff --git a/HELP.md b/HELP.md index 4a83f2d65..ad825df4e 100644 --- a/HELP.md +++ b/HELP.md @@ -524,6 +524,8 @@ standalone `iris` window if you need them. The window is freely resizable. At 1× and 2× scale the display is rendered with nearest-neighbour pixel-perfect sampling; at all other sizes trilinear filtering with mipmaps is used. `lock_aspect_ratio = false` in `iris.toml` allows free resizing with letterboxing. When the guest changes video mode (PROM, then IRIX during boot) the window snaps to the new resolution times the configured `scale` / `--2x`. +`renderer = "wgpu"` (or `--renderer wgpu`, in a build with the `wgpu` feature) draws the window through wgpu instead of OpenGL, with the same scaling rules. With a VPro whose raster unit is on the GPU (`raster = "wgpu"`) the window shares that GPU, and each frame goes from the board's memory to the screen without passing through main memory; screenshots and VNC still read frames back when they ask for them. Newport's display (Indy, Indigo2 XL) is composed on the GPU as well. If wgpu cannot open the window, iris says so and draws with OpenGL. + ## VNC iris has a built-in VNC server. Any VNC viewer (TigerVNC, RealVNC, Remmina, @@ -602,6 +604,7 @@ scale = 1 # window scale, kept across guest mode changes; --2x headless = false # no window, no graphics; the O2 boots on its serial console (audio unaffected) no_audio = false # no HAL2 / cpal (graphics unaffected) lock_aspect_ratio = true # false = free resize with letterboxing +# renderer = "gl" # "wgpu": draw the window through wgpu (builds with the wgpu feature) mouse_scroll_pixels_per_line = 40 nat_subnet = "192.168.0.0/24" # gateway .1, guest .2 @@ -636,6 +639,8 @@ heads = 1 # 2 = dual-head Newport (second REX3 in GIO slot 1) resolution = "guest" # or "1024x768", "1280x960", "1280x1024" # threads = 2 # VPro: fragment pipeline workers (more on a host with cores to spare) # tile = 32 # VPro: screen tile size, a power of two from 8 to 256 +# raster = "cpu" # VPro: "wgpu" draws on the host GPU (builds with the wgpu feature; work in progress) +# gpu_backend = "auto" # with raster = "wgpu": "vulkan", "metal", "dx12" or "gl" [clock] # fixed_mhz = 33 # Count MHz: default 33 on IP22/IP24, 97.5 on IP28 diff --git a/TESTING.md b/TESTING.md index 6c658821e..867e1141e 100644 --- a/TESTING.md +++ b/TESTING.md @@ -57,6 +57,43 @@ before adding a kernel. `bench/irix/` is the other half: real workloads under a booted IRIX (filesystem, buffer cache, IRIX's own tools, X on REX3), driven over `iris-ci`. +**GPU parity tests** — with the `wgpu` feature, the host-GPU raster path +(`src/wgpu`, `src/dev/odyssey/wgpu`) is checked against the CPU pipeline: the +same triangles, shaded fragments and clears go through both, and the two board +memories are compared word for word. They need a GPU, or lavapipe, and print +`skipped` without one. + +```sh +cargo test --features wgpu --lib wgpu::tests odyssey::wgpu +``` + +**GL suites in a live guest** — `test/gltest/glsuite.sh` runs every GL check +program inside a booted IRIX (glstate, blendtest, seltest, cicopy, pixtest in +four visuals, the visual, texture and line sweeps, glprim scenes read back +through GL and captured with `scrsave`, and piglit's VPro list), and +`test/gltest/xsweep.sh` restarts the X server in each root visual (PseudoColor +8 and 12, TrueColor 12, 15, 24 and 30, DirectColor 24) and checks a GL scene +in each. Both write to `/shared/results/TAG`; the host-side +`test/gltest/glsuite-compare.sh A B` lists every difference between two runs. +Running both on the CPU and the GPU raster unit and comparing is how the GPU +unit is checked against the reference end to end. + +```sh +mkdir -p shared/gltest && cp test/gltest/* shared/gltest/ # piglit: shared/piglit (clanker/testing/piglit-on-irix.md) +tools/irix-sh.py run 'cp /shared/gltest/glsuite.sh /shared/gltest/xsweep.sh /usr/tmp/' \ + '(nohup sh -c "sh /usr/tmp/glsuite.sh gpu; sh /usr/tmp/xsweep.sh gpu" > /usr/tmp/suite.out 2>&1 &)' +test/gltest/glsuite-compare.sh shared/results/cpu shared/results/gpu +``` + +The suite copies `/shared/gltest` to `/usr/tmp` and runs the programs from +there, so build them once in the guest (`sh irix.sh` in a copy, see the +`irix-gfx-loop` skill) and copy the binaries back into `shared/gltest`. The whole run takes about 45 minutes, most of it piglit. Short +recordings of a failing case (`vpro rec`) replay on both raster units offline: +`the_raster_units_agree_on_recorded_sessions` replays every recording in +`ignore/recordings/gpu-cases` (or `VPRO_PARITY_DIR`) on the CPU unit, the GPU +unit and the GPU unit with live-style slice ends, and compares what the board +wrote to guest memory and its final memory. + ## Disk persistence on exit `src/exit_persistence_tests.rs` builds every machine (Indy, Indigo2, Indigo2 diff --git a/clanker/CLANKER.md b/clanker/CLANKER.md index cf14e4eb5..fed6d9c87 100644 --- a/clanker/CLANKER.md +++ b/clanker/CLANKER.md @@ -61,7 +61,7 @@ found or what to watch out for while changing code, it goes in `clanker/`. | `clanker/cpu/`, `jitv2/` | CPU architecture findings; the MIPS JIT | | `clanker/devices/`, `scsi/`, `net/`, `video/` | device models, storage, the NAT stack and its services, VINO/IndyCam | | `clanker/vnc/` | the VNC server, the host input layer (`screen`/`ptr`/`key`) and pointer steering | -| `clanker/rex3/`, `gr2/`, `mgras/`, `odyssey/`, `hostgl/` | graphics boards and host GL (`odyssey/` is references only) | +| `clanker/rex3/`, `gr2/`, `mgras/`, `odyssey/`, `hostgl/`, `wgpu/` | graphics boards, host GL and the host-GPU raster path (`odyssey/` is references only) | | `clanker/gui/`, `macos/` | front-ends and platform packaging | | `clanker/snapshot/`, `perf/`, `testing/`, `build/` | snapshots and `iris-ci`, host performance, test harnesses, builds | | `clanker/irix/` | the guest: install guide (`irix-install.md`), network setup, shell quirks | diff --git a/clanker/INDEX.md b/clanker/INDEX.md index c509d8d4d..ea8842f15 100644 --- a/clanker/INDEX.md +++ b/clanker/INDEX.md @@ -254,6 +254,7 @@ Odyssey (VPro V6/V8/V10/V12): the Octane2/Fuel/Tezro graphics board. Modelled on | Note | Kind | Title | |---|---|---| +| [gaps-found-by-the-2026-11-07-sweeps.md](odyssey/gaps-found-by-the-2026-11-07-sweeps.md) | open | Model gaps the full test sweep found on both raster units: CI8 reads return 0, X screen reads over some windows are noise, 10:10:10:2 pbuffers and 12-bit z16 visuals, double-buffered gltest shows only the clear | | [bringup.md](odyssey/bringup.md) | howto | VPro on the Octane: board catalogue, Crossbow code 5, PROM/IRIX minimums, disk images | | [gotchas.md](odyssey/gotchas.md) | rule | VPro gotchas: IMPACT installs lack the driver, textport FIFO cost, the f64 depth path | | [monitor-reads-quiesce-the-board.md](odyssey/monitor-reads-quiesce-the-board.md) | design | `vpro` state commands quiesce the board through `BoardLocks::quiesce` (format under the guard, write after); pixel, screen and memory dumps read board memory lock-free as the display does | @@ -409,6 +410,20 @@ Running IRIX as a guest: install, network setup, shell quirks, X defaults. | [networking.md](irix/networking.md) | howto | IRIX 6.5 Networking Configuration | | [octane-halt-request-outlives-a-quit.md](irix/octane-halt-request-outlives-a-quit.md) | rule | Octane: quitting during the PROM's halt leaves the next boot at "Okay to power off" | +## wgpu/ + +The host-GPU raster path (`src/wgpu`, `src/dev//wgpu`): what the kernels may assume of a GPU, and how they are held to the CPU models. + +| Note | Kind | Title | +|---|---|---| +| [gpu-floats-are-not-the-hosts.md](wgpu/gpu-floats-are-not-the-hosts.md) | rule | GPU float division is not correctly rounded and products get fused: `div_exact` and the multiply-by-one guard before comparing with the CPU model (measured on RADV) | +| [vram-accesses-are-atomic.md](wgpu/vram-accesses-are-atomic.md) | design | Board memory on the GPU is accessed with atomics, because two pixels can share a word | +| [the-mirror-and-its-dirty-maps.md](wgpu/the-mirror-and-its-dirty-maps.md) | design | The GPU unit's board memory has a CPU mirror; dirty maps both ways keep them in step, and the front end must wait before it touches memory | +| [read-storage-in-place.md](wgpu/read-storage-in-place.md) | rule | Read records, slots and tables from their buffers in place: a 4.8 KB struct copied by value gave silently wrong texels on RADV once the kernel grew | +| [swaps-wait-on-stream-positions-not-fences.md](wgpu/swaps-wait-on-stream-positions-not-fences.md) | design | A swap latches once the GPU unit has submitted the stores before its arm (stream positions, no CPU wait); polled idle words could follow the same scheme (open, minor) | +| [window-id-bytes-are-written-through-to-the-mirror.md](wgpu/window-id-bytes-are-written-through-to-the-mirror.md) | rule | The front end reads window-id bytes without a wait, so the GPU unit's id and overlay byte stores are made in the mirror too; found with replays that compare both raster units | + + ## build/ Toolchain, dependency upgrades, platform build issues. @@ -425,6 +440,7 @@ Plans, task briefs and reviews written for agents. Each states its own status; m | Note | Kind | Title | |---|---|---| +| [wgpu-ui.md](plans/wgpu-ui.md) | plan | The iris window on wgpu: a presenter in `src/wgpu`, the board's device shared, VPro frames presented with no readback, then Newport's compositor in WGSL | | [cow-chd-sync-plan.md](plans/cow-chd-sync-plan.md) | plan | Plan: copy-on-write protection + "Syncing CHD file…" apply-on-shutdown | | [gui-benchmark-plan.md](plans/gui-benchmark-plan.md) | plan | Benchmark in iris-gui | | [iris-daynaport-target.md](plans/iris-daynaport-target.md) | plan | Task: add a DaynaPort SCSI/Link target to the IRIS emulator | @@ -433,6 +449,7 @@ Plans, task briefs and reviews written for agents. Each states its own status; m | [net-stack.md](plans/net-stack.md) | plan | The NAT network stack: analysis and redesign plan | | [odyssey-architecture-review.md](plans/odyssey-architecture-review.md) | plan | The VPro (Odyssey) model after the merge: data path, threads, debug tooling vs the other boards, T&L overlap, float vs fixed, texture specialisations, JIT feasibility, quad scanline raster | | [odyssey-quad-pipeline.md](plans/odyssey-quad-pipeline.md) | plan | Plan: a three-stage quad pipeline for the VPro rasteriser (rasteriser, shader, back end on threads; quake1 timedemo as the workload) | +| [wgpu-raster.md](plans/wgpu-raster.md) | plan | Plan: the boards' raster units on the host GPU through wgpu compute (board memory on the GPU, tile-binned batches; common layer landed, VPro first, IMPACT second, GR2 open) | | [odyssey-board-memory.md](plans/odyssey-board-memory.md) | plan | VPro board memory as the hardware has it: one tiled SDRAM pool addressed by base/pitch/format, replacing the fixed planes and the invented object store; measure the tile layout on a real V10 first | | [pci-dma-port.md](plans/pci-dma-port.md) | plan | PCI bus masters: one `PciDma` trait, shared on the O2, per device on the Octane's Bridge; Bridge/Xbridge DMA, PIO, interrupt and swap facts; step 1 done | | [o2-memory-and-locks.md](plans/o2-memory-and-locks.md) | plan | O2 disk I/O gap vs Indy: interrupt re-drive on every access (~205 locks), PCI BAR decode locks, debug logs on hot paths, RAM onto Physical/SparseDeviceMap/PpMemory like the Indy, remaining locks; hardware facts on memory masters | diff --git a/clanker/odyssey/gaps-found-by-the-2026-11-07-sweeps.md b/clanker/odyssey/gaps-found-by-the-2026-11-07-sweeps.md new file mode 100644 index 000000000..c2de56c73 --- /dev/null +++ b/clanker/odyssey/gaps-found-by-the-2026-11-07-sweeps.md @@ -0,0 +1,43 @@ +# Gaps the 2026-11-07 test sweeps found (both raster units) + +Status: open, 2026-11-07. Measured on the V12 model, IRIX 6.5.30, with +`test/gltest/glsuite.sh` and `xsweep.sh` (results compared with +`test/gltest/glsuite-compare.sh`). Every item fails the same way on the CPU and +the GPU raster unit: these are model gaps, not GPU regressions. + +- **Colour-index 8 reads return index 0.** `pixtest --ci 8`: 9 pass, 32 + fail, starting with `ci8/baseline/rects` and `ci8/baseline/clear` (index + 0 read where 17 and 5 were drawn). `scrsave` of a PseudoColor 8 root + (`xsweep.sh` mode `pc8`) is almost entirely black for the same reason. + The screen shows the right pixels, so the draw is right and the read + (format choice or packing for 8-bit index windows) is wrong. +- **X screen reads over some windows are noise.** `scrsave` of the + TrueColor 24 desktop is right except inside the icon catalog's panel; + over GL windows (the glprim scenes) the window area is streaks. The GL + readbacks of the same windows are right, so it is the X server's + `XGetImage` path (its reads through the board), not the drawing. +- **Visuals that fail on both units:** 10,10,10,2 pbuffers (4 cases) and + 12,12,12,12 with a 16-bit depth buffer (4 cases) in `vissweep.sh`: the + 12-bit z16 window draws nothing (clear colour and clear depth read back). +- **`gltest --db --spin` shows only the clear.** Double buffered, the + window shows the clear colour and never the triangle, on both units, + while the single-buffered run shows it. Swaps happen (the displayed + buffer bit toggles). +- **`cicopy`**: the colour-index back-to-front copy to another position + leaves the destination at the clear index (`front, copied from back`). +- **glstate**: 5 failures (fog index, index write mask, index clear value, + `PIXEL_TEXTURE_SGIS` enable), as before. +- **pixtest RGBA**: 53 failures, the ones the first V10 comparison listed + (`v10-vs-emulator-first-comparison.md`) plus the texture RGBA8 signed + types. +- **TyrQuake started from a telnet shell** (`+timedemo demo1`) stays on its + console and draws nothing, on the GPU unit at least; started from the + desktop it plays (the user's runs). Not chased. + +Also seen: `texsweep.sh` read back a lower-precision texture for one or two +of the 12- and 16-bit luminance and intensity formats on some runs only +(different formats each run). Replaying the run's recording on the CPU unit, +the GPU unit and the GPU unit with slice ends gives byte-identical host +writes, so the boards agree on what they were sent; the guest's GL library +sent something different that run (a smaller internal format, by the look of +the 4-bit steps). Recording: `ignore/recordings/gpu-cases/texsweep.rec`. diff --git a/clanker/odyssey/monitor-reads-quiesce-the-board.md b/clanker/odyssey/monitor-reads-quiesce-the-board.md index f655cdc50..b1e927596 100644 --- a/clanker/odyssey/monitor-reads-quiesce-the-board.md +++ b/clanker/odyssey/monitor-reads-quiesce-the-board.md @@ -1,6 +1,6 @@ # Monitor reads of VPro state quiesce the board; pixel reads go lock-free -Status: design, 2026-11-02; pixel side updated 2026-11-04 for board memory. +Status: design, 2026-11-02; pixel side updated 2026-11-04 for board memory, and 2026-11-07 for the GPU raster unit. Implemented in `src/dev/odyssey/debug.rs`. ## The problem @@ -41,6 +41,13 @@ UB, not a stale number. through `SharedRegs` (atomics), exactly as the display thread does, so they never stall the guest. A pixel read under a draw in flight may be a frame stale, which is what a screenshot is anyway. +- **On the GPU raster unit** (2026-11-07) board memory lives on the GPU and + `DisplayShared::vram` is only its mirror, current after a readback. So + there the pixel commands quiesce briefly to take one (`gpu_sync`) and read + the mirror as it is if the board stays busy; the state commands sync under + their guard. Without it `vpro shot` composed garbage and `vpro pix` showed + stale pixels. `quiesce` also waits for the board thread's slice to end, + since the GPU tick runs at its tail. ## Why not quiesce the pixel commands too diff --git a/clanker/plans/wgpu-raster.md b/clanker/plans/wgpu-raster.md new file mode 100644 index 000000000..2f4a75e4e --- /dev/null +++ b/clanker/plans/wgpu-raster.md @@ -0,0 +1,533 @@ +# Plan: the boards' raster units on the host GPU (wgpu compute) + +Status: 2026-11-06, branch `wgpu-raster`, in progress. Milestone 0 landed: +the common layer in `src/wgpu` (device, shader assembly, batch recorder, +bin and raster kernels, GPU memory with readback) with an end-to-end test +board that matches a CPU reference word for word on the RX 6600 XT. +Milestone 1a landed: the VPro raster unit itself (`src/dev/odyssey/wgpu`: +records, slots, the triangle, fragment and clear kernels over the model's +word rule, the generic back end) matches the CPU pipeline word for word in +eight parity tests (`cargo test --features wgpu --lib odyssey::wgpu`), +after two GPU facts were guarded (`clanker/wgpu/`). Milestone 1b landed: +the unit is in the board (`[graphics] raster = "wgpu"`; `VPRO_GPU=1` for +the test boards, `VPRO_REPLAY_GPU=1` for the replay), with the CPU `Vram` +as a mirror the unit keeps current +(`clanker/wgpu/the-mirror-and-its-dirty-maps.md`); every golden scene and +the quake1 replay's checkpoints pass on it. Next: the front end's memory +paths as records (DMA puts and image puts from a staged upload buffer, +gets as staged readbacks, copies as records), then texture on the GPU. + +## Goal + +Replace the software raster units of IMPACT (RSS engines), VPro (the +quad pipeline) and, if it ever pays, GR2 (RE3) with compute kernels on +the host GPU through wgpu. Board memory lives on the GPU and nothing on +the CPU maps it: every CPU read of it is a readback. The front ends stay +as they are: command decode, state tracking, the HLEs and triangle setup +remain CPU work; the boundary is where a primitive reaches the raster +unit today. Bit exactness is the acceptance test, through the existing +replay hashes and goldens, so the kernels are the models' arithmetic +again, not a host-GL approximation (that path was tried for VPro and +retired; see `odyssey-quad-pipeline.md`). + +Newport (REX3) is out of scope: its primitives are spans, and no batching +amortises a GPU round trip over a span. See the `rex-gpu` replay prototype +for what that looks like. + +## Why compute, and why nothing is mapped + +- The host rasteriser cannot be used for coverage plus a shader for the + rest: two fragments of different primitives on one pixel have no + ordering guarantee against a storage buffer. The fixes (fragment + shader interlock, rasterizer ordered views, raster order groups) are + not in wgpu and not on every driver. Depth has the same problem: the + boards' depth words and interpolation are not the host's. +- Hardware samplers cannot read the boards' layouts (16x16 tiles, per + engine pages, TRAM bank bit) or reproduce the 12-bit filter math. Texels + are gathered by the kernel, as `sample_quad` does today. +- A compute rasteriser needs storage buffers, atomics and indirect + dispatch and nothing else, which is wgpu's portable core. It runs + unchanged on Metal and D3D12 through naga, and identically on lavapipe, + so CI can run the goldens with no GPU. +- Memory stays unmapped because every path that touches it on the CPU + side already waits on the raster unit first (`raster_wait`, `drain_rss`, + `drain_re3`); those waits become "flush the batch, fence, read what + you need back". No ownership protocol, no imported memory, no page + tracking. The CPU rasteriser remains the other instantiation of the + same board, chosen at machine build (const generic or trait, below), + never a per-draw fallback. + +## The common layer (`src/wgpu`, landed) + +| Piece | What it is | +|---|---| +| `device.rs` `Gpu` | adapter, device, queue, adapter-sized storage limits, optional features (subgroups, int64, int64 atomic min/max, timestamps) recorded in `Caps`; async errors captured for `check()` | +| `shader.rs` `Source` | WGSL from `include_str!` fragments with `#include "name"` and `define`d constants; compile errors come back as text with the composed source numbered | +| `tiles.rs` | 8x8 tiles, `MAX_PRIMS` 4096 a batch, 128 mask words a tile, `TileGrid::tile_box`, the header and list layouts shared with `shaders/common.wgsl` | +| `batch.rs` `Batch` | the recorder: `slot(S)`, `prim(R, pixel box)`; `flush` uploads records, slots, boxes and header, resets the busy count and the indirect arguments, dispatches bin then raster, returns the submission | +| `batch.rs` `BatchPipelines` | builds both kernels around a board's `ArchShaders`: its raster hook fragments, an optional `bin_accept`, its defines, its bind group layouts for groups 2 and up | +| `memory.rs` `GpuMemory`, `Staging` | a board memory: device-local storage buffer, `write`, `clear`, `read` (copy to a staging buffer, wait, map); `Staging` is the readback buffer any GPU buffer can be read through | +| `scanout.rs` `ScanoutPipeline`, `Scanout` | the display's compose: a board's `scanout_pixel(x, y)` kernel over its memory into a frame on the GPU, one lane a pixel, read back for the window; group 0 the frame, group 1 the board's display slot, groups 2 and up the board's memory | +| `shaders/bin.wgsl` | one workgroup per primitive, lanes over its tile rows: `atomicOr` the primitive's bit into each tile's mask, append first-touched tiles to the busy list, raise the indirect arguments | +| `shaders/raster.wgsl` | one workgroup per busy tile, dispatched indirectly, one pixel a lane: `arch_tile_begin`, then every set bit in mask order through `arch_prim`, then `arch_tile_end`; clears its mask words and busy flag on the way | + +Rules the kernels rely on: + +- **One writer per tile.** Only the tile's workgroup stores to its pixels + during a batch, so primitive order per pixel is the loop order and no + atomics touch memory. Two pixels in one word (overlay cells, 16-bit + pixels) are handled inside the group. +- **Order is the bit position.** Setting bits commutes; the busy list's + order does not matter; the records array is in submission order. +- **Hooks run in uniform control flow.** The tile index and each mask + word are read with `workgroupUniformLoad`, so hooks may use barriers + and subgroup quad operations as long as every lane takes the same + branches and only stores are gated on coverage. +- **Bindings.** Group 0 is the batch; group 1 is records (binding 0) and + slots (binding 1); groups 2 and up are the board's. The bin pass binds + the indirect arguments at group 0 binding 5 through its own layout: a + buffer cannot be read-write storage and the indirect source in one + dispatch (wgpu validation, found in milestone 0). +- **Workgroup grids cap at 65535.** The busy count folds over x and y; + the bin dispatch is one group per primitive, under `MAX_PRIMS`. + +## What a board provides (`src/dev//wgpu`) + +- A setup record type `R: Pod` and a slot type `S: Pod`, laid out as its + WGSL declares them. A record names its slot; a slot is the stage state + as register words, decoded by the board's WGSL functions, so the + kernel adapts to the front end without the common layer knowing it. + Each board has its own shader set; shared helpers (edge functions, + fixed-point, Bayer tables) go in `src/wgpu/shaders` as library + fragments when two boards want them. +- One raster shader per primitive kind is the aim (triangle, line, point, + rect or span, pixel list). Until the bin pass can carry a per-record + kind to a per-kind dispatch, `arch_prim` switches on a kind field in + the record; the switch is uniform per workgroup so it costs a scalar + branch. +- Its memory as one or more `GpuMemory` (VPro: one; IMPACT: pixel pages, + CID planes, TRAM), bound from group 2 up. +- The batch boundary rule: close and fence when the front end would + observe memory, at frame end, on `Full`, and on a time cap (Windows + removes the device after a two-second dispatch). +- Readback helpers for every CPU-side reader the board has (listed per + board below), each a flush, a wait and a `GpuMemory::read` of the + bytes concerned. +- Scanout as a kernel into a frame buffer on the GPU, read back into the + board's `Rex3Screen` (5 MB a frame at 1280x1024) until the GUI shares + the device. + +Selection: a const generic on the board (`Board` or a +raster-unit trait with two implementations), instantiated when the +machine is built from `[graphics] raster = "cpu" | "wgpu"` and +`[wgpu] backend`. The CPU instantiation keeps its code; nothing is +decided per draw. + +## VPro (Odyssey), first + +The boundary is the raster thread's job: `GlDrawJob` becomes a +`PipeDraw` and `pipe::setup::run` walks it. The GPU instantiation turns +the same inputs into records and slots instead. + +- **Records.** Filled, non-smoothed triangles already travel as setup + records (`setup::Tri` with `raster::Triangle`: exact i64 edge values + at pixel (0,0), per-pixel steps, owning-edge flags, area, 12.12 vertex + colours, S/W..Q/W per vertex, z and divisor over w per vertex, inverse + w, user-plane distances). The GPU record is that struct narrowed to + what the kernel reads, i64 edges split or reduced (a 1/256 subpixel + grid over 65536 pixels fits i32 edge *steps*; the start values need + care). Clears (`ClearDraw`, `push_rect`) become rect records. Lines, + points, smoothed primitives and polygon modes are emitted today as + single shaded fragments through `Draw::fragment` (depth word, 12-bit + colour, coverage); they become a *fragment list* record kind (a box and + a range into a fragment buffer) first, and WGSL walkers later. +- **Slots.** `FragmentWords` (raster 0x404/0x415/0x458/0x81d/0x841/ + 0x204b/0x840/0x84a/0xc48/0x3c4e/0x43f/0x420; shader 0x404/0x181a/0x41e; + back end 0x404/0x420/0x452/0x41f/0xc21/0x422/0x1017/0x1023/0x424 and the + decoded `DestViews`) plus the draw-state slot's texture registers + (`TexRegs`: path, sampler, env, lod, level 0, 192 level words, scale and + bias, offset, colour tables, detail) and its `Latch`. The 64 draw-state + slots and the latch rule stay on the CPU; a GPU slot is written when a + draw issues a slot the batch has not seen. +- **Memory.** `Vram` (power of two, 128 MB on a V12) as a `GpuMemory` of + u32 words. The model's word is a u64 whose lowest address is the + highest byte; the WGSL accessors reproduce `VramPtr::load/store(a, n)` + bit for bit on the u32 pair, so hashes of the readback equal + `address_bytes()` today. +- **Back end and shader.** `store_quad`, `fragment_q`, the + packers, `depth_word`, `sample_quad` (tiled texel addressing, 8x8x2 + windows, 8-bit filter weights, 1/256 lambda, 12.12 output, detail, + post) port function for function. `BackendState::select`'s table + becomes uniform branches on the slot. +- **Other memory readers and writers**, each with its treatment: + + | Today | GPU instantiation | + |---|---| + | `dest_mem()` paths (integer drawing, `image_put`, texture builds, zoom) after `raster_wait` | flush + fence, then GPU copy or scatter kernels from an upload ring; `image_put` is the hot one (Quake's texture bursts) | + | pixel DMA put (`dma_fields` → `Rows::store_word`) | upload ring + scatter kernel | + | pixel DMA get (`dma_read_pack`) | flush + fence + rect readback, then the existing pack | + | `copy_block` | GPU copy kernel | + | `ctx_switch`, `mmio_read` of waiting registers, `reset` | flush + fence; reset clears the buffer | + | display: `Frame::compose` every 16.7 ms over all of VRAM | scanout kernel into a GPU frame; readback into `Rex3Screen` | + | monitor `vpro pix/raw/peek/shot/fbdump/hash` | quiesce as now, then readback | + | fences (`Board::fence`, `query_finish`, completions on the quad ring tail) | `Gpu::on_done` after the batch that precedes them: guest-RAM token write and softsync, as the closures do today | + | replay: `raster_wait` then `memory_hash` | flush + wait, `read_all`, blake3 over the same byte order | + | snapshots | refused for VPro today; unchanged | + +- **Threads.** The raster thread and the pool go away in the GPU + instantiation; the front end records and flushes. `RasterQueue`'s + budget and `wait_room` become the batch's `Full` and a bound on + in-flight batches. + +### What landed (milestone 1a, `src/dev/odyssey/wgpu`) + +- `Rec` (256 bytes): a triangle (the three edge functions as i64 at the + centre of plane pixel (0, 0) plus their steps, the owning-edge bits, + the inverse area, the z and divisor planes as `depth4` takes them, + 1/w at the vertices, the depth range and slack, the 12.12 vertex and + provoking colours, the offset steps, the clipped plane box, the window + origin, the alpha test's words), a CPU-shaded quad (`BackQuad`: plane + position, coverage, depth words, 12-bit colours) or a clear's + rectangle (its values and masks). `Slot` (368 bytes): the back-end + words (0x404, 0x420, 0x452, 0x41f, 0xc21 x3, 0x422, 0x424) decoded in + WGSL as `BackendState::of` decodes them, the draw facts the words lack + (stencil planes, index mode, overlay context), the six destination + views packed, the clip rectangles and the id-compare word, the polygon + stipple and its background. +- Kernels: `vram.wgsl` (the u64 word rule on u32 pairs, atomics), + `views.wgsl` (tiled addressing, RGBA8 / Z24 / byte fields), + `backend.wgsl` (`fragment_q` generically: stencil, depth, blend with + every factor and equation, logical operations, colour and index masks, + both targets; `clear_pixel`), `vpro.wgsl` (the hooks: the walk's + bounds, exact edges, `owned`, depth, stipple, flat or + perspective-correct weights, the alpha test). Shared: + `src/wgpu/shaders/i64.wgsl` and `float.wgsl`. +- `GpuRaster`: `push(rec, &slot)` with slot reuse and flush on `Full`, + `flush`, `wait` (= `raster_wait`), `write_words`, `read_words`, `hash` + in `address_bytes` order. +- Parity: random triangles of every size (edge values past 2^24 exercise + the i64 path), flat and smooth, alpha tests, stipples with and without + a background, forty random back-end states (depth functions and + writes, blends over the whole factor table, logic ops, colour masks, + stencil ops and masks, both targets, depth ranges, offsets), clears + under id tests and masks interleaved with triangles and shaded quads, + index draws into the 2-byte overlay surface. Whole memories compared. +- Every colour field the model has: RGBA8, Z24, bytes, the 12-bit + lanes and the 2- and 4-byte texel codecs (`views.wgsl` ports + field.rs). Not yet on the GPU: texture, fog, the separate specular, + user clip planes (f64 on the CPU), lines, points and smoothed + primitives (they arrive as shaded quads, which the GPU stores). + +Measured on the way (`clanker/wgpu/gpu-floats-are-not-the-hosts.md`): +RADV's division is a reciprocal and a multiply, and it fuses multiply-adds; +both broke depth-word parity until `div_exact` and the multiply-by-one +guard went in. The quad pipeline's "fixed point for the rest" would remove +the need for both. + +### Milestone 1b (landed): the unit in the board + +- **Where it sits.** `RasterCtx.gpu: Option>` + (gpu_unit.rs aliases it; without the feature the type is uninhabited + and the dispatch folds away). With a unit no raster thread starts, so + every job runs inside its submit on the front end (`RasterQueue`'s + unthreaded path, unchanged). `PipeDraw` carries the unit's pointer and + the draw's `Slot`, built in `PipeDraw::new` from the back-end words, + planes, index and overlay flags, the clip list and the stipple; + `emit` becomes `push_frag`, `clipped_triangle` pushes a triangle record + for a plain fill (no texture, fog, specular or user planes) and walks + and shades the rest on the CPU into fragment records; `gl_raster_ops` + pushes rectangles. A draw whose planes the kernels cannot address falls + back to CPU stores on a mirror brought current first (none left since + the coded fields landed). +- **Memory.** The board's `Vram` is the unit's mirror: CPU stores mark + 16 KB blocks (`DirtyBlocks` behind `VramPtr::store`), uploaded before + each batch; records mark the blocks their boxes touch in their slot's + views, read back at `raster_wait` into the mirror. So the front end's + own paths (integer drawing, image puts, copies, DMA, context, the + texture unit's samples, hashes and dumps) run unchanged and in order, + at the price of a wait and a readback whenever they run with drawing + pending. The board thread also brings the mirror current once a frame + (`gpu_display_tick`) so the display follows the drawing until a + scanout kernel feeds it. +- **Selection.** `[graphics] raster = "cpu" | "wgpu"` and + `gpu_backend`, through `Odyssey::new` to `Board::enable_gpu`, which + must run before the threads start. A runtime `Option`, not yet the + const generic: the dispatch points are a handful and coarse. +- **Checks.** The parity tests (nine, coded fields included), the 76 + golden scenes with `VPRO_GPU=1`, and the quake1 replay with + `VPRO_REPLAY_GPU=1`: all 165 checkpoints, bit for bit. +- **Measured** (quake1 replay, RX 6600 XT, release, 2026-11-06): + 925 M records in 226,779 batches (4,080 a batch: the batches fill), + 265 MiB uploaded, 20,149 MiB read back in 1,869 readbacks (10.8 MiB + each, the frame's blocks), 515 s wall against the CPU pipeline's 152 s + with two workers. The records are almost all fragment quads from + CPU-shaded textured triangles (one record a quad: `MAX_PRIMS` is the + bottleneck, not the GPU), and the readbacks are the image puts and + DMA between frames. Those two numbers are what milestone 2 (texture on + the GPU, fragment lists) and the record conversions below must bring + down. + +- **Where the time goes** (the first 10 M records, 21 checkpoints, + release, 2026-11-06; `VPRO_REPLAY_LIMIT` with `VPRO_REPLAY_LAZY=1` + skipping the readbacks between checkpoints): + + | | wall | readbacks | + |---|---|---| + | CPU, one thread | 46.1 s | | + | CPU, two workers | 18.3 s | | + | GPU unit | 59.7 s | 233 (2.7 GiB) | + | GPU unit, no readbacks | 58.1 s | 20 | + + The readbacks are 3 % of it. The GPU path loses on the CPU side: 107 M + fragment records (one 256-byte record per CPU-textured quad, 26,270 + full batches), built, uploaded and binned, on top of the CPU texturing + itself. Texture on the GPU removes the stream and the shading; that + is milestone 2 and comes before the DMA conversions. A compact + fragment list (quads in a side buffer, one record per CPU walk tile) + stays worth having for lines, points and smoothed polygons. + +### Milestone 2a (landed): texture on the GPU + +`shaders/texture.wgsl` ports the texture unit function for function: +the quad's positions from its four pixels' barycentrics (each lane +recomputes its quad's edge values, so no subgroup operation is needed), +the diagonal level of detail through `fixed::log2_q8`'s table (generated +into the shader), the filter and levels lambda chooses, the taps wrapped +and gathered from board memory with every texel codec, the 8-bit filter +weights, the mip blend and the 12.12 conversion with their 64-bit +products split, the detail image, the scale and bias and the colour +tables, then the environment (`apply12x4`) and the separate specular. A +`TexSlot` (4.8 KB) is the draw's registers the unit reads at sample time +and the CPU's `Latch` (levels, LOD clamps and scales as the CPU computed +them, so no `exp2` runs on the GPU); slots are deduplicated by hash per +batch in their own buffer, and the record names its slot. The record +grew to 352 bytes for the texture values and specular colours. + +Checks: `textured_triangles_sample_as_the_cpu_unit` (random 1D, 2D and +3D textures, every wrap, filter, class and environment, borders, detail, +LOD limits, clamps, scale and bias, colour tables, specular; whole +memories compared) and the 23 textured golden scenes, all exact. Still +on the CPU, stored by the unit: fog (f64 on the CPU) and user clip +planes (f64), lines, points and smoothed polygons. + +Found on the way: the CPU unit's `filter` read the second row of a +one-row window for a 1D LINEAR texture and multiplied it by zero; the +read is now skipped (no value changes). + +### Milestone 2b (landed): image puts as blit records + +`paint_pixel`, which every image put and integer primitive ends in, +hands its pixel to the unit when it is active: `GpuRaster::blit_pixel` +collects consecutive pixels of a row into one blit record (`REC_BLIT`) +whose words sit in a staged buffer uploaded with the batch, under the +slot's view, a mask for the byte planes, the window's tests already done +on the CPU (which reads the id bytes without waiting: they are the X +server's, never the raster unit's). Any other thing the unit does first +closes the run, so the pixels keep their place in the order. A +depth-tested image's pixels go through the unit as fragments under a +slot latched with the box. A texture or scratch image through the FIFO +(`image_put` into a scratch destination) stores into the mirror untracked +and the unit uploads its rectangle when it closes +(`GpuRaster::note_region`): the records before it are submitted first and +the upload lands before the next batch, so nothing waits. + +The mip-level builder (`texture_build_level`) stores into its level the +same way. Found on the way (`clanker/wgpu/read-storage-in-place.md`): +the texture slot copied by value into private memory gave wrong texels +once the kernel grew by one dispatch arm; records, slots and texture +slots are now read from their buffers in place, and the log2 table lives +in a buffer. + +Measured after these (2026-11-06): the first 10 M records in 5.9 s with +47 readbacks (874 MiB: 20 checkpoints, 18 context switches, 8 copies, +1 clear), against 5.3 s with none; the whole demo, 165 checkpoints +exact, in 63 s before the scratch and builder changes (CPU: 152 s with +two workers). The remaining time is the front end itself; the next +readbacks to remove are the context switch and the copies, and the +display's once the scanout runs on the GPU. + +### Milestone 3a (landed): the scan-out on the GPU + +`src/wgpu/scanout.rs` is the common driver: one lane per screen pixel +calls the board's `scanout_pixel`, into a frame buffer on the GPU read +back through `Staging`. The VPro kernel (`odyssey/wgpu/shaders/ +scanout.wgsl`) is `Frame::pixel` and `cursor_row`: the window-id byte, +the overlay, the window's display mode and its field decoded through +`load_kind`, the gamma tables or the lookup RAM, the cursor or crosshair. +Its `DisplaySlot` (38 KB a frame) is what `Frame::snapshot` resolves on +the CPU: surfaces, per-id decisions and fields, the overlay table, the +gamma tables, the cursor registers and the resolved LUT. The display +thread owns a `VproScanout` over a clone of the memory buffer +(`DisplayShared::gpu_scanout`) and composes with it in +`ScanoutReader::update`, falling back to the CPU compose on an error; it +no longer reads the mirror, so the 16 ms mirror readback is gone. Because +the kernel reads the GPU's memory, the front end now sends the mirror's +dirty blocks up at every flush, records or not, and once per run of +stores (`gpu_upload_tick`), so CPU-drawn pixels reach the screen. The +first live boot showed the other half of that: records and blit runs +stayed in the open batch until it filled or something waited, so the PROM +text and X painting reached the screen late or not at all. `GpuRaster::tick` +now submits the open batch at the end of a run of stores when no store +waits (the board going idle) or when the batch has been open 4 ms +(`MAX_OPEN`); a GL game fills batches faster than that and pays nothing. +`wait` also skips the device poll when nothing was submitted since the +last wait, since a poll with no index waits for the display's scanout too. + +Swap ordering (the user's design, 2026-11-07): rather than fences the +display waits on, the model uses stream positions. `CommandFifo` counts +stores `pushed` and `retired` (absolute, monotone; the ring's indices +wrap). A swap arm or swap-control write records `pushed` at the time +(`HostInterface::swap_after`, set from `stream_pos` by the bus path), the +GPU unit's `tick` takes `retired` and publishes it to +`DisplayShared::covered` once everything recorded up to it is submitted +(at every flush, or at a tick with nothing recorded), and +`HostInterface::retrace(covered)` latches flips only when +`swap_after <= covered`. Submitted is enough, not completed: the display's +scanout submission queues behind the batch on the same queue. The CPU unit +leaves `covered` at the maximum (its own raster-thread lag is the +pre-existing behaviour). Test: +`a_flip_waits_until_the_raster_unit_covers_the_stores_before_its_arm`. +Still to do from the same idea: make the status words the guest polls for +idle (DMA busy, the sync words) report "busy until the fence completes" +instead of `raster_wait`'s flush and wait; that needs a completion counter +(`on_submitted_work_done` per submission) and the registers' busy bits. +`vpro gpu [trace on|off]` prints the unit's counters live, with readbacks by +cause when traced. +Checks: `scanout_composes_as_the_cpu_frame` (random modes, LUT, overlays, +cursors and crosshairs over random memory, whole frames compared) and the +22 display tests with `VPRO_GPU=1`. + +The readback stays until the GUI runs on eframe's wgpu renderer and can +present the frame as a texture on the same device; that move, with the +Newport GL compositor ported to WGSL on this driver, is the next project +for the display. + +### Milestone 3b (landed): the front end stores through the unit + +The direction (2026-11-06): no more work on syncing the mirror; what +draws goes through the unit, and the mirror comes back only where a +reader needs bytes (a DMA get's rectangle, the hashes, the monitor, +`dest_mem` for the few general paths left). + +- Blit records carry a logical operation with the result's and/or + masks (`paint_integer`'s index or RGB fold), the plane mask for byte + fields, and optionally their own view (a DMA put's surface). So + `paint_integer`, `paint_aux` (the X server's window-id and overlay + writes), `clear_planes`, `buffer_store`, screen `dma_put`s and the + depth and stencil image writes all store through the unit, no CPU read. +- `Board::raster_wait` on the GPU unit is a flush and a GPU wait, no + readback: the register reads that wait (`mmio_read`), the context + switch (it touches no board memory) and the slot waits cost no memory + traffic. `planes_in(box)` reads back only the blocks the drawing marked + under a box, and nothing when none is marked; `dest_mem` (the general + path) and `gpu_sync` bring everything back. Test helpers that read the + mirror after a wait call `gpu_sync`. +- The region readback (`GpuRaster::read_back_view`) is also how a DMA + get stages its rectangle before the existing pack writes it to the + guest, and how a copy's rectangles come back before the CPU copy. The + copy kernel (gather, then scatter with the codecs, flip, byte masks and + the scale-and-bias filter, submitted between batches) is the next + conversion; after it the mirror is written by nothing hot. + +An aside from the same day: a replay run that shared the GPU with a +live IRIS session doing whole-screen readbacks hit an amdgpu ring timeout +and reported 41 GiB of uploads; neither reproduced on an idle GPU (10 M +records: 148 MiB uploaded, 4 MiB of them dirty blocks, 16 MiB regions). + +### Next: copies as a kernel, then the walkers + +Texture and scratch uploads (`dma_to_buffer` into memory no drawing +touches) no longer wait: they store through an untracked pointer and +upload their own tiles (`GpuRaster::upload_region`; the rule is in +`clanker/wgpu/the-mirror-and-its-dirty-maps.md`). + +The direction (set 2026-11-06): a DMA *get* stages its rectangle in +front-end memory (a readback of just that span) and writes it to the +guest through the bus and DMA abstraction as now; a DMA *put* or image +put becomes a blit record whose pixels sit in a staged upload buffer the +kernel fetches from, so it joins the batch in order and nobody waits; +copies become copy records. Each conversion removes one class of wait +from the replay's readback count, which is the measure. After that, +texture on the GPU (milestone 2) ends the last CPU shading, and the +scanout kernel (milestone 3) ends the display's readbacks. + +Milestone 2 (the rest): fog and the user planes once the quad pipeline +has them in f32 or fixed point, a compact fragment list for lines and +points, scanout on the GPU; the quake1 timedemo's 165 hashes on the +Radeon and on lavapipe. Milestone 3: live guest (fences, DMA, monitor, +display readback). + +## IMPACT (MGRAS), second + +The boundary is the raster FIFO: register writes with an execute bit, +plus the out-of-band entries (PIO advance, DMA line with payload, GL +enter and leave, exit). The GPU instantiation is one engine that owns +every row (the layout still follows DRBsize, as a single real engine's +does): it keeps the 0x400-word register file and `execute()`'s decode on +the CPU and emits a record per primitive instead of drawing. + +- **Records** per IR kind: block fill and character stipple (corners from + 0x046/0x047, colour by mode, stipple bits), X line and point, GL + triangle (`TriSetup`: biased float positions, s.24 slopes, 12.12 colour + planes, z.12, 2^32 texture iterators, fog), GL line (`GlLineSetup`), + transfers in (a line of pixels from the upload ring with the + `XFRMODE` format). `rss_jit`'s `PipeKey` and `RasterCtx` builders are + the slot and record content already; the WGSL mirrors + `compiler.rs`'s emitters as that mirrors the interpreter. +- **Memory.** `PixMem` (48 MiB of u64 words, `Buffer::locate` with + per-engine regions, 192x16 pages, overlay cells of 9 bits), the CID + planes (4 MiB) and TRAM (4 MiB, nibble cells, border page) as three + `GpuMemory`s. 12:12:12 (`deep36`) is in scope from the start; it is + the interpreter-only case today. +- **Other readers and writers:** display `Frame::snapshot` through + engine 0 (scanout kernel + readback); PIO reads and `dma_read_line` + after `drain_rss` (flush + readback); DMA lines in (upload ring: + TE loads to TRAM, transfer puts to pages); host GL composite + (`hostgl`, writes pages directly: a write); `state_bytes` for + recordings (readback of the regions it hashes); monitor commands. + Multi-engine TE synchronisation disappears with the second engine. +- **Test harness:** the desktop recordings in `ignore/mgras-golden/` + and the gr4-jit sweep (two boards, random state, word-for-word + pages, CID and stipple positions) rerun against the GPU board. + +## GR2 (XZ / Extreme), open + +The boundary is `Re3Sink` into `re3_fifo` (one RE3; the GE7 only stores +microcode, all geometry is HLE in `hq2.rs`/`gl.rs`): register writes +with IR execute (SHADED, FLAT, FLAT4 spans with iterators in 14-bit +fractions), the private ops (copy, Z fill, pixel format, Z control, +stencil, blend, alpha) and reads after `drain_re3`. Memory is two +1280x1024 u32 planes. Spans are short and a frame has few of them, so +the batch amortisation argument is weak; decide after IMPACT with +numbers from the span counts of a GL scene. The REX3 lesson applies +until then. + +## Measurements to take before each milestone closes + +- Batch count and primitives per batch on the quake1 replay (how often + the front end observes memory). +- Bin and raster pass time (timestamp queries) against the CPU pipeline's + tile time (`quake-replay-is-fragment-bound.md`: 151.6 s wall at 2 + workers). +- Readback bytes a frame (display, DMA gets) and their wait time. +- Hash parity on radv and on lavapipe. + +## Open questions + +- **Float parity.** Settled for add, multiply, divide and the int + conversions (`clanker/wgpu/gpu-floats-are-not-the-hosts.md`); the + texture unit's `sqrt` and `log2` paths, when they arrive, need the same + treatment (an exact `sqrt` the same way as `div_exact`; `log2` is a + table on the CPU already). The quad pipeline's "fixed point for the + rest" would make the guards unnecessary. +- **Atomics for every memory access.** Correct and simple; the cost is + unmeasured (`clanker/wgpu/vram-accesses-are-atomic.md`). +- **Fragment lists.** One record per emitted fragment would exhaust + `MAX_PRIMS` on a text-heavy frame; the fragment buffer with range + records is the first design, the WGSL walkers the second. +- **Per-kind dispatch.** Whether a per-record kind in the bin pass and + one raster pipeline per kind beats a uniform switch inside one kernel. +- **GUI.** iris-gui is on glow. Readback first; eframe's wgpu backend and a + shared device when the raster path has earned it. +- **Trusted shaders.** wgpu bounds-checks storage accesses; turn it off + per kernel (`create_shader_module_trusted`) once the goldens pass. diff --git a/clanker/plans/wgpu-ui.md b/clanker/plans/wgpu-ui.md new file mode 100644 index 000000000..d4e5e2eed --- /dev/null +++ b/clanker/plans/wgpu-ui.md @@ -0,0 +1,76 @@ +# The host window on wgpu (iris `ui.rs`) + +Status: steps 1-4 done 2026-11-07. Left: the GL renderer and +`gl_compositor.rs` stay as the default until the wgpu window has run on +macOS and Windows; then the default can move and GL become optional. The iris-gui front end +stays on its software compose and readback; this is the plain `iris` +window only. + +Measured live (V12, GPU raster unit, `--renderer wgpu`): 9106 frames from +power-on to the desktop composed and presented with 0 bytes read back; a +`screen shot` reads back exactly one frame. The display thread used 2302 +CPU ticks to the desktop against 7209 with the GL window. + +## Why + +The VPro composes its frame on the GPU (`VproScanout`), reads it back, +`Display::present` byte-swaps it into `Rex3Screen.rgba`, and the GL +renderer uploads it again as a texture: two copies across the bus per +frame for an image that started on the GPU. On one wgpu device the frame +stays there. The Newport GL compositor (`gl_compositor.rs`) can then move to +WGSL too, and iris stops needing GL at all. + +## Shape + +- **Common, `src/wgpu/present.rs`.** `Presenter`: a `wgpu::Surface` on the + winit window, its configuration (resized on the event thread's request, + recreated on `Outdated`/`Lost`), and one render pipeline that draws a + frame quad letterboxed into the window: nearest sampling at integer + scales, linear otherwise (what the GL path's integer and fallback shaders + do), then the debug overlay (alpha blended) and the status bar under it. + Frame sources: + - `Cpu { rgba, width, height }`: `queue.write_texture` of the visible rows + (Newport's software compose, GR2/IMPACT prebuilt frames, VPro on the + CPU unit). + - `Gpu { buffer, width, height }`: a storage buffer of 0x00RRGGBB words + already on the device (the VPro scan-out's output), copied into the + frame texture by a small compute pass or `copy_buffer_to_texture`; no + readback. +- **Shared device.** `Gpu` keeps its `Instance` and `Adapter`. On Vulkan, + Metal and DX12 the instance needs no display handle, so the window's + surface can come from the board's instance; `adapter.is_surface_supported` + decides. The board's `Gpu` reaches the UI through a new + `GfxDisplay::gpu()` (None for boards without one); otherwise the UI opens + its own. GLES (the `gl` backend) would need the event loop's + `OwnedDisplayHandle` in the instance: not supported at first. +- **Renderer trait.** `WgpuRenderer` implements `Renderer` (the board + display threads keep calling `present`). A new default method, + `present_gpu_frame(buffer, w, h, overlay, status, stats) -> bool`, lets the + VPro display skip its scan-out readback when the installed renderer takes + the GPU frame; screenshots and VNC (`need_readback`, `set_capture`) still + read back, on request only. +- **Overlay and status bar.** Both draw into CPU RGBA today (`DebugOverlay`, + `StatusBar` + `StatusBarTexture` for GL). The wgpu path uploads their + pixels as textures; `StatusBarTexture` gains a CPU form. +- **Choice.** `[ui] renderer = "gl" | "wgpu"`, CLI `--renderer`; default + stays `gl` until the wgpu path has run on Linux, macOS and Windows. + Threading follows the GL path's rules (`clanker/gui/`): the surface is + created on the main thread in `Ui::new`, configured and presented on the + board's display thread, and window-mutating calls stay on the event + thread (`ResizeRequest`). + +## Steps + +1. `Gpu` keeps instance and adapter; `GfxDisplay::gpu()`; VPro returns its + unit's `Gpu`. +2. `src/wgpu/present.rs` with the CPU frame source; `WgpuRenderer` in + `ui.rs` behind `[ui] renderer`; every board runs on it (CPU frames). +3. GPU frame source; the VPro display presents its scan-out buffer without + a readback when the renderer supports it. +4. Newport's compositor in WGSL (`gl_compositor.rs`'s shader, as a + `ScanoutShaders` kernel over the REX3 planes uploaded per frame), and + the GL renderer becomes optional. Done as `src/dev/ng1/wgpu` + (`Ng1Scanout`), matching `SwCompositor` on every pixel of 12 random + frames (`the_gpu_compose_matches_the_software_compositor`); seen live on + an Indy's PROM. The planes go up whole each frame (16 MB); REX3's damage + rectangles (`rex-dirty-upload`) could cut that, as the GL path does. diff --git a/clanker/wgpu/gpu-floats-are-not-the-hosts.md b/clanker/wgpu/gpu-floats-are-not-the-hosts.md new file mode 100644 index 000000000..0a2f5cef7 --- /dev/null +++ b/clanker/wgpu/gpu-floats-are-not-the-hosts.md @@ -0,0 +1,49 @@ +# GPU float division is not correctly rounded, and products get fused: guard both before comparing with the CPU model + +Status: 2026-11-06, rule, measured on an RX 6600 XT (RADV, Vulkan 1.4) with +the VPro kernels in `src/dev/odyssey/wgpu`. + +## Symptom + +The GPU raster unit's depth words differed from the CPU pipeline's by one +in the lowest mantissa bit on a few percent of pixels, under any depth +range, with every other byte of the pixel equal. + +## Cause + +Two things the host's f32 arithmetic does that a GPU's need not: + +- **Division.** Vulkan allows `x / y` 2.5 ULP; RADV computes it as a + reciprocal and a multiply. `z / d` in the depth ratio came out as + `z * rcp(d)` (the diagnostic matched that variant in most cases, and + neither variant where `rcp` itself was a unit off). +- **Contraction.** SPIR-V without `NoContraction` lets the driver fuse + `a * b + c` into one rounding, naga emits no such decoration, and RADV + fuses. Removing the guard below brought three of the eight parity tests + down (422 differing words in the random-state test). + +Add, subtract and multiply are correctly rounded everywhere wgpu runs; +those never differed. + +## Rule + +- Divide with `div_exact` from `src/wgpu/shaders/float.wgsl`: the two + mantissas divided as integers, rounded to nearest even, which is what + the host does. Use it wherever a quotient feeds a value the hashes see + (the depth ratio, the colour weights' `W_ONE / sum`; later the texture + unit's divides by Q and the level of detail). +- Keep a product apart from a sum by multiplying it by a one the compiler + cannot see: each board kernel reads `1.0` from its record (`Rec::one`) + and wraps products with `p(x) = x * nc`. `fma(x, one, c)` rounds like + `x + c`, so it is safe whether the driver fuses or not. Any product that + feeds an add or a subtract, including `1.0 - l1`, needs the wrap. +- Match the CPU's operation order exactly (the lanes code in + `pipe/raster.rs` and `pipe/interp.rs` is the reference) and reproduce + Rust's saturating `as i32` (`sat_i32`) and `as f32` of an i64 + (`i64_to_f32` in `src/wgpu/shaders/i64.wgsl`, correctly rounded) rather + than trusting the driver's conversions. +- Expect denormals to flush on the GPU; nothing in the pipelines produces + them. + +The parity tests (`cargo test --features wgpu --lib odyssey::wgpu`) are +the check: they compare whole board memories word for word. diff --git a/clanker/wgpu/read-storage-in-place.md b/clanker/wgpu/read-storage-in-place.md new file mode 100644 index 000000000..8e81d0bec --- /dev/null +++ b/clanker/wgpu/read-storage-in-place.md @@ -0,0 +1,43 @@ +# Read records, slots and tables from their buffers in place: a large struct copied by value gave silently wrong results + +Status: 2026-11-06, rule, measured on an RX 6600 XT (RADV, Mesa 26.1.8, +ACO) with the VPro kernels. + +## Symptom + +The textured-triangle parity test passed, then failed on 10,626 words +(texels about two thirds of the CPU's) after a *fourth arm* was added to +the raster hook's dispatch (`switch recs[pi].kind`), with nothing on the +executed path changed. Any fourth arm did it, a `switch` or an if/else +chain, with or without a new storage binding. `RADV_DEBUG=noopt` and the +shader cache made no difference; the result was deterministic. + +## Cause + +The texture code copied the 4.8 KB `TexSlot` by value (`let t = +tex_slots[i]`) and passed it by value through three call levels, and +each lane carried a 1 KB `var` log2 table with an initialiser. +Those copies live in per-lane private (scratch) memory. Reading the slot +fields in place (`tex_slots[ti].field`) and moving the table to a small +storage buffer made the test pass with the four arms; converting the +352-byte record and 368-byte slot copies to in-place reads followed as a +precaution. The failing shape's SPIR-V (naga-cli 30.0.0 from the dumped +WGSL) passes `spirv-val --target-env vulkan1.2`, so the fault is in the +driver's handling of that much private memory, or in a naga construct +that validates but means something else; either way the rule below +removes the exposure and is faster. + +## Rule + +- In a kernel, never bind a large struct to a `let` or pass it by value. + Pass the buffer index and read `buffer[i].field` where it is used; copy + only small things (a `TexLevel`, a decoded `Backend`). +- Tables go in storage buffers, not `var` arrays with + initialisers (every lane initialises its own copy on entry). +- Initialise every function-scope array an unused tap or texel might be + read from, even where a zero weight discards it: WGSL promises zero, + but keep the promise visible. +- When a parity test that passed starts failing after an unrelated + change, suspect private memory first and bisect with the shader alone + (`IRIS_WGPU_DUMP=` writes the composed WGSL of every kernel; + `naga` and `spirv-val` take it from there). diff --git a/clanker/wgpu/swaps-wait-on-stream-positions-not-fences.md b/clanker/wgpu/swaps-wait-on-stream-positions-not-fences.md new file mode 100644 index 000000000..8ee16e658 --- /dev/null +++ b/clanker/wgpu/swaps-wait-on-stream-positions-not-fences.md @@ -0,0 +1,38 @@ +# Swaps wait on stream positions, not on fences + +Status: design, 2026-11-07 (ea947636). One part open, see the end. + +**Problem.** With the GPU raster unit the drawing a swap should show may +still sit in an unsubmitted batch when the retrace latches the flip, and +the display composes from GPU memory. Making the display thread wait for +the GPU per frame costs the overlap we want. + +**Design.** Order by position in the command stream, not by GPU fences: + +- `CommandFifo` counts stores `pushed` and `retired` (absolute, monotone; + the ring's own indices wrap). +- A swap arm or swap-control write records `pushed` at that moment + (`HostInterface::swap_after`, from `stream_pos`, which the bus path sets + before the front register write). +- `GpuRaster::tick(idle, retired)` runs at the end of every board slice; it + publishes `retired` to `DisplayShared::covered` once everything recorded + up to it is **submitted** (every flush, or a tick with nothing recorded). +- `HostInterface::retrace(covered)` latches flips only when + `swap_after <= covered`; otherwise the flip stays armed for a later + retrace. + +Submitted is enough, completed is not needed: the scan-out's own +submission goes on the same wgpu queue, behind the batch, so the composed +frame shows it. The CPU raster unit leaves `covered` at `u64::MAX` and +flips latch at once, as before. + +Test: `a_flip_waits_until_the_raster_unit_covers_the_stores_before_its_arm` +(tests/display.rs). + +**Open, minor.** The status words a guest polls for idle (DMA busy, the +sync words, `access_waits` reads) still go through `raster_wait`: flush and +wait for the GPU. The same idea would let them answer "busy" until a +per-submission completion counter (`queue.on_submitted_work_done`) passes +the position of the last store before the read, without blocking the +board's thread. Measured live it does not matter yet: steady-state GL +Quake made 72 waits over the whole session and no readbacks. diff --git a/clanker/wgpu/the-mirror-and-its-dirty-maps.md b/clanker/wgpu/the-mirror-and-its-dirty-maps.md new file mode 100644 index 000000000..9455a1348 --- /dev/null +++ b/clanker/wgpu/the-mirror-and-its-dirty-maps.md @@ -0,0 +1,69 @@ +# The GPU unit's board memory has a CPU mirror, and two dirty maps keep the two in step + +Status: 2026-11-06, design, `src/dev/odyssey/wgpu/mod.rs` (`GpuRaster`), +`src/dev/odyssey/vram.rs` (`DirtyBlocks`), `Board::raster_wait`. + +The plan says board memory lives on the GPU and nothing on the CPU maps +it. The front end still has code that reads and writes pixels itself +(integer drawing, image puts, copies, pixel DMA, context save, the +texture unit's samples, the hashes and dumps). Rather than rewrite each +of those for the first milestone, the board keeps its `Vram` as a +*mirror* of the GPU buffer, and the raster unit keeps the two in step: + +- **CPU to GPU.** `Vram::track_dirty` gives the memory a map of 16 KB + blocks; `VramPtr::store` marks the block it writes (one relaxed + `fetch_or`, skipped when no map is kept, which is the CPU unit's case). + `GpuRaster::flush` takes the map and uploads the marked runs with + `queue.write_buffer` before it submits the batch, so the batch sees + them. The display's scanout kernel reads the GPU buffer, so the blocks + also go up at the end of every run of stores (`GpuRaster::tick`), and + the open batch is submitted there too once the board is going idle or + the batch has been open 4 ms: a batch that waited to fill held PROM + text and X painting off the screen in the first live boot. +- **GPU to CPU.** `GpuRaster::push` marks, in its own map, the blocks + each record's box may touch in each view of its slot (the tile-major + layout keeps a box's tiles in one contiguous span of addresses). + `read_back` (what `Board::raster_wait` becomes) flushes, waits, reads + the marked runs back in one copy and writes them into the mirror + without marking the CPU map (`write_words_untracked`). + +The invariant everything rests on is the one the CPU raster thread +already imposed: **the front end waits for the raster unit before it +touches memory itself.** A CPU store with records still in the batch +would be uploaded ahead of them at the next flush, reordering it before +drawing that was issued earlier. The one known hole is the same as the +CPU pipeline's: a texture the GPU drew (a pbuffer used as a texture) +is sampled from the mirror, which is current only as of the last wait. + +**A writer that skips the wait must not use the block map.** A block is +16 KB; if the mirror is stale in part of it (the GPU drew there and no +readback has run), uploading the whole block clobbers the GPU's bytes +with the mirror's. So a path that writes without waiting, which is only +allowed into memory no pending drawing touches (texture and scratch +uploads, `dma_to_buffer` with `memory`), stores through +`VramPtr::untracked()` and then calls `GpuRaster::upload_region` for the +rectangle it wrote: the upload is those tiles alone. The same tile spans +are what a blit record will reference once puts into the screen carry +their pixels in the batch. + +What this buys: every CPU path is correct unchanged, and a draw the +kernels cannot take (a coded colour field, texture, fog, user planes, +lines) still goes through the unit in order, either as CPU-shaded quads +the unit stores (`emit` becomes `push_frag`) or, for an unsupported +destination, as CPU stores onto a mirror brought current first. + +Since 2026-11-06 the front end's own pixel paths store *through the +unit* as blit records (logical operations, masks and own views included), +`raster_wait` is a GPU wait with no readback, and only readers bring +blocks back, by the box they read (`planes_in`, `read_back_view`): a DMA +get, a copy's rectangles, the hashes and the monitor. The invariant above +still governs the paths that store into the mirror. + +What it costs: a readback of the touched blocks under a reader's box, and +the upload of every CPU write. The replay prints both +(`gpu unit: ... MiB uploaded, ... MiB read back in N readbacks`). The +way out, as the plan's VPro table says, is to turn the CPU paths that run +between draws into records: pixel DMA puts and image puts as blit records +reading from a staged upload buffer, copies as copy records, and DMA gets +as readbacks of their own rectangle staged in the front end and written +to the guest through the bus. Each one removes a wait. diff --git a/clanker/wgpu/vram-accesses-are-atomic.md b/clanker/wgpu/vram-accesses-are-atomic.md new file mode 100644 index 000000000..5d583a9fe --- /dev/null +++ b/clanker/wgpu/vram-accesses-are-atomic.md @@ -0,0 +1,20 @@ +# Board memory on the GPU is accessed with atomics, because two pixels can share a word + +Status: 2026-11-06, design, `src/dev/odyssey/wgpu/shaders/vram.wgsl`. + +The raster pass gives each 8x8 tile one workgroup and each pixel one lane, +so a pixel's own fields are touched by one lane and need no atomics. But +the window-id buffer and the overlay hold 2-byte pixels: lanes x and x+1 +read-modify-write the same 32-bit word when they store their byte, and a +plain load-or-store loses one of the two. The VPro memory binding is +`array>`: whole words go through `atomicStore`, partial fields +through `atomicAnd` then `atomicOr` of the lane's own bits, and loads +through `atomicLoad`. Relaxed atomics on one location are program-ordered +within a lane, which is all the order the kernels need (a lane's depth +test reads what the same lane stored for an earlier primitive). + +The cost is that every access goes through the L2 rather than the lane's +nearer caches. When it shows in a profile, bind the buffer a second time +as `array` for the 4-byte aligned fields (colour, depth with its +stencil byte) and keep the atomics for byte fields. The parity test +`index_draws_share_words_between_pixels` covers the shared-word case. diff --git a/clanker/wgpu/window-id-bytes-are-written-through-to-the-mirror.md b/clanker/wgpu/window-id-bytes-are-written-through-to-the-mirror.md new file mode 100644 index 000000000..f1f3a1d11 --- /dev/null +++ b/clanker/wgpu/window-id-bytes-are-written-through-to-the-mirror.md @@ -0,0 +1,34 @@ +# Window-id bytes are written through to the mirror + +Status: rule, 2026-11-07. + +**Symptom.** On the GPU raster unit, 12 of the visual sweep's cases failed +that pass on the CPU unit (4,4,4,4, 8,8,8,8 with stencil, 10,10,10,2 +windows): colours read back as raw packed words (`ffe02c0f` for pink on a +10,10,10,2 window), stencil reads of 255/128/64, and `pixtest` +`buffers/swap` failed. The failures moved between runs. + +**Cause.** The front end reads the window-id byte itself, with no wait for +the raster unit: `Board::drawable_codec` (the format of a drawable's +pixels, for DMA reads and copies), `Board::main_id` and +`Board::id_compare_passes`. On the CPU unit that is safe because only the +front end writes those bytes. On the GPU unit the X server's window-id +writes (`paint_aux`, `paint_integer`, `buffer_store`, `clear_planes`) became +GPU blits, so the mirror's id bytes went stale until some readback +happened to cover their block. A read then took the wrong window's display +mode and so the wrong pixel format. + +**Rule.** A store the front end sends to the GPU into a byte field (the +window-id buffer's id and overlay bytes, `Kind::Byte`) is also made in the +mirror, through an untracked pointer, exactly as the blit kernel makes it +(`draw.rs` `mirror_byte_store`, matching `vpro.wgsl` `blit`). The mirror's +id bytes stay current, nothing is uploaded twice, and a later readback of +the block brings back the same bytes. Any new GPU path that writes id or +overlay bytes must do the same, or the front end must read them back first. + +**How it was found.** Short guest recordings of the failing cases +(`vpro rec`), replayed on both units with `VPRO_REPLAY_WRITES` hashing the +board's writes to guest memory; three diverged. The regression test +`the_raster_units_agree_on_recorded_sessions` replays local recordings on +the CPU unit, the GPU unit and the GPU unit with live-style slice ends +(`VPRO_REPLAY_TICK`) and compares host writes and memory. diff --git a/debug.md b/debug.md index 904d8891d..934f6aa79 100644 --- a/debug.md +++ b/debug.md @@ -490,6 +490,7 @@ guest. `y = 0` is the top display line throughout. | `vpro hash` | The hash of board memory a replay checkpoint takes (`golden/replay.rs`), to compare a live board with a recording | | `vpro unimp` | Offsets and command words the board reported as not modelled | | `vpro refusals [reset]` | Bus accesses answered busy, by reason, offset and width, with the longest time one access was kept retrying | +| `vpro gpu [trace on\|off]` | With `[graphics] raster = "wgpu"`: the GPU raster unit's counters (records and batches, how the batches were submitted, bytes uploaded and read back, waits and readbacks) and the GPU scan-out's (frames composed, bytes read back; with `renderer = "wgpu"` frames are presented from the GPU and only screenshots, frame requests and VNC read them back); `trace on` counts readbacks by the board function that caused them from then on, shown under the counters. The same lines appear in `vpro status`. On the GPU unit the commands that read board memory (`pix`, `raw`, `peek`, `shot`, `fbdump`, `hash`, `mem` and the state commands) first read the unit's memory back into the board's copy, when the board can be quiesced | | `vpro trace [fifo,draw,cpu,dma\|all]` / `vpro trace flush\|off` / `vpro trace` | Annotated trace (`gputrace` builds; also `IRIS_ODSY_TRACE=` from power-on): `FIFO` lines are command words as the parser completes them, named, with their operands as words and as floats where the word carries floats, plus copy rects, pattern and matrix loads, image boxes and bursts, queries and unknown words; `DRAW` lines each assembled primitive sent to the raster unit (`draw TRIANGLES n=3 ctx=1 textured depth queued cost 96`) and each clear; `CPU` lines every bus access that is not a command store, repeats collapsed; `DMA` lines the page-table descriptors. `flush` writes the buffer out (it is written at most twice a second otherwise); alone, the trace's state | | `vpro rec ` / `vpro rec mark\|off` / `vpro rec` | Replayable recording (record.rs; also `IRIS_ODSY_REC=` from power-on): every bus store and load and the DMA reads, gzip. A start, a `mark` and `off` wait for the board to go idle and write the memory hash (`vpro hash`) into the file, so a replay can check board memory at those points; a recording started mid-session cannot reproduce the live memory and says so when replayed, but still replays deterministically. Alone, the recording's state. Replay: `VPRO_REPLAY= cargo test --release --lib odyssey::golden::replay -- --nocapture` (see `src/dev/odyssey/testdata/README.md`) | diff --git a/src/config.rs b/src/config.rs index 4d4e37ee2..39ca0ab42 100644 --- a/src/config.rs +++ b/src/config.rs @@ -974,6 +974,33 @@ pub struct GraphicsSection { /// two from 8 to 256. #[serde(default = "default_graphics_tile")] pub tile: u32, + /// VPro: where the raster unit runs: `cpu` (the threads above) or + /// `wgpu` (the host GPU through wgpu compute; needs the `wgpu` + /// feature). + #[serde(default)] + pub raster: RasterUnit, + /// The wgpu backend for `raster = "wgpu"`: `auto`, `vulkan`, `metal`, + /// `dx12` or `gl`. + #[serde(default = "default_gpu_backend")] + pub gpu_backend: String, +} + +/// What draws the host window (`renderer`, `--renderer`). +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default, clap::ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum UiRenderer { + #[default] + Gl, + Wgpu, +} + +/// Where a board's raster unit runs (`[graphics] raster`, `--raster`). +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default, clap::ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum RasterUnit { + #[default] + Cpu, + Wgpu, } impl GraphicsSection { @@ -988,6 +1015,7 @@ impl GraphicsSection { fn default_graphics_heads() -> u8 { 1 } fn default_graphics_threads() -> usize { 2 } fn default_graphics_tile() -> u32 { 32 } +fn default_gpu_backend() -> String { "auto".into() } impl Default for GraphicsSection { fn default() -> Self { @@ -997,6 +1025,8 @@ impl Default for GraphicsSection { resolution: crate::dev::ng1::vc2_timings::NewportResolution::default(), threads: default_graphics_threads(), tile: default_graphics_tile(), + raster: RasterUnit::Cpu, + gpu_backend: default_gpu_backend(), } } } @@ -1818,6 +1848,14 @@ pub struct MachineConfig { #[serde(default = "default_lock_aspect_ratio")] pub lock_aspect_ratio: bool, + /// What draws the window: `gl` (default) or `wgpu` (the host GPU + /// through wgpu, on the same device as a board drawing there, so a + /// VPro frame composed on the GPU is presented without a readback; + /// needs a build with the `wgpu` feature). The iris-gui front end has + /// its own window and ignores this. + #[serde(default)] + pub renderer: UiRenderer, + /// Optional file path that will receive every byte emitted on ttyd1 /// (the IRIX serial console) in `--ci` mode. Append-only. Useful for /// keeping a continuously-updated transcript of the install or test run. @@ -2118,6 +2156,7 @@ impl Default for MachineConfig { network: NetworkSection::default(), mouse_scroll_pixels_per_line: default_scroll_pixels_per_line(), lock_aspect_ratio: default_lock_aspect_ratio(), + renderer: UiRenderer::default(), scsi_deferred_int: default_scsi_deferred_int(), audio: AudioConfig::default(), machine: MachineSection::default(), @@ -2534,6 +2573,15 @@ impl MachineConfig { if self.graphics.threads == 0 { return Err("graphics.threads must be at least 1".into()); } + if self.renderer == UiRenderer::Wgpu && !cfg!(feature = "wgpu") { + return Err("renderer = \"wgpu\" needs a build with the wgpu feature".into()); + } + if self.graphics.raster == RasterUnit::Wgpu && !cfg!(feature = "wgpu") { + return Err("graphics.raster = \"wgpu\" needs a build with the wgpu feature".into()); + } + if !["auto", "vulkan", "metal", "dx12", "gl"].contains(&self.graphics.gpu_backend.as_str()) { + return Err(format!("graphics.gpu_backend {:?} must be auto, vulkan, metal, dx12 or gl", self.graphics.gpu_backend)); + } if !self.graphics.tile.is_power_of_two() || !(8..=256).contains(&self.graphics.tile) { return Err(format!("graphics.tile {} must be a power of two from 8 to 256", self.graphics.tile)); } @@ -2789,6 +2837,14 @@ pub struct Cli { #[arg(long = "graphics", visible_alias = "board", visible_alias = "graphics-board", value_name = "BOARD")] pub graphics: Option, + /// VPro: where the raster unit runs: `cpu` (default) or `wgpu` (the host GPU through wgpu compute; needs a build with the `wgpu` feature). + #[arg(long = "raster", value_name = "UNIT")] + pub raster: Option, + + /// What draws the window: `gl` (default) or `wgpu` (needs a build with the `wgpu` feature). + #[arg(long = "renderer", value_name = "RENDERER")] + pub renderer: Option, + /// Enable GDB stub on the given TCP port (e.g. --gdb-port 1234). /// Connect with: target remote localhost: #[arg(long = "gdb-port", value_name = "PORT")] @@ -2965,6 +3021,8 @@ impl Cli { if let Some(cpu) = self.cpu { cfg.machine.cpu = cpu; } if let Some(cache) = self.cache { cfg.machine.cache = cache; } if let Some(b) = self.graphics { cfg.graphics.board = Some(b); } + if let Some(r) = self.raster { cfg.graphics.raster = r; } + if let Some(r) = self.renderer { cfg.renderer = r; } if self.scale2x { cfg.scale = 2; } if self.headless { cfg.headless = true; } if self.no_audio { cfg.no_audio = true; } diff --git a/src/debug_overlay.rs b/src/debug_overlay.rs index 22ee62be8..3549a31bd 100644 --- a/src/debug_overlay.rs +++ b/src/debug_overlay.rs @@ -91,6 +91,31 @@ impl DebugOverlay { /// Render overlay into the CPU buffer and upload to the GL texture. /// Returns the GL texture handle. pub fn render(&mut self, src: &OverlaySource<'_>, gl: &glow::Context) -> glow::Texture { + self.render_pixels(src); + // Upload to GL texture and return handle + let tex = self.ensure_tex(gl); + unsafe { + gl.bind_texture(glow::TEXTURE_2D, Some(tex)); + gl.pixel_store_i32(glow::UNPACK_ROW_LENGTH, 2048); + let u8_slice = std::slice::from_raw_parts( + self.buf.as_ptr() as *const u8, + self.buf.len() * 4, + ); + gl.tex_sub_image_2d( + glow::TEXTURE_2D, 0, + 0, 0, src.width as i32, src.height as i32, + glow::RGBA, glow::UNSIGNED_BYTE, + glow::PixelUnpackData::Slice(Some(u8_slice)), + ); + gl.pixel_store_i32(glow::UNPACK_ROW_LENGTH, 0); + } + tex + } + + /// The overlay drawn into its CPU buffer (stride 2048, 0xAABBGGRR, + /// transparent where nothing is shown), with no GL: what a renderer + /// other than the GL one uploads itself. + pub fn render_pixels(&mut self, src: &OverlaySource<'_>) -> &[u32] { let width = src.width; let height = src.height; @@ -383,25 +408,7 @@ impl DebugOverlay { } } } - - // Upload to GL texture and return handle - let tex = self.ensure_tex(gl); - unsafe { - gl.bind_texture(glow::TEXTURE_2D, Some(tex)); - gl.pixel_store_i32(glow::UNPACK_ROW_LENGTH, 2048); - let u8_slice = std::slice::from_raw_parts( - self.buf.as_ptr() as *const u8, - self.buf.len() * 4, - ); - gl.tex_sub_image_2d( - glow::TEXTURE_2D, 0, - 0, 0, src.width as i32, src.height as i32, - glow::RGBA, glow::UNSIGNED_BYTE, - glow::PixelUnpackData::Slice(Some(u8_slice)), - ); - gl.pixel_store_i32(glow::UNPACK_ROW_LENGTH, 0); - } - tex + &self.buf } pub fn destroy(&mut self, gl: &glow::Context) { diff --git a/src/dev/ng1/mod.rs b/src/dev/ng1/mod.rs index 72828f4be..60ae15eb2 100644 --- a/src/dev/ng1/mod.rs +++ b/src/dev/ng1/mod.rs @@ -33,6 +33,9 @@ pub mod rex3_jit_queue; #[cfg(feature = "rex-jit")] pub mod rex3_jit; pub mod vc2; +/// The display composed on the host GPU (the wgpu window renderer). +#[cfg(feature = "wgpu")] +pub mod wgpu; pub mod vc2_timings; pub mod xmap9; pub mod cmap; diff --git a/src/dev/ng1/rex3.rs b/src/dev/ng1/rex3.rs index 1bce0f5da..8330da248 100644 --- a/src/dev/ng1/rex3.rs +++ b/src/dev/ng1/rex3.rs @@ -46,6 +46,29 @@ pub trait Renderer: Send { fn compositor_status(&self) -> String { "compositor=sw shader=n/a".to_string() } /// Whether the GL compositor is in use, or is about to be. fn gl_compositor(&self) -> bool { false } + + /// Whether `present_gpu_frame` presents frames that are already on the + /// board's GPU (the wgpu renderer, on the board's own device). A board + /// that composes on the GPU then skips its readback. + #[cfg(feature = "wgpu")] + fn takes_gpu_frames(&self) -> bool { false } + + /// Present a frame of 0x00RRGGBB words (rows of `width`) that is on the + /// board's GPU, with `screen` giving the overlay's sources; the status + /// bar and overlay as `present` draws them. `screen.rgba` is not updated. + #[cfg(feature = "wgpu")] + #[allow(clippy::too_many_arguments)] + fn present_gpu_frame( + &mut self, + _frame: &wgpu::Buffer, + _width: u32, + _height: u32, + _screen: &mut crate::disp::Rex3Screen, + _overlay: &mut crate::debug_overlay::DebugOverlay, + _status: &mut crate::disp::StatusBar, + _sbtex: &mut crate::disp::StatusBarTexture, + _stats: &crate::disp::BarStats, + ) {} } pub const REX3_SIZE: u32 = 0x2000; // 8KB diff --git a/src/dev/ng1/wgpu/mod.rs b/src/dev/ng1/wgpu/mod.rs new file mode 100644 index 000000000..b51bede6c --- /dev/null +++ b/src/dev/ng1/wgpu/mod.rs @@ -0,0 +1,151 @@ +//! Newport's display composed on the host GPU: the GL compositor's work +//! (`gl_compositor.rs`) and the software one's (`SwCompositor`) as a kernel +//! on the common scan-out driver (`crate::wgpu::scanout`), so the wgpu +//! window renderer composes REX3 frames on the device it presents with. +//! +//! The planes are uploaded each frame from the `CompositorSource` the +//! renderers already get; the slot carries the VC2 cursor registers, the +//! XMAP9 modes, the Bt445 RAMDAC and the CMAP. + +use std::sync::Arc; + +use bytemuck::{Pod, Zeroable}; + +use crate::compositor::CompositorSource; +use crate::dev::ng1::vc2::{ + VC2_CTRL_CURSOR_EN, VC2_CTRL_CURSOR_SIZE, VC2_REG_CURRENT_CURSOR_X, VC2_REG_CURSOR_ENTRY_PTR, VC2_REG_DISPLAY_CONTROL, + VC2_REG_WORKING_CURSOR_Y, +}; +use crate::wgpu::shader::Fragment; +use crate::wgpu::{Gpu, Scanout, ScanoutPipeline, ScanoutShaders}; + +#[cfg(test)] +mod tests; + +const SCANOUT: Fragment = Fragment::new("ng1_scanout.wgsl", include_str!("shaders/scanout.wgsl")); + +/// REX3's planes the display reads: 2048 x 1024. +const PLANE_WORDS: usize = 2048 * 1024; +/// The window-id map's bytes: one a screen pixel, rows of 2048. +const DID_BYTES: usize = crate::disp::FB_STRIDE * crate::disp::FB_MAX_H; +/// The VC2's RAM in 16-bit words. +const VC2_WORDS: usize = 32768; +/// The largest frame. +const MAX_PIXELS: u32 = (crate::disp::FB_STRIDE * crate::disp::FB_MAX_H) as u32; + +/// A frame's registers and tables (`shaders/scanout.wgsl`, `Ng1Slot`). +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct Ng1Slot { + head: [u32; 16], + xmap: [u32; 32], + ramdac: [u32; 256], + cmap: [u32; 8192], +} + +impl Ng1Slot { + fn of(src: &CompositorSource<'_>) -> Ng1Slot { + let r = src.vc2_regs; + let ctrl = r[VC2_REG_DISPLAY_CONTROL as usize]; + let mut head = [0u32; 16]; + head[0] = ((r[VC2_REG_CURRENT_CURSOR_X as usize] as i32) - 31 + src.cursor_x_adjust) as u32; + head[1] = ((r[VC2_REG_WORKING_CURSOR_Y as usize] as i32) - 31) as u32; + head[2] = r[VC2_REG_CURSOR_ENTRY_PTR as usize] as u32; + head[3] = ((ctrl & VC2_CTRL_CURSOR_EN != 0) as u32) | ((ctrl & VC2_CTRL_CURSOR_SIZE != 0) as u32) << 1; + head[4] = src.xmap_cursor_cmap as u32; + head[5] = src.xmap_popup_cmap as u32; + head[6] = (src.topscan + 1) as u32; + head[7] = src.vc2_ram.len().min(VC2_WORDS) as u32; + Ng1Slot { head, xmap: *src.xmap_mode, ramdac: *src.ramdac, cmap: *src.cmap } + } +} + +/// The display's compose on the GPU. +pub struct Ng1Scanout { + pipeline: ScanoutPipeline, + scanout: Scanout, + rgb: wgpu::Buffer, + aux: wgpu::Buffer, + did: wgpu::Buffer, + vc2: wgpu::Buffer, + group: wgpu::BindGroup, + /// Bytes of planes uploaded so far. + pub uploaded: u64, +} + +impl Ng1Scanout { + pub fn new(gpu: &Arc) -> Result { + let entry = |binding| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Storage { read_only: true }, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }; + let layout = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("ng1 scanout planes"), + entries: &[entry(0), entry(1), entry(2), entry(3)], + }); + let pipeline = ScanoutPipeline::new(gpu, &ScanoutShaders { label: "ng1", body: &[SCANOUT], library: &[], defines: &[], layouts: &[&layout] })?; + let scanout = Scanout::new(gpu, &pipeline, MAX_PIXELS, size_of::() as u64); + let usage = wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST; + let rgb = gpu.buffer("ng1 rgb plane", 4 * PLANE_WORDS as u64, usage); + let aux = gpu.buffer("ng1 aux plane", 4 * PLANE_WORDS as u64, usage); + let did = gpu.buffer("ng1 window ids", DID_BYTES as u64, usage); + let vc2 = gpu.buffer("ng1 vc2 ram", 2 * VC2_WORDS as u64, usage); + let group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("ng1 scanout planes"), + layout: &layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: rgb.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: aux.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: did.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: vc2.as_entire_binding() }, + ], + }); + gpu.check()?; + Ok(Ng1Scanout { pipeline, scanout, rgb, aux, did, vc2, group, uploaded: 0 }) + } + + /// The device it runs on. + pub fn gpu(&self) -> &Arc { + self.scanout.gpu() + } + + /// The planes and the slot of `src` up to the GPU. + fn upload(&mut self, src: &CompositorSource<'_>) -> Vec { + let q = &self.scanout.gpu().queue; + let words = |s: &[u32]| PLANE_WORDS.min(s.len()) & !1; + let n = words(src.fb_rgb); + q.write_buffer(&self.rgb, 0, bytemuck::cast_slice(&src.fb_rgb[..n])); + let m = words(src.fb_aux); + q.write_buffer(&self.aux, 0, bytemuck::cast_slice(&src.fb_aux[..m])); + // The window-id map's rows on screen, whole words. + let d = (src.height.max(1) * crate::disp::FB_STRIDE).min(src.did.len()).min(DID_BYTES) & !3; + q.write_buffer(&self.did, 0, &src.did[..d]); + let v = src.vc2_ram.len().min(VC2_WORDS) & !1; + q.write_buffer(&self.vc2, 0, bytemuck::cast_slice(&src.vc2_ram[..v])); + self.uploaded += 4 * (n + m) as u64 + d as u64 + 2 * v as u64; + bytemuck::bytes_of(&Ng1Slot::of(src)).to_vec() + } + + /// Compose `src` and submit it with no readback; the buffer the frame + /// is in (0x00RRGGBB, rows of `src.width`), for a renderer on this + /// device. + pub fn compose(&mut self, src: &CompositorSource<'_>) -> Result { + let slot = self.upload(src); + self.scanout.compose(&self.pipeline, src.width as u32, src.height as u32, &slot, &[&self.group])?; + Ok(self.scanout.output().clone()) + } + + /// The buffer the last frame was composed into. + pub fn output(&self) -> wgpu::Buffer { + self.scanout.output().clone() + } + + /// Compose `src` and read the frame back into `out` (0x00RRGGBB, rows + /// of `src.width`). + pub fn run(&mut self, src: &CompositorSource<'_>, out: &mut [u32]) -> Result<(), String> { + let slot = self.upload(src); + self.scanout.run(&self.pipeline, src.width as u32, src.height as u32, &slot, &[&self.group], out) + } +} diff --git a/src/dev/ng1/wgpu/shaders/scanout.wgsl b/src/dev/ng1/wgpu/shaders/scanout.wgsl new file mode 100644 index 000000000..09a09be6a --- /dev/null +++ b/src/dev/ng1/wgpu/shaders/scanout.wgsl @@ -0,0 +1,162 @@ +// Newport's display on the GPU: per screen pixel, the window id picks an +// XMAP9 mode; the VC2 cursor, then a popup, then the overlay, then the +// main pixel (through the CMAP, or expanded RGB) give the colour; the +// Bt445 RAMDAC's gamma tables finish it. The same rules as +// `SwCompositor::compose_pixels` (src/compositor.rs), word for word. +// +// The slot (group 1) holds the frame's registers and tables; group 2 holds +// the planes: REX3's RGB and auxiliary planes (2048 x 1024, the frame's row +// y read from row (topscan + y) mod 1024), the window-id map (one byte a +// screen pixel, four to a word, rows of 2048) and the VC2's RAM (16-bit +// words, two to a word, the lower address in the low half). + +const H_CURSOR_X: u32 = 0u; // the cursor's hot spot, as i32 bits +const H_CURSOR_Y: u32 = 1u; +const H_CURSOR_ENTRY: u32 = 2u; +const H_FLAGS: u32 = 3u; // bit 0: cursor on; bit 1: 64 x 64 cursor +const H_CURSOR_CMAP: u32 = 4u; +const H_POPUP_CMAP: u32 = 5u; +const H_TOPSCAN: u32 = 6u; // the first row shown, plus one +const H_VC2_WORDS: u32 = 7u; + +struct Ng1Slot { + head: array, + xmap: array, + ramdac: array, + cmap: array, +} + +@group(1) @binding(0) var slot: Ng1Slot; +@group(2) @binding(0) var fb_rgb: array; +@group(2) @binding(1) var fb_aux: array; +@group(2) @binding(2) var did_map: array; +@group(2) @binding(3) var vc2_ram: array; + +fn vc2_word(a: u32) -> u32 { + let w = vc2_ram[a >> 1u]; + return select(w & 0xffffu, w >> 16u, (a & 1u) != 0u); +} + +fn did_at(i: u32) -> u32 { + return (did_map[i >> 2u] >> (8u * (i & 3u))) & 0xffu; +} + +// A CMAP entry, opaque; black past the table. +fn cmap_at(a: u32) -> u32 { + if a < 8192u { + return slot.cmap[a] | 0xff000000u; + } + return 0xff000000u; +} + +// The RGB expansions (Rex3::expand_4_rgb, _8_, _12_): 0x00BBGGRR. +fn expand4(v: u32) -> u32 { + let b = (v >> 3u) & 1u; + let g = (0xaau * ((v >> 2u) & 1u)) | (0x55u * ((v >> 1u) & 1u)); + let r = v & 1u; + return ((b * 255u) << 16u) | (g << 8u) | (r * 255u); +} + +fn expand8(v: u32) -> u32 { + let b = (0xaau * ((v >> 7u) & 1u)) | (0x55u * ((v >> 6u) & 1u)); + let g = (0x92u * ((v >> 5u) & 1u)) | (0x49u * ((v >> 4u) & 1u)) | (0x24u * ((v >> 3u) & 1u)); + let r = (0x92u * ((v >> 2u) & 1u)) | (0x49u * ((v >> 1u) & 1u)) | (0x24u * (v & 1u)); + return (b << 16u) | (g << 8u) | r; +} + +fn expand12(v: u32) -> u32 { + return ((((v >> 8u) & 0xfu) * 0x11u) << 16u) | ((((v >> 4u) & 0xfu) * 0x11u) << 8u) | ((v & 0xfu) * 0x11u); +} + +// The VC2 cursor's colour index at screen (x, y); 0 where it shows nothing. +fn cursor_index(x: u32, y: u32) -> u32 { + let flags = slot.head[H_FLAGS]; + if (flags & 1u) == 0u { + return 0u; + } + let cx = i32(x) - bitcast(slot.head[H_CURSOR_X]); + let cy = i32(y) - bitcast(slot.head[H_CURSOR_Y]); + if cx < 0 || cy < 0 { + return 0u; + } + let ux = u32(cx); + let uy = u32(cy); + let shift = 15u - (ux % 16u); + let entry = slot.head[H_CURSOR_ENTRY]; + let words = slot.head[H_VC2_WORDS]; + if (flags & 2u) != 0u { + if ux < 64u && uy < 64u { + let a = entry + uy * 4u + ux / 16u; + if a < words { + return (vc2_word(a) >> shift) & 1u; + } + } + return 0u; + } + if ux < 32u && uy < 32u { + let a = entry + uy * 2u + ux / 16u; + if a + 64u < words { + return ((vc2_word(a) >> shift) & 1u) | (((vc2_word(a + 64u) >> shift) & 1u) << 1u); + } + } + return 0u; +} + +fn scanout_pixel(x: u32, y: u32) -> u32 { + let fb_y = (slot.head[H_TOPSCAN] + y) & 0x3ffu; + let idx = fb_y * 2048u + x; + let mode = slot.xmap[did_at(y * 2048u + x) & 31u]; + + let buf_sel = (mode & 1u) != 0u; + let ovl_buf_sel = ((mode >> 1u) & 1u) != 0u; + let msb_cmap = (mode >> 3u) & 0x1fu; + let pix_mode = (mode >> 8u) & 3u; + let pix_size = (mode >> 10u) & 3u; + let aux_pix_mode = (mode >> 16u) & 7u; + let aux_msb_cmap = (mode >> 19u) & 0x1fu; + + let raw_rgb = fb_rgb[idx]; + let raw_aux = fb_aux[idx]; + let pup = (raw_aux >> 2u) & 3u; + let overlay = select((raw_aux >> 8u) & 0xffu, (raw_aux >> 16u) & 0xffu, ovl_buf_sel); + + var pixel = 0u; + switch pix_size { + case 0u: { pixel = (raw_rgb >> select(0u, 4u, buf_sel)) & 0xfu; } + case 1u: { pixel = (raw_rgb >> select(0u, 8u, buf_sel)) & 0xffu; } + case 2u: { pixel = (raw_rgb >> select(0u, 12u, buf_sel)) & 0xfffu; } + default: { pixel = raw_rgb & 0xffffffu; } + } + + let cursor = cursor_index(x, y); + var colour = 0xff000000u; + if cursor != 0u { + colour = cmap_at((slot.head[H_CURSOR_CMAP] << 5u) | cursor); + } else if pup != 0u { + colour = cmap_at((slot.head[H_POPUP_CMAP] << 5u) | pup); + } else if (aux_pix_mode == 2u || aux_pix_mode == 6u || aux_pix_mode == 7u) && overlay != 0u { + colour = cmap_at((aux_msb_cmap << 8u) | overlay); + } else if pix_mode == 0u { + var a = pixel; + if pix_size <= 1u { + a = (msb_cmap << 8u) | pixel; + } else if pix_size == 2u { + a = ((msb_cmap & 0x10u) << 8u) | pixel; + } + colour = cmap_at(a); + } else { + switch pix_size { + case 0u: { colour = expand4(pixel) | 0xff000000u; } + case 1u: { colour = expand8(pixel) | 0xff000000u; } + case 2u: { colour = expand12(pixel) | 0xff000000u; } + default: { colour = pixel | 0xff000000u; } + } + } + + // The RAMDAC's per-channel gamma. The CMAP's word holds blue in bits + // 23:16 and red in 7:0; the frame's word is 0x00RRGGBB. + let b = (slot.ramdac[(colour >> 16u) & 0xffu] >> 16u) & 0xffu; + let g = (slot.ramdac[(colour >> 8u) & 0xffu] >> 8u) & 0xffu; + let r = slot.ramdac[colour & 0xffu] & 0xffu; + return (r << 16u) | (g << 8u) | b; +} diff --git a/src/dev/ng1/wgpu/tests.rs b/src/dev/ng1/wgpu/tests.rs new file mode 100644 index 000000000..0df6eff9f --- /dev/null +++ b/src/dev/ng1/wgpu/tests.rs @@ -0,0 +1,126 @@ +//! The GPU compose against the software one, word for word. + +use std::sync::Arc; + +use super::Ng1Scanout; +use crate::compositor::{CompositorSource, SwCompositor}; +use crate::dev::ng1::vc2::{ + VC2_CTRL_CURSOR_EN, VC2_CTRL_CURSOR_SIZE, VC2_REG_CURRENT_CURSOR_X, VC2_REG_CURSOR_ENTRY_PTR, VC2_REG_DISPLAY_CONTROL, + VC2_REG_WORKING_CURSOR_Y, +}; +use crate::wgpu::{Gpu, GpuOptions}; + +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 33) as u32 + } + + fn below(&mut self, n: u32) -> u32 { + self.next() % n + } +} + +fn gpu() -> Option> { + match Gpu::new(&GpuOptions::default()) { + Ok(g) => Some(g), + Err(e) => { + eprintln!("skipped: {e}"); + None + } + } +} + +/// Random planes, window ids, XMAP modes, CMAP, RAMDAC, cursor and +/// scroll, composed by `SwCompositor` and by the kernel: every pixel the +/// same (the software word is 0xAABBGGRR, the kernel's 0x00RRGGBB). +#[test] +fn the_gpu_compose_matches_the_software_compositor() { + let Some(gpu) = gpu() else { return }; + let mut scan = Ng1Scanout::new(&gpu).unwrap_or_else(|e| panic!("{e}")); + let mut sw = SwCompositor::new(); + let mut rng = Lcg(7); + let stride = crate::disp::FB_STRIDE; + let mut fb_rgb = vec![0u32; stride * crate::disp::FB_MAX_H]; + let mut fb_aux = vec![0u32; stride * crate::disp::FB_MAX_H]; + let mut did = vec![0u8; stride * crate::disp::FB_MAX_H]; + let mut vc2_ram = vec![0u16; 32768]; + for round in 0..12 { + let (width, height) = [(1280usize, 1024usize), (1024, 768), (640, 480), (333, 211)][round % 4]; + // Planes over the rows the scroll can reach; aux bits that make + // popups and overlays now and then, not everywhere. + for v in fb_rgb[..stride * 1024].iter_mut() { + *v = rng.next(); + } + for v in fb_aux[..stride * 1024].iter_mut() { + let r = rng.next(); + *v = if r & 7 == 0 { r } else { r & !0xc }; + } + for v in did[..stride * height].iter_mut() { + *v = rng.below(32) as u8; + } + for v in vc2_ram.iter_mut() { + *v = rng.next() as u16; + } + let mut xmap = [0u32; 32]; + for m in xmap.iter_mut() { + *m = rng.next() & 0x00ff_ffff; + } + let mut cmap = [0u32; 8192]; + for c in cmap.iter_mut() { + *c = rng.next() & 0x00ff_ffff; + } + let mut ramdac = [0u32; 256]; + for r in ramdac.iter_mut() { + *r = rng.next() & 0x00ff_ffff; + } + let mut vc2_regs = [0u16; 32]; + vc2_regs[VC2_REG_CURRENT_CURSOR_X as usize] = rng.below(width as u32 + 64) as u16; + vc2_regs[VC2_REG_WORKING_CURSOR_Y as usize] = rng.below(height as u32 + 64) as u16; + vc2_regs[VC2_REG_CURSOR_ENTRY_PTR as usize] = [0u16, 0x3f00, 0x7fc0, 0x7ff0][rng.below(4) as usize]; + vc2_regs[VC2_REG_DISPLAY_CONTROL as usize] = + if rng.below(4) != 0 { VC2_CTRL_CURSOR_EN } else { 0 } | if rng.below(2) == 0 { VC2_CTRL_CURSOR_SIZE } else { 0 }; + let src = CompositorSource { + fb_rgb: &fb_rgb, + fb_aux: &fb_aux, + fb_rgb_damage: &[], + fb_aux_damage: &[], + did: &did, + xmap_mode: &xmap, + xmap_config: 0, + xmap_cursor_cmap: rng.below(256) as u8, + xmap_popup_cmap: rng.below(256) as u8, + cmap: &cmap, + ramdac: &ramdac, + vc2_ram: &vc2_ram, + vc2_regs: &vc2_regs, + topscan: rng.below(1024) as usize, + cursor_x_adjust: rng.below(9) as i32 - 4, + width, + height, + status_bar_only: false, + prebuilt: None, + }; + sw.compose_pixels(&src); + let mut out = vec![0u32; width * height]; + scan.run(&src, &mut out).unwrap_or_else(|e| panic!("{e}")); + let want = sw.pixels(); + let mut bad = 0; + for y in 0..height { + for x in 0..width { + let s = want[y * stride + x]; + let w = ((s & 0xff) << 16) | (s & 0xff00) | ((s >> 16) & 0xff); + let g = out[y * width + x]; + if g != w { + if bad < 4 { + eprintln!("round {round} ({x}, {y}): gpu {g:06x} software {w:06x}, mode {:#x}", xmap[(did[y * stride + x] & 31) as usize]); + } + bad += 1; + } + } + } + assert_eq!(bad, 0, "round {round}: {bad} pixels differ"); + } +} diff --git a/src/dev/odyssey/board.rs b/src/dev/odyssey/board.rs index fbe3f7601..888426ca9 100644 --- a/src/dev/odyssey/board.rs +++ b/src/dev/odyssey/board.rs @@ -166,6 +166,8 @@ pub(super) struct ImageRaster { /// The fragment state and depth word a depth-tested image's pixels take, /// decoded once per box (Board::image_box_loaded()). pub fragment: Option>, + /// The slot those fragments go through on the GPU unit. + pub gpu_slot: Option, pub cur_x: u32, pub cur_y: u32, pub fmt: u32, @@ -742,6 +744,7 @@ impl Default for Board { y2: 0, box_words: [0; 10], fragment: None, + gpu_slot: None, cur_x: 0, cur_y: 0, fmt: 0, @@ -1246,6 +1249,13 @@ impl Board { // CHOICE: memory reads zero after a reset (the board's SDRAM holds // whatever it held; the PROM and the driver clear what they use). self.vram.words_mut().fill(0); + if !self.raster.threaded() { + self.raster.run_now(|ctx| { + if let Some(g) = &mut ctx.gpu { + g.clear_all(); + } + }); + } self.stream.poly_marks.fill(0); // Back to the boot size, and restart the phase. let mode = self.scanout_mode(); @@ -1279,8 +1289,40 @@ impl Board { s.slot = slot; s.configure()?; s.allocate()?; + // VPRO_GPU=1 puts every test board on the GPU unit. + #[cfg(all(test, feature = "wgpu"))] + if std::env::var("VPRO_GPU").is_ok() { + s.enable_gpu(&crate::wgpu::GpuOptions::default())?; + } Ok(s) } + + /// Draw on the host GPU from here on (gpu_unit.rs): the raster unit's + /// work goes into batches on a GPU buffer, and this memory becomes the + /// front end's mirror of it, brought current by `raster_wait`. Before + /// the threads start. + #[cfg(feature = "wgpu")] + pub fn enable_gpu(&mut self, opts: &crate::wgpu::GpuOptions) -> Result<(), String> { + use super::wgpu::{GpuRaster, PLANE_SPAN}; + + if self.raster.threaded() { + return Err("the GPU unit must be chosen before the board's threads start".into()); + } + let gpu = crate::wgpu::Gpu::new(opts)?; + let mut unit = GpuRaster::new(&gpu, self.vram.size(), PLANE_SPAN, PLANE_SPAN)?; + self.vram.track_dirty(); + unit.attach_mirror(self.vram.ptr()); + // The display composes on the GPU too, from the same memory. + let (vram, words) = unit.vram_buffer(); + let scanout = super::wgpu::scanout::VproScanout::new(&gpu, &vram, words)?; + let _ = self.display.gpu_scanout.set(parking_lot::Mutex::new(scanout)); + // Flips latch only once the unit has submitted the drawing before + // them (HostInterface::retrace). + unit.publish_stream(self.display.covered.clone()); + crate::dlog!(crate::devlog::LogModule::Odyssey, "sgi-odyssey: raster unit on the GPU: {}", gpu.caps); + self.raster.run_now(|ctx| ctx.gpu = Some(Box::new(unit))); + Ok(()) + } } impl Board { diff --git a/src/dev/odyssey/copy.rs b/src/dev/odyssey/copy.rs index 08a0b1f10..d0f9d8b57 100644 --- a/src/dev/odyssey/copy.rs +++ b/src/dev/odyssey/copy.rs @@ -46,6 +46,7 @@ //! address is not tested: the X server's window moves copy outside the clip //! rectangles in force (traced), as screen-to-screen copies do. +use super::pipe::backend::DestMem; use super::bits::sextract32; use super::board::Board; use super::consts::{tex, ODSY_DID_MAIN_SHIFT, ODSY_PLANE_MASK_SHIFT}; @@ -151,7 +152,8 @@ impl Board { /// is the packing it asks for, not the window's. None where the mode /// names no field there. pub fn drawable_codec(&self, x: i32, y: i32, at: u8) -> Option { - let did = self.dest_mem().did.load1(x, y)? as u32; + // The id bytes are the front end's own: no wait (Board::id_compare_passes). + let did = DestMem::of(self.vram_ptr(), &self.dest_views()).did.load1(x, y)? as u32; let mode = self.regs().dispmode((did >> ODSY_DID_MAIN_SHIFT) as usize); super::frame::mode_fields(mode).into_iter().find(|f| f.at == at).map(|f| f.codec) } @@ -162,10 +164,15 @@ impl Board { let (w, h) = c.size(); let (src, dst) = (c.src(), c.dst()); - self.raster_wait(); if w == 0 || h == 0 { return; } + // The copy's own rectangles come back below; the CPU unit waits + // for everything. + let gpu = self.gpu_unit(); + if !gpu { + self.raster_wait(); + } // The destination's format is the pixel mode's field at the offset // the copy names; word 7's field size is the unit moved, the colour // as the source holds it (INFERRED from the traces: the library's @@ -227,6 +234,25 @@ impl Board { let raw = src_codec(src.at) == dst_codec && !filter; // The whole source first, so an overlapping copy reads what was there. + if gpu { + // The source and destination rectangles, current in the mirror. + let region = |rows: &Rows, x: i32, y: i32| { + self.raster.run_now(|ctx| { + if let Some(g) = &mut ctx.gpu { + if let Err(e) = g.read_back_view(&rows.v, x, y, x.wrapping_add(w as i32 - 1), y.wrapping_add(h as i32 - 1)) { + crate::dlog!(crate::devlog::LogModule::Odyssey, "sgi-odyssey: gpu readback: {e}"); + } + } + }); + }; + let (sx, sy) = ((src.x as i32).wrapping_add(ox), (src.y as i32).wrapping_add(oy)); + region(&from, sx, sy); + if let Some(f) = &ab { + region(&f[0], sx, sy); + region(&f[1], sx, sy); + } + region(&to, (dst.x as i32).wrapping_add(dx0), (dst.y as i32).wrapping_add(dy0)); + } let n = w as usize * h as usize; let mut words = vec![0u32; n]; let mut raws = vec![0u64; if raw { n } else { 0 }]; diff --git a/src/dev/odyssey/debug.rs b/src/dev/odyssey/debug.rs index 5c64d6537..ce8b6f677 100644 --- a/src/dev/odyssey/debug.rs +++ b/src/dev/odyssey/debug.rs @@ -83,6 +83,7 @@ pub(super) const HELP: &[&str] = &[ "vpro hash the memory hash a replay checkpoint takes", "vpro unimp offsets and command words reported as not modelled", "vpro refusals [reset] bus refusals by reason and register", + "vpro gpu [trace on|off] the GPU raster unit's counters; trace counts readbacks by cause", "vpro trace [fifo,draw,cpu,dma|all] annotated trace (gputrace builds); alone: its state", "vpro trace flush|off write the trace out / close it", "vpro rec [|mark|off] replayable recording: start, mark, stop; alone: its state", @@ -164,6 +165,16 @@ fn write_png(path: &Path, width: usize, height: usize, data: &[u8], color: png:: impl Odyssey { /// The frame the scan-out registers describe now, and board memory. fn screen(&self) -> Result<(Frame, VramPtr, DisplayRegs), String> { + // The GPU unit's mirror is current only after a readback: take one + // when the board can be quiesced, else read it as it is. The CPU + // unit's memory is the board's own, read lock-free as the display + // reads it. + #[cfg(feature = "wgpu")] + if self.state.shared.gpu_scanout.get().is_some() { + if let Some(g) = self.state.quiesce(QUIESCE) { + g.gpu_sync(); + } + } let regs = self.state.shared.regs.read(); let frame = Frame::snapshot(®s, self.displayed()).ok_or_else(|| "vpro: the scan-out is not set up (PB&J 0x2800-0x2809)".to_string())?; let mem = self.state.shared.vram.get().ok_or_else(|| "vpro: board memory is not allocated".to_string())?; @@ -175,9 +186,14 @@ impl Odyssey { self.state.front.lock().displayed_buffer } - /// A state command's view of the board, or the busy answer. + /// A state command's view of the board, or the busy answer. On the GPU + /// raster unit the mirror is brought current first, so commands that + /// read board memory read what the unit drew. fn quiesced(&self) -> Result, String> { - self.state.quiesce(QUIESCE).ok_or_else(|| { + self.state.quiesce(QUIESCE).map(|g| { + g.gpu_sync(); + g + }).ok_or_else(|| { format!( "vpro: board busy for {} ms (FIFO {} words in {} stores, raster {} jobs); try again", QUIESCE.as_millis(), @@ -249,6 +265,12 @@ impl Odyssey { format!("{}\n", memory_hash(&g)) } ["unimp"] => self.unimp_text()?, + ["gpu"] => self.gpu_text().unwrap_or_else(|| "the raster unit is the CPU's (gpu off)\n".to_string()), + ["gpu", "trace", on @ ("on" | "off")] => { + let on = *on == "on"; + self.gpu_run(|g| g.trace_waits(on)).ok_or("the raster unit is the CPU's (gpu off)")?; + format!("gpu readback tracing {}\n", if on { "on" } else { "off" }) + } ["refusals"] => { let mut v = Vec::new(); self.state.refusals.report(&mut v).map_err(|e| e.to_string())?; @@ -265,6 +287,28 @@ impl Odyssey { // ── state commands ─────────────────────────────────────────────────────── + /// Run `f` on the GPU raster unit, if the board draws on one. + fn gpu_run(&self, f: impl FnOnce(&mut super::gpu_unit::GpuRaster) -> T) -> Option { + if self.state.raster.threaded() { + return None; + } + self.state.raster.run_now(|ctx| ctx.gpu.as_mut().map(|g| f(g))) + } + + /// The GPU unit's counters (`GpuRaster::report`) and the GPU scan-out's + /// (frames composed, bytes read back: a frame the window presents from + /// the GPU is not read back), None on the CPU unit. + fn gpu_text(&self) -> Option { + #[cfg_attr(not(feature = "wgpu"), allow(unused_mut))] + let mut out = self.gpu_run(|g| g.report())?; + #[cfg(feature = "wgpu")] + if let Some(s) = self.state.shared.gpu_scanout.get() { + let (frames, bytes) = s.lock().counts(); + out.push_str(&format!(" gpu scanout: {frames} frames composed, {} MiB read back\n", bytes >> 20)); + } + Some(out) + } + fn status_text(&self) -> Result { let mut out = String::new(); let fifo = &self.state.fifo; @@ -299,6 +343,9 @@ impl Odyssey { raster.queued(), if raster.threaded() { "running" } else { "stopped" } ); + if let Some(g) = self.gpu_text() { + out.push_str(&g); + } { let r = self.display.reader.lock(); let _ = writeln!(out, " display: frame {}x{}, {} retraces", r.frame_width, r.frame_height, r.clock.frames); diff --git a/src/dev/odyssey/display.rs b/src/dev/odyssey/display.rs index b220bfbb9..8aba33ce0 100644 --- a/src/dev/odyssey/display.rs +++ b/src/dev/odyssey/display.rs @@ -1,6 +1,7 @@ //! What the board shows: the scanout's colour lookup and gamma, the retrace //! with its swap latching, and the window iris presents the frame in. +use super::pipe::backend::DestMem; use super::board::Board; use super::consts::{ dispmode, lut, overlay, ODSY_CMAP_UNVERIFIED_GROUP, ODSY_DID_MAIN_SHIFT, @@ -43,6 +44,9 @@ pub(super) struct Display { /// The registers the display reads, for the host's cursor query; set /// when the board hands the display out. shared: OnceLock>, + /// Consumers (VNC) that read `screen.rgba` every frame + /// (`GfxDisplay::set_capture`): while any does, frames are read back. + capture: AtomicU32, } struct Local { @@ -67,6 +71,47 @@ impl Display { shown: Mutex::new((0, 0)), status, shared: OnceLock::new(), + capture: AtomicU32::new(0), + } + } + + /// Whether this frame can stay on the GPU: the installed renderer takes + /// frames on the board's device, and nothing wants `screen.rgba` filled + /// (a screenshot, a frame request, a capturing VNC client). + #[cfg(feature = "wgpu")] + pub(super) fn gpu_frames_wanted(&self) -> bool { + !self.screenshot.load(Ordering::Acquire) + && !self.frame.load(Ordering::Acquire) + && self.capture.load(Ordering::Acquire) == 0 + && self.renderer.lock().as_ref().is_some_and(|r| r.takes_gpu_frames()) + } + + /// Present a frame composed on the GPU (0x00RRGGBB, rows of `width`) + /// through the installed renderer; `screen.rgba` keeps its last frame. + #[cfg(feature = "wgpu")] + pub(super) fn present_gpu(&self, frame: &wgpu::Buffer, width: u32, height: u32) { + let (w, h) = (width as usize, height as usize); + let mut screen = self.screen.lock(); + screen.width = w.min(FB_STRIDE); + screen.height = h; + screen.prebuilt = true; + screen.status_bar_only = false; + *self.shown.lock() = (w, h); + let mut renderer = self.renderer.lock(); + if let Some(renderer) = renderer.as_mut() { + let mut local = self.local.lock(); + let local = local.get_or_insert_with(|| Local { + overlay: crate::debug_overlay::DebugOverlay::new(), + status: crate::disp::StatusBar::new(), + texture: crate::disp::StatusBarTexture::new(), + size: (0, 0), + }); + if local.size != (w, h) { + renderer.resize(w, h); + local.size = (w, h); + } + let stats = self.status.snapshot(std::time::Instant::now(), display_fifo_pending()); + renderer.present_gpu_frame(frame, width, height, &mut screen, &mut local.overlay, &mut local.status, &mut local.texture, &stats); } } @@ -165,7 +210,8 @@ impl Board { /// Pixel (x, y)'s main window id: bits 7:3 of its window-id byte in the /// buffer the scan-out names (PB&J 0x2800 base, 0x2807 stride). pub fn main_id(&mut self, x: u32, y: u32) -> u32 { - let did = self.dest_mem().did.load1(x as i32, y as i32).unwrap_or(0); + // The id bytes are the front end's own: no wait (Board::id_compare_passes). + let did = DestMem::of(self.vram_ptr(), &self.dest_views()).did.load1(x as i32, y as i32).unwrap_or(0); ((did as u32) >> ODSY_DID_MAIN_SHIFT) & (dispmode::COUNT - 1) } @@ -307,8 +353,19 @@ impl crate::gfx_display::GfxDisplay for Display { fn frame_pending(&self) -> bool { self.frame.load(Ordering::Acquire) } + fn set_capture(&self, on: bool) { + if on { + self.capture.fetch_add(1, Ordering::AcqRel); + } else { + let _ = self.capture.fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| n.checked_sub(1)); + } + } fn hw_cursor(&self) -> Option { // A racy read of the registers, as the display thread's frame does. self.shared.get()?.regs.hw_cursor() } + #[cfg(feature = "wgpu")] + fn gpu(&self) -> Option> { + Some(self.shared.get()?.gpu_scanout.get()?.lock().gpu()) + } } diff --git a/src/dev/odyssey/dma.rs b/src/dev/odyssey/dma.rs index de1314885..c8a48c41f 100644 --- a/src/dev/odyssey/dma.rs +++ b/src/dev/odyssey/dma.rs @@ -161,7 +161,18 @@ impl Board { let pixel = self.decode_pixel(code, src).unwrap_or(0); let index = code_is_index(code); let word = if index { pixel } else { (pixel & !0xffu32) | 0xff }; - to.store_word(x as i32, y as i32, self.image_transfer(word, index)); + let word = self.image_transfer(word, index); + #[cfg(feature = "wgpu")] + if self.gpu_unit() { + // A blit into the put's own surface, in order, no wait. + if let Ok(view) = super::gpu_unit::Slot::own_view(to) { + let key = super::gpu_unit::BlitKey { view: super::gpu_unit::BlitView::Own(view), ..super::gpu_unit::BlitKey::store(0, u32::MAX) }; + let slot = super::gpu_unit::Slot::none(); + self.raster.run_now(|ctx| ctx.gpu.as_mut().expect("the unit").blit_pixel(&slot, &key, x as i32, y as i32, word)); + return; + } + } + to.store_word(x as i32, y as i32, word); } /// The A and B fields of the surface a DMA header names: word 0's @@ -169,8 +180,12 @@ impl Board { /// 2's stride in tiles, word 4's pixel size (19:16), field size (15:13) /// and field offsets (23:20 A, 27:24 B, in 2-byte units), the field's /// code and class (7:0) for its packing. - pub fn dma_fields(&self, hdr: &[u32], read: bool) -> ([Rows; 2], bool) { - self.raster_wait(); + /// With `wait`, after the raster unit's queued work; a put into memory + /// no drawing touches skips it on the GPU unit (`dma_to_buffer`). + pub fn dma_fields(&self, hdr: &[u32], read: bool, wait: bool) -> ([Rows; 2], bool) { + if wait { + self.raster_wait(); + } let base = hdr[dma::HDR_DEST]; let fmt = hdr[dma::HDR_FORMAT]; let drawable = base & dma::SENTINEL != 0; @@ -246,16 +261,27 @@ impl Board { /// count runs out, then the cursor steps over skip uncounted bytes. No byte /// outside the entries is read. pub fn dma_to_buffer(&mut self, bus: &DmaBus, table: u64, seq: u64, hdr: &[u32]) { - let ([a, b], drawable) = self.dma_fields(hdr, false); - // Word 4 bit 31: the front (A) field, else the back (B). - let to = if hdr[dma::HDR_FORMAT] & dma::FMT_FRONT != 0 { a } else { b }; // Memory, not the screen: a scratch surface the pixel mode selects, // or any address but the frame buffer the scan-out shows. Its texels // keep the source's alpha and pass no window tests. The library // uploads a texture or a zoomed image's scratch while the window is // still the pixel mode (traced), so the select alone does not say. + let drawable = hdr[dma::HDR_DEST] & dma::SENTINEL != 0; let screen = self.pbj(super::frame::pbj::PIXEL_BASE); let memory = Format::decode(self.pixel_mode[2]).scratch || (!drawable && hdr[dma::HDR_DEST] & ADDR_MASK != screen); + // On the GPU unit a put into memory the drawing does not touch + // need not wait for it: the texels go into the mirror and up with + // the next batch, and the draws that sample them have already run + // inside their submit. + let direct = memory && self.gpu_unit(); + let ([a, b], drawable) = self.dma_fields(hdr, false, !direct); + // Word 4 bit 31: the front (A) field, else the back (B). + let mut to = if hdr[dma::HDR_FORMAT] & dma::FMT_FRONT != 0 { a } else { b }; + if direct { + // Stored into the mirror untracked and uploaded as the pixels + // written (GpuRaster::upload_region), below. + to = Rows::new(to.mem().untracked(), to.v); + } let obj = memory.then_some(()); let mut e = [DmaEntry::default(); dma::MAX_PAGES]; let x = hdr[dma::HDR_ORIGIN] & 0xffff; @@ -357,6 +383,14 @@ impl Board { } } } + if direct { + let (x0, y0) = (x as i32, y as i32); + self.raster.run_now(|ctx| { + if let Some(g) = &mut ctx.gpu { + g.upload_region(&to.v, x0, y0, x0.wrapping_add(w as i32 - 1), y0.wrapping_add(h as i32 - 1)); + } + }); + } } } @@ -439,21 +473,27 @@ pub(super) fn dma_draw_sentinel_form(hdr: &[u32], top_down: &mut bool) -> Option impl Board { #[cfg(test)] /// The depth plane's word at plane pixel (x, y), which is in the plane. - /// The destination's depth and stencil fields (Board::dest_mem()). + /// The destination's depth and stencil fields (Board::planes_in()). pub fn depth_load(&mut self, x: i64, y: i64) -> u32 { - self.dest_mem().depth.load_word(x as i32, y as i32).unwrap_or(0) + self.planes_in(x, y, x, y).depth.load_word(x as i32, y as i32).unwrap_or(0) } pub fn depth_store(&mut self, x: i64, y: i64, word: u32) { - self.dest_mem().depth.store_word(x as i32, y as i32, word); + if self.gpu_blit_view(2, x, y, word, u32::MAX) { + return; + } + self.planes_in(x, y, x, y).depth.store_word(x as i32, y as i32, word); } pub fn stencil_load(&mut self, x: i64, y: i64) -> u32 { - self.dest_mem().stencil.load1(x as i32, y as i32).unwrap_or(0) as u32 + self.planes_in(x, y, x, y).stencil.load1(x as i32, y as i32).unwrap_or(0) as u32 } pub fn stencil_store(&mut self, x: i64, y: i64, value: u8) { - self.dest_mem().stencil.store1(x as i32, y as i32, value); + if self.gpu_blit_view(3, x, y, value as u32, u32::MAX) { + return; + } + self.planes_in(x, y, x, y).stencil.store1(x as i32, y as i32, value); } /// Stage the pixel bytes first so a failed source read cannot leave a @@ -822,7 +862,9 @@ impl Board { } return false; } - let (fields, drawable) = self.dma_fields(hdr, true); + // On the GPU unit the read's own rectangle comes back below + // (dma_from_board), not the whole drawing. + let (fields, drawable) = self.dma_fields(hdr, true, !self.gpu_unit()); let a = fields[0].v; // Depth and stencil through the A field's place: a 24-bit word and // its stencil byte (field.rs). @@ -1107,6 +1149,12 @@ impl Board { if (!raw && l.is_none()) || !self.dma_read_source(hdr, &mut src) { return; } + if self.gpu_unit() { + // What the read covers, current in the mirror. + let (x0, y0, x1, y1) = (src.x, src.y, src.x + w as i64 - 1, src.y + h as i64 - 1); + let planes = super::pipe::backend::DestMem { colour: src.fields, depth: src.depth, stencil: src.stencil, did: src.fields[0], overlay: src.fields[1] }; + self.gpu_read_back(&planes, x0, y0, x1, y1); + } let Ok(n) = bus.entries(table, seq, &mut e) else { return; }; @@ -1647,13 +1695,24 @@ impl Board { }; let mem = self.vram_ptr(); let from = super::view::Rows::new(mem, super::view::View::colour(&surface, field, super::view::MAX_DIM)); - let to = self.dest_mem().colour[0]; + // The destination is a texture level: on the GPU unit it is stored + // untracked and uploaded as a rectangle, with no wait (as a scratch + // image through the FIFO, Board::image_put()). + let gpu = self.gpu_unit(); + let to = if gpu { super::pipe::backend::DestMem::of(mem.untracked(), &self.dest_views()).colour[0] } else { self.dest_mem().colour[0] }; for y in 0..hd { for x in 0..wd { let v = from.load_word(2 * x as i32, 2 * y as i32).unwrap_or(0); to.store_word(x as i32, y as i32, v); } } + if gpu && wd > 0 && hd > 0 { + self.raster.run_now(|ctx| { + let g = ctx.gpu.as_mut().expect("the unit"); + g.note_region(&to.v, 0, 0); + g.note_region(&to.v, wd as i32 - 1, hd as i32 - 1); + }); + } true } diff --git a/src/dev/odyssey/draw.rs b/src/dev/odyssey/draw.rs index 7fd0f0191..9e84dca90 100644 --- a/src/dev/odyssey/draw.rs +++ b/src/dev/odyssey/draw.rs @@ -48,19 +48,53 @@ impl Board { /// the front end's own stores: it waits for the raster unit's queued /// jobs first. pub fn dest_mem(&self) -> DestMem { - self.raster_wait(); + // Everything the drawing wrote, back: the general path; the hot + // ones use planes_in() or store through the unit. + self.gpu_sync(); DestMem::of(self.vram_ptr(), &self.dest_views()) } + /// The destination for a front-end path that reads or writes the + /// pixels of one box (inclusive): on the GPU unit, only the blocks the + /// drawing touched under that box come back, and nothing waits when + /// none did; with the CPU unit, `dest_mem`. + pub fn planes_in(&self, x0: i64, y0: i64, x1: i64, y1: i64) -> DestMem { + if !self.gpu_unit() { + return self.dest_mem(); + } + let planes = DestMem::of(self.vram_ptr(), &self.dest_views()); + self.gpu_read_back(&planes, x0, y0, x1, y1); + planes + } + + /// Bring the mirror current under a box of every plane of `planes` + /// (GpuRaster::read_back_view). + pub fn gpu_read_back(&self, planes: &DestMem, x0: i64, y0: i64, x1: i64, y1: i64) { + let clamp = |v: i64| v.clamp(-1, MAX_DIM as i64) as i32; + let (x0, y0, x1, y1) = (clamp(x0), clamp(y0), clamp(x1), clamp(y1)); + self.raster.run_now(|ctx| { + if let Some(g) = &mut ctx.gpu { + for rows in [&planes.colour[0], &planes.colour[1], &planes.depth, &planes.stencil, &planes.did, &planes.overlay] { + if let Err(e) = g.read_back_view(&rows.v, x0, y0, x1, y1) { + crate::dlog!(LogModule::Odyssey, "sgi-odyssey: gpu readback: {e}"); + } + } + } + }); + } + /// Field b of the destination: 0 the A (front) field, 1 the B (back). pub fn buffer_load(&self, b: u32, x: i64, y: i64) -> u32 { - self.dest_mem().colour[(b & 1) as usize].load_word(x as i32, y as i32).unwrap_or(0) + self.planes_in(x, y, x, y).colour[(b & 1) as usize].load_word(x as i32, y as i32).unwrap_or(0) } /// Store field b of the destination. A byte field (the window-id /// buffer's id and overlay bytes) is stored under the plane mask. pub fn buffer_store(&mut self, b: u32, x: i64, y: i64, word: u32) { - let rows = self.dest_mem().colour[(b & 1) as usize]; + if self.gpu_blit_buffers(1 << (b & 1), x, y, word, None, u32::MAX, 0) { + return; + } + let rows = self.planes_in(x, y, x, y).colour[(b & 1) as usize]; if rows.v.kind == Kind::Byte { let mask = (self.plane_write_mask >> ODSY_PLANE_MASK_SHIFT) as u8; if let Some(old) = rows.load1(x as i32, y as i32) { @@ -155,7 +189,9 @@ impl Board { pub fn id_compare_passes(&self, x: i64, y: i64) -> bool { let c = self.win.id_compare; let mask = (c >> ODSY_ID_COMPARE_MASK_SHIFT) as u8; - c & ODSY_ID_COMPARE_ENABLE == 0 || self.dest_mem().did.load1(x as i32, y as i32).is_some_and(|id| (id ^ c as u8) & mask == 0) + // The id bytes are the X server's, written by the front end alone, + // never by the raster unit: no wait before reading them. + c & ODSY_ID_COMPARE_ENABLE == 0 || DestMem::of(self.vram_ptr(), &self.dest_views()).did.load1(x as i32, y as i32).is_some_and(|id| (id ^ c as u8) & mask == 0) } /// The one pixel test every write but a copy destination passes: the plane @@ -227,12 +263,102 @@ impl Board { if !self.pixel_allowed(x, y) { return; } + if self.gpu_paint(buffers, x, y, pixel) { + return; + } for b in 0..ODSY_BUFFERS { if buffers & (1u32 << b) != 0 { self.buffer_store(b, x, y, pixel); } } } + + /// `paint_pixel`'s stores on the GPU unit, in order behind the drawing + /// and with no wait: the pixel joins a blit record. False without the + /// unit, or for planes its kernels cannot address. + fn gpu_paint(&mut self, buffers: u32, x: i64, y: i64, pixel: u32) -> bool { + self.gpu_blit_buffers(buffers, x, y, pixel, None, u32::MAX, 0) + } + + /// A store into each colour buffer of `buffers` on the GPU unit, with + /// `op` combining the pixel with the old word first and its result + /// and-ed and or-ed (Board::paint_integer, paint_aux); a byte field is + /// stored under the plane mask (Board::buffer_store). False without + /// the unit, or for planes its kernels cannot address. + fn gpu_blit_buffers(&mut self, buffers: u32, x: i64, y: i64, pixel: u32, op: Option, and: u32, or: u32) -> bool { + if !self.gpu_unit() { + return false; + } + let planes = DestMem::of(self.vram_ptr(), &self.dest_views()); + let Some(slot) = self.gpu_dest_slot(&planes) else { + return false; + }; + let plane_mask = (self.plane_write_mask >> ODSY_PLANE_MASK_SHIFT) as u8; + self.raster.run_now(|ctx| { + let g = ctx.gpu.as_mut().expect("the unit"); + for b in 0..ODSY_BUFFERS { + if buffers & (1u32 << b) == 0 { + continue; + } + let mask = if planes.colour[b as usize].v.kind == Kind::Byte { plane_mask as u32 } else { u32::MAX }; + let key = super::gpu_unit::BlitKey { mask, op: op.map_or(0, |o| 0x100 | o as u32), and, or, ..super::gpu_unit::BlitKey::store(b, mask) }; + g.blit_pixel(&slot, &key, x as i32, y as i32, pixel); + mirror_byte_store(&planes.colour[b as usize], x, y, pixel, &key); + } + }); + true + } + + /// A store into one view of the destination (`backend.wgsl` V_*) on the + /// GPU unit, the bits of `mask`. False without the unit. + pub fn gpu_blit_view(&mut self, view: u32, x: i64, y: i64, word: u32, mask: u32) -> bool { + if !self.gpu_unit() { + return false; + } + let planes = DestMem::of(self.vram_ptr(), &self.dest_views()); + let Some(slot) = self.gpu_dest_slot(&planes) else { + return false; + }; + let key = super::gpu_unit::BlitKey::store(view, mask); + self.raster.run_now(|ctx| ctx.gpu.as_mut().expect("the unit").blit_pixel(&slot, &key, x as i32, y as i32, word)); + let rows = [planes.colour[0], planes.colour[1], planes.depth, planes.stencil, planes.did, planes.overlay]; + if let Some(r) = rows.get(view as usize) { + mirror_byte_store(r, x, y, word, &key); + } + true + } + + /// The slot of the destination and window for the front end's own + /// stores, when the kernels can address the planes. + pub fn gpu_dest_slot(&self, planes: &DestMem) -> Option { + let words = super::pipe::state::RasterWords { clip_rect: self.win.clip_rect, clip_kept: self.win.clip_kept, id_compare: self.win.id_compare, ..Default::default() }; + super::gpu_unit::Slot::clear(planes, &words).ok() + } +} + +/// The window-id buffer's bytes (a byte field: the id and overlay bytes) +/// are read by the front end itself, with no wait for the raster unit +/// (`Board::id_compare_passes`, `drawable_codec`, `main_id`). On the GPU unit +/// the unit stores them, so the front end makes each such store in the +/// mirror too, as the blit kernel makes it (vpro.wgsl `blit`), through an +/// untracked pointer: the mirror's id bytes stay current, nothing is +/// uploaded twice, and a later readback of the block brings back the same +/// bytes. Other fields are left to the unit and its readbacks. +fn mirror_byte_store(rows: &super::view::Rows, x: i64, y: i64, word: u32, key: &super::gpu_unit::BlitKey) { + if rows.v.kind != Kind::Byte || key.window { + return; + } + let rows = super::view::Rows::new(rows.mem().untracked(), rows.v); + let (x, y) = (x as i32, y as i32); + let Some(old) = rows.load1(x, y) else { return }; + let old = old as u32; + let new = if key.op == 0 && key.mask == u32::MAX { + word + } else { + let val = if key.op != 0 { (logic_op((key.op & 0xf) as u8, word, old) & key.and) | key.or } else { word }; + (old & !key.mask) | (val & key.mask) + }; + rows.store1(x, y, new as u8); } /// The plane word a fill stores: the packed RGB pixel of an RGB fill, or the @@ -252,6 +378,9 @@ impl Board { return; } let buffers = self.draw_buffers(); + if self.gpu_blit_buffers(buffers, x, y, index, (op as u32 != LOGIC_COPY).then_some(op), 0xff, 0) { + return; + } for b in 0..ODSY_BUFFERS { if buffers & (1u32 << b) != 0 { let byte = if op as u32 == LOGIC_COPY { index } else { logic_op(op, index, self.buffer_load(b, x, y)) }; @@ -290,6 +419,9 @@ impl Board { if !self.pixel_allowed(x, y) { return; } + if self.gpu_blit_buffers(buffers, x, y, src, Some(op), if is_index { ODSY_INDEX_MASK } else { u32::MAX }, if is_index { 0 } else { 0xff }) { + return; + } for b in 0..ODSY_BUFFERS { if buffers & (1u32 << b) != 0 { let out = logic_op(op, src, self.buffer_load(b, x, y)); @@ -329,6 +461,8 @@ impl Board { pub fn clear_planes(&mut self, x: u32, y: u32, w: u32, h: u32, value: u8, mask: u8) { let buffers = self.draw_buffers(); let id_test = self.id_compare_active(); + let gpu = self.gpu_unit(); + let planes = if gpu { DestMem::of(self.vram_ptr(), &self.dest_views()) } else { self.planes_in(x as i64, y as i64, x as i64 + w as i64 - 1, y as i64 + h as i64 - 1) }; for row in 0..h { let py = y as i64 + row as i64; let runs = self.row_runs(py); @@ -338,9 +472,20 @@ impl Board { for px in a..e { if !id_test || self.id_compare_passes(px, py) { + if gpu { + for b in 0..ODSY_BUFFERS { + if buffers & (1u32 << b) != 0 && !self.gpu_blit_view(b, px, py, value as u32, mask as u32) { + let rows = self.planes_in(px, py, px, py).colour[b as usize]; + if let Some(old) = rows.load1(px as i32, py as i32) { + rows.store1(px as i32, py as i32, (old & !mask) | (value & mask)); + } + } + } + continue; + } for b in 0..ODSY_BUFFERS { if buffers & (1u32 << b) != 0 { - let rows = self.dest_mem().colour[b as usize]; + let rows = planes.colour[b as usize]; if let Some(old) = rows.load1(px as i32, py as i32) { rows.store1(px as i32, py as i32, (old & !mask) | (value & mask)); } @@ -488,14 +633,33 @@ impl Board { } let word = if is_index || preserve_alpha { rgba } else { (rgba & !0xffu32) | 0xff }; if self.dest_views().scratch { - let rows = self.dest_mem().colour[0]; - rows.store_word(self.image.cur_x as i32, self.image.cur_y as i32, word); + // A texture or scratch image: memory no drawing touches. On the + // GPU unit it goes into the mirror untracked and up as a + // rectangle when the image is done (GpuRaster::note_region), + // with no wait. + let (x, y) = (self.image.cur_x as i32, self.image.cur_y as i32); + if self.gpu_unit() { + let rows = DestMem::of(self.vram_ptr().untracked(), &self.dest_views()).colour[0]; + rows.store_word(x, y, word); + self.raster.run_now(|ctx| ctx.gpu.as_mut().expect("the unit").note_region(&rows.v, x, y)); + } else { + let rows = self.dest_mem().colour[0]; + rows.store_word(x, y, word); + } } else if let Some(f) = self.image.fragment.as_deref().filter(|_| !is_index) { let word = self.image_transfer(word, is_index); let (x, y) = (self.image.cur_x as i64 + self.win.origin[0] as i64, self.image.cur_y as i64 + self.win.origin[1] as i64); if self.pixel_allowed(x, y) { let src = [word >> 24, word >> 16, word >> 8, word].map(|c| widen8(c & 0xff)); - f.0.fragment_q(&self.dest_mem(), x as i32, y as i32, f.1, src); + let zq = f.1; + match self.image.gpu_slot { + // The unit tests and stores it in order, as a fragment. + Some(slot) => { + let q = super::pipe::ring::BackQuad { x: x as i32, y: y as i32, cov: 1, zq: [zq, 0, 0, 0], rgba: [src.map(|c| c as u16), [0; 4], [0; 4], [0; 4]], ..Default::default() }; + self.raster.run_now(|ctx| ctx.gpu.as_mut().expect("the unit").push_frag(&q, &slot)); + } + None => f.0.fragment_q(&self.dest_mem(), x as i32, y as i32, zq, src), + } } } else { let word = self.image_transfer(word, is_index); @@ -531,9 +695,17 @@ impl Board { let z01 = (z / (2.0 * super::consts::gl_cmd::DEPTH_S)).clamp(0.0, 1.0); let [n, f] = self.assembly.xf.depth_range.map(f64::from); let zq = word_for_fraction((n + (f - n) * z01).clamp(0.0, 1.0)); - let backend = BackendState::of(&self.fragment_words().backend, true, true); + let words = self.fragment_words(); + let backend = BackendState::of(&words.backend, true, true); + // On the GPU unit the image's pixels go through its back end as + // fragments under this slot. + self.image.gpu_slot = None; + if self.gpu_unit() { + let planes = DestMem::of(self.vram_ptr(), &self.dest_views()); + self.image.gpu_slot = super::gpu_unit::Slot::new(&words.backend, &planes, false, false, &words.raster, None, None).ok(); + } if self.trace.on(super::trace::TRACE_FIFO) { - let depth_kind = self.dest_mem().depth.v.kind; + let depth_kind = self.dest_views().depth.kind; self.trace_note(format_args!( "image depth {z01} -> word {zq:#x}: depth test {} write {} func {:?}, targets {:?}, depth plane {:?}", backend.depth_test, backend.depth_write, backend.depth_func, backend.targets, depth_kind diff --git a/src/dev/odyssey/fifo.rs b/src/dev/odyssey/fifo.rs index bdce82ec4..f582e8cf0 100644 --- a/src/dev/odyssey/fifo.rs +++ b/src/dev/odyssey/fifo.rs @@ -46,7 +46,7 @@ use super::consts::{cfifo, intr, ODSY_WORK_QUEUE_MAX}; use super::interface::HostInterface; use super::worker::GAP; use std::cell::Cell; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering::{Relaxed, SeqCst}}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering::{Relaxed, SeqCst}}; use std::thread::JoinHandle; use crossbeam_utils::CachePadded; @@ -126,6 +126,12 @@ struct Levels { words: AtomicU32, /// Stores pushed and not yet taken out. stores: AtomicU32, + /// Positions in the stream of stores: how many have ever been pushed, + /// and how many the board has run (retired or abandoned). A swap arm + /// notes the first; the GPU raster unit publishes how far behind the + /// second its submissions reach (`DisplayShared::covered`). + pushed: AtomicU64, + retired: AtomicU64, } pub(super) struct CommandFifo { @@ -182,7 +188,7 @@ impl Default for CommandFifo { // place because the ring is too big for the stack. ring: unsafe { Box::new_zeroed().assume_init() }, pop: Mutex::new(()), - levels: CachePadded::new(Levels { words: AtomicU32::new(0), stores: AtomicU32::new(0) }), + levels: CachePadded::new(Levels { words: AtomicU32::new(0), stores: AtomicU32::new(0), pushed: AtomicU64::new(0), retired: AtomicU64::new(0) }), threshold_6b2: CachePadded::new(AtomicU32::new(0)), running: CachePadded::new(AtomicBool::new(false)), asleep: CachePadded::new(AtomicBool::new(false)), @@ -234,6 +240,17 @@ impl CommandFifo { self.level() != 0 } + /// Stores ever pushed: the position in the stream after the last one. + pub fn pushed(&self) -> u64 { + self.levels.pushed.load(SeqCst) + } + + /// Stores the board has run or abandoned: the position in the stream + /// its state reflects. + pub fn retired(&self) -> u64 { + self.levels.retired.load(SeqCst) + } + /// Whether the level is at or above the model's high-water threshold. pub fn high_water(&self) -> bool { let mark = self.threshold_6b2.load(SeqCst); @@ -337,6 +354,7 @@ impl CommandFifo { self.levels.stores.fetch_sub(1, SeqCst); return false; } + self.levels.pushed.fetch_add(1, SeqCst); #[cfg(test)] if let Some(consumer) = self.test_after_ring_push.lock().take() { consumer(self, front); @@ -383,6 +401,7 @@ impl CommandFifo { let words: u32 = ran.iter().map(QueuedStore::words).sum(); let _c = self.pop.lock(); self.ring.consume_n_at(0, n); + self.levels.retired.fetch_add(n as u64, SeqCst); // Publish the head with the run, so the room the counts below give // back is room the ring shows a producer too (has_room). self.ring.flush_head(); @@ -409,6 +428,7 @@ impl CommandFifo { let q = QueuedStore::from_entry(addr, value); self.levels.words.fetch_sub(q.words(), SeqCst); self.levels.stores.fetch_sub(1, SeqCst); + self.levels.retired.fetch_add(1, SeqCst); } } diff --git a/src/dev/odyssey/frame.rs b/src/dev/odyssey/frame.rs index b74ee99bc..334241d22 100644 --- a/src/dev/odyssey/frame.rs +++ b/src/dev/odyssey/frame.rs @@ -90,12 +90,12 @@ pub(super) struct PixelParts { pub(super) struct Frame { pub width: u32, pub height: u32, - fb: Surface, + pub(super) fb: Surface, /// Per main window id, its two colour fields (mode_fields()). - fields: [[Field; 2]; dispmode::COUNT as usize], - wid: Surface, - wid_layout: Layout, - ctx: ResolveCtx, + pub(super) fields: [[Field; 2]; dispmode::COUNT as usize], + pub(super) wid: Surface, + pub(super) wid_layout: Layout, + pub(super) ctx: ResolveCtx, } impl Frame { diff --git a/src/dev/odyssey/gl.rs b/src/dev/odyssey/gl.rs index 2a28ee01a..aca198fcd 100644 --- a/src/dev/odyssey/gl.rs +++ b/src/dev/odyssey/gl.rs @@ -285,7 +285,8 @@ impl GlDrawJob { // SAFETY: the job holds its slot until it is given back. let st: &'static DrawState = unsafe { &*job.slot }; let tex = Unit { regs: &st.tex, latch: &st.latch, mem: job.pixels.obj }; - let mut p = PipeDraw::new(&job.words, tex, job.pixels.state.mem, job.index, job.pixels.state.to_aux, xf.depth_range(), job.depth.slack, ctx.ring.as_ref()); + let (ring, gpu) = (ctx.ring.as_ref(), ctx.gpu.as_deref_mut()); + let mut p = PipeDraw::new(&job.words, tex, job.pixels.state.mem, job.index, job.pixels.state.to_aux, xf.depth_range(), job.depth.slack, ring, fr.words_stipple.as_ref(), job.background, gpu); if p.ring.is_none() { ctx.drain_back(); } @@ -606,6 +607,27 @@ impl Board { continue; }; let st = &pixels.state; + if let Some(g) = ctx.gpu.as_deref_mut() { + // The unit clears its part of each rectangle. + let words = super::pipe::state::RasterWords { clip_rect: st.clip_rect, clip_kept: st.clip_kept, id_compare: st.id_compare, ..Default::default() }; + match super::gpu_unit::Slot::clear(&clear.planes, &words) { + Ok(slot) => { + for r in &st.clip_rect[..st.clip_kept as usize] { + let (x0, y0, x1, y1) = (op.x0.max(r.x0), op.y0.max(r.y0), op.x1.min(r.x1), op.y1.min(r.y1)); + if x0 <= x1 && y0 <= y1 { + g.push_rect(x0, y0, x1, y1, &clear, &slot); + } + } + continue; + } + Err(e) => { + crate::dlog!(LogModule::Odyssey, "sgi-odyssey: gpu unit: {} cleared on the CPU", e.0); + if let Err(e) = g.read_back() { + crate::dlog!(LogModule::Odyssey, "sgi-odyssey: gpu readback: {e}"); + } + } + } + } let Some(ring) = &ring else { for y in op.y0..=op.y1 { for r in &st.clip_rect[..st.clip_kept as usize] { diff --git a/src/dev/odyssey/golden/mod.rs b/src/dev/odyssey/golden/mod.rs index 6cfbe3f4a..9942c9ca2 100644 --- a/src/dev/odyssey/golden/mod.rs +++ b/src/dev/odyssey/golden/mod.rs @@ -250,7 +250,7 @@ fn every_scene_draws_the_same_on_the_raster_and_back_end_threads() { c.board.raster.start_threads(true); assert!(scenes::run_scene(name, &mut c)); - c.board.raster_wait(); + c.board.gpu_sync(); let threaded = Frame::of(&c.board); c.board.raster.stop(); assert_eq!(threaded.hash(), inline.hash(), "scene {name}: {:?}", distance(&threaded, &inline, &Tolerance::default())); diff --git a/src/dev/odyssey/golden/replay.rs b/src/dev/odyssey/golden/replay.rs index 9d6f80e4c..5039bd919 100644 --- a/src/dev/odyssey/golden/replay.rs +++ b/src/dev/odyssey/golden/replay.rs @@ -14,7 +14,8 @@ //! `cargo test ... replay_bless -- --ignored` re-takes every recording's //! checkpoints. `VPRO_REPLAY_LIMIT=` stops after n records, so a dump //! shows the board there; `VPRO_REPLAY_THREADS=1` draws on the raster and -//! back-end threads. +//! back-end threads; `VPRO_REPLAY_GPU=1` draws on the GPU raster unit +//! (with the `wgpu` feature). use crate::dev::odyssey::board::{Board, ODSY_HEIGHT, ODSY_WIDTH}; use crate::dev::odyssey::consts::{cmd, gl_cmd, image, reg, ODSY_BUZZ_1}; @@ -39,12 +40,20 @@ pub struct ReplayMem { reads: Vec, next: Mutex, pub mismatches: Mutex, + /// What the board wrote to guest memory (pixel reads, fences, query + /// results), hashed in order with each write's address: two replays of + /// one recording on different raster units read back the same pixels + /// exactly when this matches. + pub writes: Mutex<(blake3::Hasher, u64)>, + /// `VPRO_REPLAY_WRITES=`: each write as a line, address and words. + log: Option>>, } impl ReplayMem { pub fn new(recs: &[Rec]) -> Self { let reads = recs.iter().filter(|r| matches!(r, Rec::DmaBlock { .. } | Rec::DmaByte { .. })).cloned().collect(); - ReplayMem { reads, next: Mutex::new(0), mismatches: Mutex::new(0) } + let log = std::env::var("VPRO_REPLAY_WRITES").ok().and_then(|f| std::fs::File::create(f).ok()).map(|f| Mutex::new(std::io::BufWriter::new(f))); + ReplayMem { reads, next: Mutex::new(0), mismatches: Mutex::new(0), writes: Mutex::new((blake3::Hasher::new(), 0)), log } } pub fn consumed(&self) -> (usize, usize) { (*self.next.lock(), self.reads.len()) @@ -67,7 +76,15 @@ impl BusDevice for ReplayMem { } } } - fn write8(&self, _addr: u64, _val: u8) -> u32 { + fn write8(&self, addr: u64, val: u8) -> u32 { + let mut w = self.writes.lock(); + w.0.update(&addr.to_be_bytes()); + w.0.update(&[val]); + w.1 += 1; + if let Some(l) = &self.log { + use std::io::Write; + let _ = writeln!(l.lock(), "{addr:#x} b {val:02x}"); + } BUS_OK } fn read_block(&self, addr: u64, buf: &mut [u64]) -> u32 { @@ -83,7 +100,22 @@ impl BusDevice for ReplayMem { } } } - fn write_block(&self, _addr: u64, _buf: &[u64]) -> u32 { + fn write_block(&self, addr: u64, buf: &[u64]) -> u32 { + let mut w = self.writes.lock(); + w.0.update(&addr.to_be_bytes()); + for v in buf { + w.0.update(&v.to_be_bytes()); + } + w.1 += 8 * buf.len() as u64; + if let Some(l) = &self.log { + use std::io::Write; + let mut l = l.lock(); + let _ = write!(l, "{addr:#x} {}", buf.len()); + for v in buf { + let _ = write!(l, " {v:016x}"); + } + let _ = writeln!(l); + } BUS_OK } } @@ -98,6 +130,8 @@ pub struct Report { pub dma_total: usize, pub dma_mismatched: u64, pub reads_differ: u64, + /// The board's writes to guest memory: their hash and byte count. + pub host_writes: (String, u64), /// The recording's marks (`vpro rec`), and those whose hash the replayed /// planes did not reach. pub marks: usize, @@ -138,7 +172,31 @@ fn programs_scan_out(recs: &[Rec]) -> bool { /// Feed `recs` through a fresh board's registers, `setup` run on it first. /// `board_out` receives the board when it is done. +/// How a replay draws: on the GPU raster unit or not, with its readbacks +/// skipped (`lazy`, for timing), ending a board slice every `tick` stores. +#[derive(Clone, Copy, Debug, Default)] +pub struct Opts { + pub gpu: bool, + pub lazy: bool, + pub tick: Option, +} + +impl Opts { + /// `VPRO_REPLAY_GPU`, `VPRO_REPLAY_LAZY`, `VPRO_REPLAY_TICK`. + pub fn from_env() -> Opts { + Opts { + gpu: std::env::var("VPRO_REPLAY_GPU").is_ok(), + lazy: std::env::var("VPRO_REPLAY_LAZY").is_ok(), + tick: std::env::var("VPRO_REPLAY_TICK").ok().and_then(|v| v.parse::().ok()).filter(|&n| n > 0), + } + } +} + pub fn run(recs: &[Rec], setup: impl FnOnce(&mut Board), every: usize) -> (Report, Box) { + run_with(recs, setup, every, Opts::from_env()) +} + +pub fn run_with(recs: &[Rec], setup: impl FnOnce(&mut Board), every: usize, opts: Opts) -> (Report, Box) { let mut b = Board::new("v12", 11).expect("board"); // A recording with marks (version 2) was taken from a board the machine // reset at power-on (the depth plane at all ones): start there, as the @@ -156,6 +214,21 @@ pub fn run(recs: &[Rec], setup: impl FnOnce(&mut Board), every: usize) -> (Repor let mem = Arc::new(ReplayMem::new(recs)); b.guest = Some(mem.clone()); setup(&mut b); + // VPRO_REPLAY_GPU=1: draw on the GPU raster unit. + if opts.gpu { + #[cfg(feature = "wgpu")] + { + b.enable_gpu(&crate::wgpu::GpuOptions::default()).expect("the GPU raster unit"); + // VPRO_REPLAY_LAZY=1: no readbacks between checkpoints, to time + // the GPU path alone (the planes the front end reads go stale, + // so the checkpoints are not expected to hold). + if opts.lazy { + b.raster.run_now(|ctx| ctx.gpu.as_mut().expect("the unit").lazy = true); + } + } + #[cfg(not(feature = "wgpu"))] + panic!("VPRO_REPLAY_GPU needs a build with the wgpu feature"); + } // VPRO_REPLAY_THREADS=1: draw on the raster and back-end threads, as a // running machine does; the checkpoints must not move. let threads = std::env::var("VPRO_REPLAY_THREADS").is_ok(); @@ -177,16 +250,27 @@ pub fn run(recs: &[Rec], setup: impl FnOnce(&mut Board), every: usize) -> (Repor // VPRO_REPLAY_EVERY= with VPRO_REPLAY_SHOTS: the screen every n // records too, as rec-NNNNNNNNN.png. let shot_every = std::env::var("VPRO_REPLAY_EVERY").ok().and_then(|v| v.parse::().ok()).filter(|&n| n > 0); + // VPRO_REPLAY_TICK=: end a slice every n stores, as the board's thread + // does between runs of stores (`gpu_upload_tick`, the board going idle): + // the GPU unit then submits mid-stream as it does live. + let tick = opts.tick; + let mut stores = 0usize; for (i, r) in recs.iter().enumerate().take(limit) { if let (Some(n), Some(dir)) = (shot_every, &shots) { if i % n == 0 && i != 0 { - b.raster_wait(); + b.gpu_sync(); let _ = std::fs::create_dir_all(dir); save_screen(&b, &format!("{dir}/rec-{i:09}.png")); } } match r { - Rec::Write { size, off, val } => b.mmio_write(*off as u64, *val, *size as u32), + Rec::Write { size, off, val } => { + b.mmio_write(*off as u64, *val, *size as u32); + stores += 1; + if tick.is_some_and(|n| stores % n == 0) { + b.gpu_upload_tick(true); + } + } Rec::Read { size, off, val } => { let mask = if *size >= 8 { u64::MAX } else { (1u64 << (8 * *size as u32)) - 1 }; if b.mmio_read(*off as u64, *size as u32) & mask != val & mask { @@ -195,7 +279,7 @@ pub fn run(recs: &[Rec], setup: impl FnOnce(&mut Board), every: usize) -> (Repor } Rec::DmaBlock { .. } | Rec::DmaByte { .. } => {} Rec::Mark(h) => { - b.raster_wait(); + b.gpu_sync(); let got = memory_hash_bytes(&b); rep.marks += 1; if got != *h { @@ -212,7 +296,7 @@ pub fn run(recs: &[Rec], setup: impl FnOnce(&mut Board), every: usize) -> (Repor } } if every != 0 && (i + 1) % every == 0 && rep.hashes.last().is_none_or(|(j, _)| *j != i + 1) { - b.raster_wait(); + b.gpu_sync(); rep.hashes.push((i + 1, memory_hash(&b))); if let Some(dir) = &shots { let _ = std::fs::create_dir_all(dir); @@ -220,15 +304,37 @@ pub fn run(recs: &[Rec], setup: impl FnOnce(&mut Board), every: usize) -> (Repor } } } - b.raster_wait(); + b.gpu_sync(); if threads { b.raster.stop(); } + #[cfg(feature = "wgpu")] + if b.gpu_unit() { + let (batches, records, dropped, up, blocks, tiles, down, readbacks) = b.raster.run_now(|ctx| { + let g = ctx.gpu.as_ref().expect("the unit"); + let (batches, records) = g.counts(); + (batches, records, g.dropped, g.uploaded, g.uploaded_blocks, g.uploaded_tiles, g.read_back_bytes, g.readbacks) + }); + eprintln!("gpu unit: {records} records in {batches} batches ({dropped} dropped), {} MiB uploaded ({} MiB dirty blocks, {} MiB regions), {} MiB read back in {readbacks} readbacks", up >> 20, blocks >> 20, tiles >> 20, down >> 20); + b.raster.run_now(|ctx| { + if let Some(by) = &ctx.gpu.as_ref().expect("the unit").waits_by { + let mut v: Vec<_> = by.iter().collect(); + v.sort_by(|a, b| b.1.cmp(a.1)); + for (who, n) in v { + eprintln!(" {n:8} readbacks from {who}"); + } + } + }); + } if every != 0 && recs.len() % every != 0 && rep.hashes.last().is_none_or(|(j, _)| *j != recs.len()) { rep.hashes.push((recs.len(), memory_hash(&b))); } (rep.dma_used, rep.dma_total) = mem.consumed(); rep.dma_mismatched = *mem.mismatches.lock(); + { + let w = mem.writes.lock(); + rep.host_writes = (w.0.clone().finalize().to_hex()[..16].to_string(), w.1); + } (rep, b) } @@ -280,8 +386,10 @@ pub fn replay_file(path: &Path, png: Option<&str>) -> Result { let recs = record::load(&path.to_string_lossy()).map_err(|e| format!("{}: {e}", path.display()))?; let t0 = std::time::Instant::now(); let (rep, board) = run(&recs, |_| {}, EVERY); + // The memory as drawn, on either raster unit. + board.gpu_sync(); eprintln!( - "{}: {} records in {:.1} s, DMA reads {}/{} ({} mismatched), {} bus loads differ, {} of {} marks reached{}", + "{}: {} records in {:.1} s, DMA reads {}/{} ({} mismatched), {} bus loads differ, host writes {} bytes {}, memory {}, {} of {} marks reached{}", path.display(), rep.records, t0.elapsed().as_secs_f64(), @@ -289,6 +397,9 @@ pub fn replay_file(path: &Path, png: Option<&str>) -> Result { rep.dma_total, rep.dma_mismatched, rep.reads_differ, + rep.host_writes.1, + rep.host_writes.0, + &memory_hash(&board)[..16], rep.marks - rep.marks_failed.len(), rep.marks, if rep.mid_session { " (started mid-session: the marks are informational)" } else { "" } @@ -309,7 +420,10 @@ pub fn replay_file(path: &Path, png: Option<&str>) -> Result { } // Read last, so a recording without checkpoints still gives its // picture and memory. - let want = load_hashes(path)?; + let mut want = load_hashes(path)?; + if let Some(limit) = std::env::var("VPRO_REPLAY_LIMIT").ok().and_then(|v| v.parse::().ok()) { + want.retain(|(i, _)| *i <= limit); + } let failed: Vec = want.iter().filter(|(i, h)| rep.hashes.iter().find(|(j, _)| j == i).is_none_or(|(_, g)| g != h)).map(|(i, _)| *i).collect(); if want.is_empty() || !failed.is_empty() || want.len() != rep.hashes.len() { return Err(format!("{}: {} of {} checkpoints failed, at records {:?}", path.display(), failed.len(), want.len(), &failed[..failed.len().min(8)])); @@ -347,6 +461,48 @@ fn recordings() -> Vec { } /// Every recording in the recordings directory replays to its checkpoints. +/// The two raster units agree on recorded guest sessions: each recording in +/// `VPRO_PARITY_DIR` (default `ignore/recordings/gpu-cases`) replays on the +/// CPU unit, on the GPU unit, and on the GPU unit ending a board slice every +/// 64 stores as a running machine does; all three must write the same bytes +/// to guest memory (the pixels the guest reads back) and leave the same +/// board memory. The recordings are short guest test programs (vissweep +/// cases, pixtest groups, cicopy); like the others they are not in git, and +/// the test skips without them. +#[cfg(feature = "wgpu")] +#[test] +fn the_raster_units_agree_on_recorded_sessions() { + let dir = std::env::var("VPRO_PARITY_DIR").map(PathBuf::from).unwrap_or_else(|_| PathBuf::from("ignore/recordings/gpu-cases")); + let mut files: Vec = std::fs::read_dir(&dir) + .map(|d| d.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.extension().is_some_and(|x| x == "rec")).collect()) + .unwrap_or_default(); + files.sort(); + if files.is_empty() { + eprintln!("SKIPPED: no recordings in {}", dir.display()); + return; + } + let mut bad = vec![]; + for f in &files { + let recs = record::load(&f.to_string_lossy()).expect("a recording"); + let outcome = |opts: Opts| { + let (rep, b) = run_with(&recs, |_| {}, 0, opts); + b.gpu_sync(); + (rep.host_writes, memory_hash(&b)) + }; + let cpu = outcome(Opts::default()); + let gpu = outcome(Opts { gpu: true, ..Default::default() }); + let ticked = outcome(Opts { gpu: true, tick: Some(64), ..Default::default() }); + eprintln!("{}: host writes {} bytes", f.display(), cpu.0 .1); + if gpu != cpu { + bad.push(format!("{}: the GPU unit differs ({:?} vs {:?})", f.display(), gpu, cpu)); + } + if ticked != cpu { + bad.push(format!("{}: the GPU unit with slice ends differs ({:?} vs {:?})", f.display(), ticked, cpu)); + } + } + assert!(bad.is_empty(), "{}", bad.join("\n")); +} + /// With no recordings the test says so and passes, since they are not in the /// tree; `VPRO_REPLAY_REQUIRE=1` (the script's `replay`) makes that a failure. #[test] @@ -446,6 +602,7 @@ fn a_replayed_session_reaches_the_planes_the_stores_do() { direct.mmio_write(*off as u64, *val, *size as u32); } } + direct.gpu_sync(); assert_eq!(memory_hash(&direct), rep.hashes.last().unwrap().1); let mut changed = session.clone(); @@ -511,6 +668,7 @@ fn marks_are_checked_against_the_replayed_planes() { let mut fresh = Board::new("v12", 11).unwrap(); fresh.reset(); window(&mut fresh); + fresh.gpu_sync(); let h0 = memory_hash_bytes(&fresh); let (rep, b) = run(&session, window, 0); assert_eq!(rep.hashes.len(), 0); @@ -520,6 +678,7 @@ fn marks_are_checked_against_the_replayed_planes() { fresh.mmio_write(*off as u64, *val, *size as u32); } } + fresh.gpu_sync(); let h1 = memory_hash_bytes(&fresh); let mut marked = vec![Rec::Mark(h0)]; diff --git a/src/dev/odyssey/golden/scenes.rs b/src/dev/odyssey/golden/scenes.rs index 0782306ee..69795abae 100644 --- a/src/dev/odyssey/golden/scenes.rs +++ b/src/dev/odyssey/golden/scenes.rs @@ -160,7 +160,7 @@ impl Ctx { op.clip.n = 1; op.clip.r[0] = ClipRect { x0: 0, y0: 0, x1: BENCH_W_MAX, y1: BENCH_H_MAX }; self.board.gl_raster(&op); - self.board.raster_wait(); + self.board.gpu_sync(); } /// Draw one primitive through the board's own GL draw. diff --git a/src/dev/odyssey/gpu_unit.rs b/src/dev/odyssey/gpu_unit.rs new file mode 100644 index 000000000..5e19e2096 --- /dev/null +++ b/src/dev/odyssey/gpu_unit.rs @@ -0,0 +1,162 @@ +//! The GPU raster unit as the board sees it (`wgpu/`, behind the `wgpu` +//! feature): the unit, a draw's slot on it, and the refusal a plane the +//! kernels cannot address gets. Without the feature the types are +//! uninhabited, so the board's dispatch compiles to the CPU path alone. + +/// Which view a blit stores into: one of its slot's, or one of its own +/// (a DMA put's surface). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "own-view blits are made only with the unit"))] +pub(crate) enum BlitView { + Slot(u32), + Own([u32; 4]), +} + +/// How a run of blit pixels stores: the view, the bits stored (`mask`, +/// all ones for a plain store), the window's tests, and a logical +/// operation (`op`, 0 none, else 0x100 | the GL code) whose result is +/// masked and or-ed (`and`, `or`) before the store, as Board::paint_integer +/// does. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct BlitKey { + pub view: BlitView, + pub mask: u32, + pub window: bool, + pub op: u32, + pub and: u32, + pub or: u32, +} + +impl BlitKey { + /// A plain store into slot view `view`. + pub fn store(view: u32, mask: u32) -> BlitKey { + BlitKey { view: BlitView::Slot(view), mask, window: false, op: 0, and: u32::MAX, or: 0 } + } +} + +#[cfg(feature = "wgpu")] +pub(super) use super::wgpu::scanout::VproScanout as GpuScanout; +#[cfg(feature = "wgpu")] +pub(super) use super::wgpu::{GpuRaster, Slot, TexSlot}; + +#[cfg(not(feature = "wgpu"))] +mod stub { + use super::super::pipe::backend::{ClearDraw, DestMem}; + use super::super::pipe::ring::BackQuad; + use super::super::pipe::setup::Tri; + use super::super::pipe::state::{BackendWords, RasterWords}; + + pub(crate) enum GpuRaster {} + + #[derive(Clone, Copy, Debug)] + pub(crate) enum Slot {} + + #[derive(Clone, Copy, Debug)] + pub(crate) enum TexSlot {} + + impl TexSlot { + pub fn new(_r: &super::super::pipe::texture::TexRegs, _l: &super::super::pipe::texture::Latch) -> TexSlot { + unreachable!("built without the wgpu feature") + } + } + + /// A plane the kernels cannot address (here: no kernels at all). + #[derive(Debug, Clone, PartialEq, Eq)] + pub(crate) struct Unsupported(pub String); + + impl GpuRaster { + pub fn read_back(&mut self) -> Result<(), String> { + match *self {} + } + + pub fn read_back_now(&mut self) -> Result<(), String> { + match *self {} + } + + pub fn wait(&mut self) -> Result<(), String> { + match *self {} + } + + #[allow(clippy::too_many_arguments, reason = "the record's sources")] + pub fn push_tri(&mut self, _t: &Tri, _ox: i32, _oy: i32, _bounds: (i32, i32, i32, i32), _steps: i64, _depth_range: [f64; 2], _depth_slack: f64, _mode: u32, _alpha: u32, _slot: &Slot, _tex: Option<&TexSlot>, _specular: bool) { + match *self {} + } + + pub fn push_frag(&mut self, _q: &BackQuad, _slot: &Slot) { + match *self {} + } + + pub fn push_rect(&mut self, _x0: i32, _y0: i32, _x1: i32, _y1: i32, _c: &ClearDraw, _slot: &Slot) { + match *self {} + } + + pub fn clear_all(&mut self) { + match *self {} + } + + pub fn blit_pixel(&mut self, _s: &Slot, _key: &super::BlitKey, _x: i32, _y: i32, _word: u32) { + match *self {} + } + + pub fn note_region(&mut self, _v: &super::super::view::View, _x: i32, _y: i32) { + match *self {} + } + + pub fn tick(&mut self, _idle: bool, _retired: u64) { + match *self {} + } + + + pub fn trace_waits(&mut self, _on: bool) { + match *self {} + } + + pub fn report(&self) -> String { + match *self {} + } + + pub fn read_back_view(&mut self, _v: &super::super::view::View, _x0: i32, _y0: i32, _x1: i32, _y1: i32) -> Result<(), String> { + match *self {} + } + + pub fn upload_region(&mut self, _v: &super::super::view::View, _x0: i32, _y0: i32, _x1: i32, _y1: i32) { + match *self {} + } + } + + #[allow(dead_code, reason = "the own-view blit paths are compiled with the wgpu feature only")] + impl Slot { + pub fn new(_w: &BackendWords, _planes: &DestMem, _index: bool, _to_aux: bool, _raster: &RasterWords, _stipple: Option<&[u32; 32]>, _background: Option<[f64; 4]>) -> Result { + Err(Unsupported("built without the wgpu feature".into())) + } + + pub fn clear(_planes: &DestMem, _raster: &RasterWords) -> Result { + Err(Unsupported("built without the wgpu feature".into())) + } + + pub fn none() -> Slot { + unreachable!("built without the wgpu feature") + } + + pub fn own_view(_rows: &super::super::view::Rows) -> Result<[u32; 4], Unsupported> { + Err(Unsupported("built without the wgpu feature".into())) + } + } +} + +#[cfg(not(feature = "wgpu"))] +mod stub_scanout { + /// The display thread's scan-out on the GPU. + pub(crate) enum GpuScanout {} + + impl GpuScanout { + pub fn run(&mut self, _frame: &super::super::frame::Frame, _regs: &super::super::reader::DisplayRegs, _width: u32, _height: u32, _out: &mut [u32]) -> Result<(), String> { + match *self {} + } + } +} + +#[cfg(not(feature = "wgpu"))] +pub(super) use stub::{GpuRaster, Slot, TexSlot}; +#[cfg(not(feature = "wgpu"))] +pub(super) use stub_scanout::GpuScanout; diff --git a/src/dev/odyssey/interface.rs b/src/dev/odyssey/interface.rs index cdeb850f9..49727c3a8 100644 --- a/src/dev/odyssey/interface.rs +++ b/src/dev/odyssey/interface.rs @@ -62,6 +62,12 @@ pub(super) struct HostInterface { pub route_dest_lower: u32, pub route_vectors: u32, pub swap_arm: u32, + /// The command stream's position (`CommandFifo::pushed`) when the + /// latest flip was armed, and the position at the store being written + /// (set by the bus path before `write`). A flip latches only once the + /// raster unit's submissions cover the stores before its arm. + pub swap_after: u64, + pub stream_pos: u64, pub swap_pending: u32, pub swap_control: u32, pub swap_control_companion: u32, @@ -101,6 +107,8 @@ impl Default for HostInterface { route_dest_lower: 0, route_vectors: 0, swap_arm: 0, + swap_after: 0, + stream_pos: 0, swap_pending: 0, swap_control: 0, swap_control_companion: 0, @@ -137,11 +145,16 @@ impl HostInterface { /// in the same frame's update; a frame is never presented half from each /// buffer. /// - /// Whether a swap latched. The latch needs this lock alone. - pub fn retrace(&mut self) -> bool { - let flips = self.swap_arm; - - if self.swap_set != 0 || self.swap_clear != 0 { + /// Whether a swap latched. The latch needs this lock alone. `covered` + /// is the stream position the raster unit's submitted work reaches + /// (`DisplayShared::covered`): a flip armed after stores not yet + /// submitted stays armed for a later retrace, so the buffer it shows + /// holds the drawing issued before it. + pub fn retrace(&mut self, covered: u64) -> bool { + let ready = self.swap_after <= covered; + let flips = if ready { self.swap_arm } else { 0 }; + + if ready && (self.swap_set != 0 || self.swap_clear != 0) { self.displayed_buffer = (self.displayed_buffer | self.swap_set) & !self.swap_clear; self.swap_set = 0; self.swap_clear = 0; @@ -396,6 +409,7 @@ impl HostInterface { // when a flip has not shown after two retraces. Whether the // board ORs or replaces is unknown; ORing loses no flip. self.swap_arm |= value as u32; + self.swap_after = self.stream_pos; crate::pf_instant!(crate::perfetto::events::ODSY_SWAP_ARM, value); } Some(swap::PENDING) => self.swap_pending = value as u32, @@ -406,11 +420,13 @@ impl HostInterface { self.swap_control = value as u32; self.swap_set |= value as u32; self.swap_clear &= !(value as u32); + self.swap_after = self.stream_pos; } Some(swap::CONTROL_COMPANION) => { self.swap_control_companion = value as u32; self.swap_clear |= value as u32; self.swap_set &= !(value as u32); + self.swap_after = self.stream_pos; } Some(ODSY_DISPLAYED_BUFFER) => { // Board-produced status; a guest write is not seen in the @@ -452,7 +468,11 @@ pub(super) fn front_mmio_read(f: &Mutex, addr: u64, size: u32) -> /// when it needs the board's state. pub(super) fn front_mmio_write(f: &Mutex, fifo: &CommandFifo, addr: u64, value: u64, size: u32) -> bool { if front_register(addr, size, true) { - f.lock().write(addr, value, size); + let mut g = f.lock(); + // The store falls after every command store the processor made + // before it: a swap arm waits for them (HostInterface::retrace). + g.stream_pos = fifo.pushed(); + g.write(addr, value, size); return true; } // A queued store needs the board's thread to run it, and the queue to diff --git a/src/dev/odyssey/mmio.rs b/src/dev/odyssey/mmio.rs index 61b7bc84f..5318dd30c 100644 --- a/src/dev/odyssey/mmio.rs +++ b/src/dev/odyssey/mmio.rs @@ -54,7 +54,10 @@ impl Board { // Registers that do not wait for the board's work live in the front // (interface.rs). if front_register(addr, size, true) { - self.front.lock().write(addr, value, size); + let mut f = self.front.lock(); + // Where in the command stream this store falls, for a swap arm. + f.stream_pos = self.fifo.pushed(); + f.write(addr, value, size); return; } @@ -556,7 +559,9 @@ impl BoardLocks { return Some(g); } self.raster.request_flush(); - if !self.fifo.pending() && self.raster.pending() == 0 { + // The board's thread ends a slice after its last store + // (`work_step`'s GPU tick): idle means the slice is over too. + if !self.fifo.pending() && self.raster.pending() == 0 && !self.fifo.slice.load(Ordering::SeqCst) { // SAFETY: as `bus_read`: the FIFO and the raster queue are // empty, so the board's thread is idle, and no processor // can push a store while `submit` is held. diff --git a/src/dev/odyssey/mod.rs b/src/dev/odyssey/mod.rs index b31adbdc2..b3dab555f 100644 --- a/src/dev/odyssey/mod.rs +++ b/src/dev/odyssey/mod.rs @@ -77,6 +77,7 @@ mod frame; mod golden; mod geometry; mod gl; +mod gpu_unit; mod assembly; mod interface; mod clip; @@ -95,6 +96,8 @@ mod scanout; mod trace; mod view; mod vram; +#[cfg(feature = "wgpu")] +mod wgpu; mod worker; #[cfg(test)] mod tests; @@ -132,7 +135,10 @@ pub struct Odyssey { impl Odyssey { /// A board of `product` (v12, v10, v8, v6) at `widget`: its interrupt - /// packets reach `irq` (HEART), its DMA reaches `mem`. + /// packets reach `irq` (HEART), its DMA reaches `mem`. `pool` is the + /// CPU raster unit's workers and tile size, `raster` which unit draws + /// and, for the GPU, the wgpu backend (`[graphics]`). + #[allow(clippy::too_many_arguments, reason = "the board's wiring")] pub fn new( widget: u8, product: &str, @@ -140,10 +146,17 @@ impl Odyssey { mem: Arc, status: Arc, pool: (usize, u32), + raster: (crate::config::RasterUnit, &str), ) -> Result, String> { let mut s = Board::new(product, widget as u32)?; // The fragment pipeline's workers and tile size (`[graphics]`). s.raster.set_pool(pool.0, pool.1); + if raster.0 == crate::config::RasterUnit::Wgpu { + #[cfg(feature = "wgpu")] + s.enable_gpu(&crate::wgpu::GpuOptions { backend: raster.1.to_string(), ..Default::default() })?; + #[cfg(not(feature = "wgpu"))] + return Err("graphics.raster = \"wgpu\" needs a build with the wgpu feature".into()); + } // The recorder's slot is there whether or not a recording runs, so // the monitor can start one (`vpro rec`); the DMA path records // through it. @@ -215,6 +228,25 @@ impl Odyssey { let mut reader = display.reader.lock(); reader.attach(&state.shared); reader.poll(monotonic_ns()); + // On the GPU raster unit with the wgpu renderer on the same device, + // the frame is composed and presented there with no readback. + #[cfg(feature = "wgpu")] + if display.gpu_frames_wanted() { + crate::pf_begin!(crate::perfetto::events::ODSY_FRAME_PLANES); + let composed = reader.update_on_gpu(); + crate::pf_end!(crate::perfetto::events::ODSY_FRAME_PLANES); + if let Some((buffer, w, h)) = composed { + set_display_fifo_pending(state.fifo.level() as usize + state.raster.pending() as usize); + crate::pf_end!(crate::perfetto::events::ODSY_CAPTURE); + drop(reader); + if display.due(true) { + crate::pf_begin!(crate::perfetto::events::ODSY_PRESENT); + display.present_gpu(&buffer, w, h); + crate::pf_end!(crate::perfetto::events::ODSY_PRESENT); + } + return; + } + } crate::pf_begin!(crate::perfetto::events::ODSY_FRAME_PLANES); let composed = reader.update(); crate::pf_end!(crate::perfetto::events::ODSY_FRAME_PLANES); @@ -270,7 +302,7 @@ impl Device for Odyssey { fn register_commands(&self) -> Vec<(String, String)> { vec![( "vpro".to_string(), - "VPro (Odyssey) graphics: status|regs|fifo|state|geom|tex|win|mem|backend|lut|dispmode|cursor|pix|raw|peek|shot|fbdump|hash|unimp|refusals|trace|rec (vpro help)".to_string(), + "VPro (Odyssey) graphics: status|regs|fifo|state|geom|tex|win|mem|backend|lut|dispmode|cursor|pix|raw|peek|shot|fbdump|hash|unimp|refusals|gpu|trace|rec (vpro help)".to_string(), )] } diff --git a/src/dev/odyssey/pipe/interp.rs b/src/dev/odyssey/pipe/interp.rs index bcad02a4c..446fea9e8 100644 --- a/src/dev/odyssey/pipe/interp.rs +++ b/src/dev/odyssey/pipe/interp.rs @@ -89,6 +89,13 @@ impl TriTex { TriTex { v: v.map(|a| a.map(|x| x as f32)) } } + /// The values at the vertices: S/W, T/W, R/W and Q/W, each at the + /// three vertices (the GPU record carries them). + #[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "used by the GPU raster unit"))] + pub fn values(&self) -> &[[f32; 3]; 4] { + &self.v + } + /// The texture coordinates of a quad whose pixels' barycentrics are l1 /// and l2. #[inline(always)] diff --git a/src/dev/odyssey/pipe/mod.rs b/src/dev/odyssey/pipe/mod.rs index 5ade22db3..444c39e3f 100644 --- a/src/dev/odyssey/pipe/mod.rs +++ b/src/dev/odyssey/pipe/mod.rs @@ -54,8 +54,20 @@ pub(crate) struct PipeDraw { pub shade: bool, /// Some quad of the draw went into the ring. pub used: std::cell::Cell, + /// The GPU raster unit, when the board draws on it and the draw's + /// planes are ones its kernels address, else null; the draw's slot on + /// it; and the raster enable and alpha-test words its triangle records + /// carry. + pub gpu: *mut super::gpu_unit::GpuRaster, + pub gpu_slot: Option, + pub gpu_words: (u32, u32), + /// The draw's texture on the unit, when it textures. + pub gpu_tex: Option>, } +/// A draw the GPU unit could not take was reported. +static GPU_REFUSED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + impl PipeDraw { /// Store a shaded quad (q.draw is set here): on the back-end thread, or /// now. @@ -64,6 +76,10 @@ impl PipeDraw { if q.cov == 0 { return; } + if let Some(g) = self.gpu_unit() { + g.push_frag(q, self.gpu_slot.as_ref().expect("a slot with the unit")); + return; + } match &self.ring { Some(r) => { q.draw = &self.back_draw; @@ -81,8 +97,51 @@ impl PipeDraw { /// (Board::gl_to_aux), whose destination is the window-id buffer. The /// transform side supplies the depth range and the depth's rounding /// slack. + /// The GPU unit, when this draw goes through it. + #[inline(always)] + #[allow(clippy::mut_from_ref, reason = "the unit is reached through the draw's pointer, not borrowed from the draw")] + pub fn gpu_unit(&self) -> Option<&mut super::gpu_unit::GpuRaster> { + if self.gpu.is_null() { + return None; + } + // SAFETY: the unit lives in the raster context the draw runs under, + // on the one thread that runs draws with a GPU unit (no raster + // thread starts with one), and nothing else reaches it while a + // draw runs (gl.rs run_frame takes it from the context). + Some(unsafe { &mut *self.gpu }) + } + #[allow(clippy::too_many_arguments, reason = "the stages' inputs")] - pub fn new(w: &FragmentWords, tex: texture::Unit<'static>, planes: DestMem, index: bool, to_aux: bool, depth_range: [f64; 2], slack: f64, ring: Option<&Arc>) -> Box { + pub fn new( + w: &FragmentWords, + tex: texture::Unit<'static>, + planes: DestMem, + index: bool, + to_aux: bool, + depth_range: [f64; 2], + slack: f64, + ring: Option<&Arc>, + stipple: Option<&[u32; 32]>, + background: Option<[f64; 4]>, + gpu: Option<&mut super::gpu_unit::GpuRaster>, + ) -> Box { + // The draw's slot on the GPU unit; a draw into planes the kernels + // cannot address takes the CPU path on a memory brought current. + let (gpu, gpu_slot): (*mut super::gpu_unit::GpuRaster, Option) = match gpu { + Some(g) => match super::gpu_unit::Slot::new(&w.backend, &planes, index, to_aux, &w.raster, stipple, background) { + Ok(s) => (g as *mut _, Some(s)), + Err(e) => { + if !GPU_REFUSED.swap(true, std::sync::atomic::Ordering::Relaxed) { + crate::dlog!(crate::devlog::LogModule::Odyssey, "sgi-odyssey: gpu unit: {} drawn on the CPU", e.0); + } + if let Err(e) = g.read_back() { + crate::dlog!(crate::devlog::LogModule::Odyssey, "sgi-odyssey: gpu readback: {e}"); + } + (std::ptr::null_mut(), None) + } + }, + None => (std::ptr::null_mut(), None), + }; let mut backend = BackendState::of(&w.backend, true, !index); if to_aux { backend.targets = aux_targets(backend.targets, index); @@ -100,7 +159,13 @@ impl PipeDraw { backend.depth_test && !backend.stencil, ); raster.depth32 = true; + if !gpu.is_null() { + // The unit stores behind this thread: no depth pre-test against + // the memory here. + raster.pretest = None; + } + let gpu_tex = (!gpu.is_null() && tex.regs.target() != texture::Target::None).then(|| Box::new(super::gpu_unit::TexSlot::new(tex.regs, tex.latch))); Box::new(PipeDraw { raster, @@ -115,6 +180,10 @@ impl PipeDraw { back_draw: BackendDraw { state: backend, planes }, shade: false, used: std::cell::Cell::new(false), + gpu, + gpu_slot, + gpu_words: (w.raster.mode, w.shader.alpha), + gpu_tex, }) } } diff --git a/src/dev/odyssey/pipe/raster.rs b/src/dev/odyssey/pipe/raster.rs index fea9dbc86..f4d0e5c4b 100644 --- a/src/dev/odyssey/pipe/raster.rs +++ b/src/dev/odyssey/pipe/raster.rs @@ -402,10 +402,11 @@ impl Triangle { } } -/// Walk a triangle into quads under st, handing each quad with any drawn -/// pixel to emit. `within`, where given, bounds the walk further, in -/// window pixels (the viewport the transform side keeps). -pub(crate) fn triangle_quads(st: &RasterState, t: &Triangle, within: Option<(i32, i32, i32, i32)>, mut emit: impl FnMut(&Quad)) { +/// The window pixels the walk of t under st covers, inclusive: the +/// triangle's bounds within `within` (window pixels), the scissor (window +/// coordinates), the clip rectangles and the plane (plane coordinates); +/// None when nothing is left. +pub(crate) fn walk_bounds(st: &RasterState, t: &Triangle, within: Option<(i32, i32, i32, i32)>) -> Option<(i32, i32, i32, i32)> { let (mut x0, mut y0, mut x1, mut y1) = t.bounds; if let Some((wx0, wy0, wx1, wy1)) = within { @@ -414,9 +415,6 @@ pub(crate) fn triangle_quads(st: &RasterState, t: &Triangle, within: Option<(i32 x1 = x1.min(wx1); y1 = y1.min(wy1); } - - // The walk is bounded by the scissor, in window coordinates, and by - // the clip rectangles and the plane, in plane coordinates. if let Some((sx0, sy0, sx1, sy1)) = st.scissor { x0 = x0.max(sx0); y0 = y0.max(sy0); @@ -425,7 +423,7 @@ pub(crate) fn triangle_quads(st: &RasterState, t: &Triangle, within: Option<(i32 } let n = st.clip_kept.min(4) as usize; if n == 0 { - return; + return None; } { let cx0 = st.clip_rect[..n].iter().map(|r| r.x0).min().unwrap(); @@ -443,8 +441,19 @@ pub(crate) fn triangle_quads(st: &RasterState, t: &Triangle, within: Option<(i32 x1 = x1.min((st.width as i32 - 1).wrapping_sub(st.ox)); y1 = y1.min((MAX_DIM as i32 - 1).wrapping_sub(st.oy)); if x0 > x1 || y0 > y1 { - return; + return None; } + Some((x0, y0, x1, y1)) +} + +/// Walk a triangle into quads under st, handing each quad with any drawn +/// pixel to emit. `within`, where given, bounds the walk further, in +/// window pixels (the viewport the transform side keeps). +pub(crate) fn triangle_quads(st: &RasterState, t: &Triangle, within: Option<(i32, i32, i32, i32)>, mut emit: impl FnMut(&Quad)) { + let Some((x0, y0, x1, y1)) = walk_bounds(st, t, within) else { + return; + }; + let area = t.area as f64; let steps = if st.offset_fill { st.offset_steps } else { 0 }; let dz1 = t.z[1] - t.z[0]; diff --git a/src/dev/odyssey/pipe/setup.rs b/src/dev/odyssey/pipe/setup.rs index 84b487a27..61a4d70df 100644 --- a/src/dev/odyssey/pipe/setup.rs +++ b/src/dev/odyssey/pipe/setup.rs @@ -411,7 +411,21 @@ impl Draw<'_> { }; let rst = super::raster::RasterState { offset_steps: p.fill_steps, offset_fill: true, ..p.raster }; - if let (Some(live), Some(ring)) = (self.live, p.ring.as_ref().filter(|_| p.shade)) { + if let Some(g) = p.gpu_unit() { + // The unit walks, textures and shades what its kernels cover; a + // fogged or user-clipped triangle is shaded here and its quads + // stored by the unit (emit()). + let tex = p.gpu_tex.as_deref().filter(|_| self.textured); + let simple = !p.shader.fog && self.xf.user_planes() == 0 && (tex.is_some() || !self.textured); + if simple { + if let Some((x0, y0, x1, y1)) = super::raster::walk_bounds(&rst, &ts.tri, Some((px0, py0, px1, py1))) { + let (ox, oy) = (rst.ox, rst.oy); + let bounds = (ox.wrapping_add(x0), oy.wrapping_add(y0), ox.wrapping_add(x1), oy.wrapping_add(y1)); + g.push_tri(&ts, ox, oy, bounds, rst.offset_steps, rst.depth_range, rst.depth_slack, p.gpu_words.0, p.gpu_words.1, p.gpu_slot.as_ref().expect("a slot with the unit"), tex, p.shader.specular); + } + return; + } + } else if let (Some(live), Some(ring)) = (self.live, p.ring.as_ref().filter(|_| p.shade)) { // The shader thread shades the quads in the ring, from the draw's // context and the triangle's record. live.capture(self); diff --git a/src/dev/odyssey/pipe/texture.rs b/src/dev/odyssey/pipe/texture.rs index 5f41a0c15..d4c614546 100644 --- a/src/dev/odyssey/pipe/texture.rs +++ b/src/dev/odyssey/pipe/texture.rs @@ -1118,7 +1118,10 @@ unsafe fn filter(win: *const [u32; 4], at: [usize; 3], w: [u32; 3], linear: bool } let [a, b, c] = w; let plane = |z: usize| -> [u32; 4] { - let (c00, c10, c01, c11) = (at_(0, 0, z), at_(1, 0, z), at_(0, 1, z), at_(1, 1, z)); + let (c00, c10) = (at_(0, 0, z), at_(1, 0, z)); + // A 1D texture's window has one row: its second row's weight is + // zero and the row is not read. + let (c01, c11) = if b != 0 { (at_(0, 1, z), at_(1, 1, z)) } else { (c00, c10) }; std::array::from_fn(|k| { let top = c00[k] * (256 - a) + c10[k] * a; let bot = c01[k] * (256 - a) + c11[k] * a; diff --git a/src/dev/odyssey/raster_bench.rs b/src/dev/odyssey/raster_bench.rs index 7e203c3b2..1b1c27d0c 100644 --- a/src/dev/odyssey/raster_bench.rs +++ b/src/dev/odyssey/raster_bench.rs @@ -554,13 +554,13 @@ impl Bench { d.polygon_smooth = self.polygon_smooth; self.board.lib_draw(&mut d); } - self.board.raster_wait(); + self.board.gpu_sync(); } /// Draw d on the board as the library would (Board::lib_draw()). pub fn draw(&mut self, d: &mut LibDraw) { self.board.lib_draw(d); - self.board.raster_wait(); + self.board.gpu_sync(); } /// The depth plane's words in the square from (0, 0) to side - 1, row diff --git a/src/dev/odyssey/raster_unit.rs b/src/dev/odyssey/raster_unit.rs index 90be0f90b..401a3be55 100644 --- a/src/dev/odyssey/raster_unit.rs +++ b/src/dev/odyssey/raster_unit.rs @@ -105,11 +105,16 @@ pub(super) struct RasterCtx { /// The quad ring to the shader and back-end threads, while the raster /// thread runs (pipe/ring.rs). pub ring: Option>, + /// The raster unit on the host GPU (gpu_unit.rs), when the board draws + /// there: then no raster thread runs, every job runs inside its + /// submit on the front end, and the drawing goes into the unit's + /// batches. + pub gpu: Option>, } impl RasterCtx { fn new() -> Self { - RasterCtx { line_phase: 0, ring: None } + RasterCtx { line_phase: 0, ring: None, gpu: None } } /// Wait for the back-end thread to store everything queued to it, so @@ -590,6 +595,11 @@ impl RasterQueue { self.closing.store(false, SeqCst); // SAFETY: no raster thread runs yet, so the context is this thread's. let ctx = unsafe { &mut *self.ctx.get() }; + if ctx.gpu.is_some() { + // The GPU unit draws inside each submit; the front end is the + // only thread that touches it. + return; + } if ctx.ring.is_none() && backend { // The shader's thread shades in the ring, so it comes with the // back end's. @@ -778,7 +788,56 @@ impl Board { /// Wait for the raster unit to finish its queued jobs. The front end /// calls it before it touches board memory itself, /// and before it writes a completion word that must follow the drawing. + /// With the GPU unit, that means its batches are drawn and what they + /// wrote is back in the memory the front end reads. pub fn raster_wait(&self) { self.raster.wait_idle(); + if !self.raster.threaded() { + // The unit's batches drawn; memory comes back where a reader + // asks for it (planes_in, dest_mem, gpu_sync). + self.raster.run_now(|ctx| { + if let Some(g) = &mut ctx.gpu { + if let Err(e) = g.wait() { + crate::dlog!(crate::devlog::LogModule::Odyssey, "sgi-odyssey: gpu wait: {e}"); + } + } + }); + } + } + + /// With the GPU unit, the end of a run of stores: the mirror's changes + /// go up, and the open batch is submitted when the board is going idle + /// or has held it long enough (`GpuRaster::tick`), so the scanout + /// composing from the GPU's memory shows what the guest drew; nothing + /// waits. + pub fn gpu_upload_tick(&self, idle: bool) { + if !self.raster.threaded() { + let retired = self.fifo.retired(); + self.raster.run_now(|ctx| { + if let Some(g) = &mut ctx.gpu { + g.tick(idle, retired); + } + }); + } + } + + /// `raster_wait`, and with the GPU unit the mirror brought current even + /// when the unit is set to skip readbacks (the replay's checkpoints). + pub fn gpu_sync(&self) { + self.raster_wait(); + if !self.raster.threaded() { + self.raster.run_now(|ctx| { + if let Some(g) = &mut ctx.gpu { + if let Err(e) = g.read_back_now() { + crate::dlog!(crate::devlog::LogModule::Odyssey, "sgi-odyssey: gpu readback: {e}"); + } + } + }); + } + } + + /// Whether the board draws on the GPU unit. + pub fn gpu_unit(&self) -> bool { + !self.raster.threaded() && self.raster.run_now(|ctx| ctx.gpu.is_some()) } } diff --git a/src/dev/odyssey/reader.rs b/src/dev/odyssey/reader.rs index 53aa51f20..8ad89a74f 100644 --- a/src/dev/odyssey/reader.rs +++ b/src/dev/odyssey/reader.rs @@ -343,6 +343,16 @@ pub(super) struct DisplayShared { mode: ModeCell, /// The board's memory, set when the board allocates it. pub vram: OnceLock, + /// The scan-out on the GPU, set with the GPU raster unit: the display + /// composes there from the memory the unit draws into, instead of + /// from the mirror. + pub gpu_scanout: OnceLock>, + /// The command stream position (`CommandFifo::retired`) the raster + /// unit's submitted work covers: a flip armed after it waits + /// (`HostInterface::retrace`). The GPU unit publishes it + /// (`GpuRaster::tick`); the CPU unit leaves it at the maximum, so flips + /// latch at once as before. + pub covered: Arc, pub front: Arc>, /// Swaps latched at a retrace. pub swaps: AtomicU64, @@ -354,6 +364,8 @@ impl DisplayShared { regs: SharedRegs::default(), mode: ModeCell::new(), vram: OnceLock::new(), + gpu_scanout: OnceLock::new(), + covered: Arc::new(AtomicU64::new(u64::MAX)), front, swaps: AtomicU64::new(0), } @@ -380,7 +392,7 @@ impl DisplayShared { /// A vertical retrace: the host interface latches the swaps armed and /// raises the retrace source (HostInterface::retrace()). pub fn retrace(&self) { - if self.front.lock().retrace() { + if self.front.lock().retrace(self.covered.load(Ordering::Acquire)) { self.swaps.fetch_add(1, Ordering::Relaxed); } } @@ -403,6 +415,8 @@ pub(super) struct ScanoutReader { /// The registers of the frame in progress. snap: DisplayRegs, snap_gen: u64, + /// The GPU scan-out failed once (reported), and the CPU composes. + gpu_failed: bool, } impl ScanoutReader { @@ -462,6 +476,36 @@ impl ScanoutReader { } } + /// As `update`, on the GPU scan-out with no readback: the frame stays + /// in the scan-out's buffer, returned with its size, for a renderer on + /// the same device. None when there is no GPU scan-out, no frame, or it + /// fails (the caller then takes the `update` path). + #[cfg(feature = "wgpu")] + pub fn update_on_gpu(&mut self) -> Option<(wgpu::Buffer, u32, u32)> { + let shared = self.shared(); + let scanout = shared.gpu_scanout.get()?; + if self.gpu_failed { + return None; + } + self.sync(&shared); + if self.frame.is_empty() { + return None; + } + let displayed = shared.front.lock().displayed_buffer; + self.snap_gen = shared.regs.snapshot(&mut self.snap, self.snap_gen); + let frame = Frame::snapshot(&self.snap, displayed)?; + let (w, h) = (frame.width.min(self.frame_width), frame.height.min(self.frame_height)); + let composed = scanout.lock().compose(&frame, &self.snap, w, h); + match composed { + Ok(buffer) => Some((buffer, w, h)), + Err(e) => { + self.gpu_failed = true; + crate::dlog!(crate::devlog::LogModule::Odyssey, "sgi-odyssey: gpu scanout: {e}; composing on the CPU"); + None + } + } + } + /// Compose the whole frame from board memory as the scan-out registers /// describe it (frame.rs); black while they describe none. False with /// no scanout mode, when there is no frame. @@ -480,6 +524,28 @@ impl ScanoutReader { self.frame.fill(0); return true; }; + if let Some(scanout) = shared.gpu_scanout.get() { + let (w, h) = (frame.width.min(self.frame_width), frame.height.min(self.frame_height)); + let stride = self.frame_width as usize; + // The kernel writes rows of w; a narrower frame is spread over + // the presented rows after. + match scanout.lock().run(&frame, &self.snap, w, h, &mut self.frame) { + Ok(()) => { + if w as usize != stride { + for y in (1..h as usize).rev() { + self.frame.copy_within(y * w as usize..(y + 1) * w as usize, y * stride); + } + } + return true; + } + Err(e) => { + if !self.gpu_failed { + self.gpu_failed = true; + crate::dlog!(crate::devlog::LogModule::Odyssey, "sgi-odyssey: gpu scanout: {e}; composing on the CPU"); + } + } + } + } // The memory lives as long as the board; the display reads it while // the board's units write it, as the scan-out does. frame.compose(*mem, &self.snap, &mut self.frame, self.frame_width as usize, self.frame_height as usize); diff --git a/src/dev/odyssey/testdata/README.md b/src/dev/odyssey/testdata/README.md index b7083668f..c6b3c1dc2 100644 --- a/src/dev/odyssey/testdata/README.md +++ b/src/dev/odyssey/testdata/README.md @@ -56,7 +56,7 @@ The checkpoints (`replay/.hashes`) are what is committed. `VPRO_REPLAY=,` sets the -pool's workers and tile size, default 2 and 32); `VPRO_REPLAY_LIMIT=` stops after n records; +pool's workers and tile size, default 2 and 32); `VPRO_REPLAY_GPU=1` draws on the GPU raster unit (a build with the `wgpu` feature; `VPRO_REPLAY_LAZY=1` skips the readbacks between checkpoints to time the GPU path alone); `VPRO_REPLAY_LIMIT=` stops after n records and checks the checkpoints reached; `VPRO_REPLAY_OUT=` saves the final colour buffer (unmapped: no colour map, window ids or cursor, so it is the planes, not the screen). `VPRO_REPLAY_SHOTS=` saves the composed screen at each mark as diff --git a/src/dev/odyssey/tests/catalogue.rs b/src/dev/odyssey/tests/catalogue.rs index 5fc2006d5..659702ab2 100644 --- a/src/dev/odyssey/tests/catalogue.rs +++ b/src/dev/odyssey/tests/catalogue.rs @@ -57,7 +57,7 @@ mod widget { fn board() -> (Arc, Arc) { let heart = Arc::new(Heart::new(HeartConfig::octane([128 << 20, 0, 0, 0]), host_clock())); let ram = Arc::new(HeartRam::try_new([128, 0, 0, 0]).expect("ram")); - let v = Odyssey::new(WIDGET, "v12", heart.clone(), ram.dma_view(), crate::disp::StatusSource::detached(), (2, 32)).expect("board"); + let v = Odyssey::new(WIDGET, "v12", heart.clone(), ram.dma_view(), crate::disp::StatusSource::detached(), (2, 32), (crate::config::RasterUnit::Cpu, "auto")).expect("board"); (heart, v) } diff --git a/src/dev/odyssey/tests/display.rs b/src/dev/odyssey/tests/display.rs index 6c7e1e082..afa925719 100644 --- a/src/dev/odyssey/tests/display.rs +++ b/src/dev/odyssey/tests/display.rs @@ -166,6 +166,42 @@ fn an_armed_swap_toggles_the_displayed_buffer_at_the_retrace() { assert_eq!(reg_read(&mut s, ODSY_DISPLAYED_BUFFER), 0x5); } +/// A flip armed after command stores the raster unit has not submitted +/// yet waits: the retrace leaves it armed until the unit's submissions +/// cover the stream up to the arm (`DisplayShared::covered`, published by +/// the GPU unit's tick), so the buffer shown holds the drawing issued +/// before the swap. The CPU unit leaves the position at its maximum and +/// flips latch at once, as the tests above see. +#[test] +fn a_flip_waits_until_the_raster_unit_covers_the_stores_before_its_arm() { + use std::sync::atomic::Ordering::SeqCst; + const ARM_BIT: u32 = 0x0100_0000; + let (mut s, _rec) = board("v12"); + route_to_heart(&mut s); + + // Two stores queued in the command stream, not yet run. + assert!(s.fifo.push(&s.front, cmd::FIFO, (swap::SYNC_A as u64) << 32 | 0x8000_2000, 8)); + assert!(s.fifo.push(&s.front, cmd::FIFO, (swap::SYNC_B as u64) << 32 | 0x8000_2000, 8)); + assert_eq!(s.fifo.pushed(), 2); + assert_eq!(s.fifo.retired(), 0); + + // The unit has submitted the drawing from the first store only. + s.display.covered.store(1, SeqCst); + reg_write(&mut s, swap::ARM, ARM_BIT); + retrace(&mut s); + assert_eq!(reg_read(&mut s, swap::ARM), ARM_BIT, "the flip stays armed"); + assert_eq!(reg_read(&mut s, ODSY_DISPLAYED_BUFFER), 0, "nothing flips before the stores before the arm are drawn"); + + s.display.covered.store(2, SeqCst); + retrace(&mut s); + assert_eq!(reg_read(&mut s, swap::ARM), 0); + assert_eq!(reg_read(&mut s, ODSY_DISPLAYED_BUFFER), ARM_BIT, "the flip latches once they are"); + + // The queued stores run; the stream positions meet. + s.work_drain(0); + assert_eq!(s.fifo.retired(), 2); +} + /// Arm writes between two retraces accumulate: separate windows issued /// before one retrace all flip there, and the interrupt handler's re-arm /// of a window still pending does not cancel the others. diff --git a/src/dev/odyssey/tests/dma.rs b/src/dev/odyssey/tests/dma.rs index a1c8e4c41..f5ccdb26c 100644 --- a/src/dev/odyssey/tests/dma.rs +++ b/src/dev/odyssey/tests/dma.rs @@ -62,7 +62,7 @@ fn object_header(rows: u32, irq: bool) -> [u32; 8] { /// Pixel (x, y) of the surface at `base` (`stride` tiles, 1 << `log2` /// bytes a pixel) as the model's word, its field packed as `codec`. fn mem_word(s: &Board, base: u32, stride: u32, log2: u8, codec: Codec, x: i32, y: i32) -> u32 { - s.raster_wait(); + s.gpu_sync(); let surface = Surface::new(base, stride, log2); Rows::new(s.vram_ptr(), View::colour(&surface, Field { at: 0, codec }, MAX_DIM)).load_word(x, y).unwrap() } diff --git a/src/dev/odyssey/tests/index.rs b/src/dev/odyssey/tests/index.rs index 744bcee68..29c6f0a67 100644 --- a/src/dev/odyssey/tests/index.rs +++ b/src/dev/odyssey/tests/index.rs @@ -64,7 +64,7 @@ fn index_triangle_then(s: &mut Board, st: &LibState, index: u32, words: impl FnO } fn index_at(s: &Board, buffer: usize, x: i32, y: i32) -> u32 { - s.raster_wait(); + s.gpu_sync(); s.dest_mem().colour[buffer].load_word(x, y).unwrap_or(0) } diff --git a/src/dev/odyssey/tests/mod.rs b/src/dev/odyssey/tests/mod.rs index 60293b5e0..8dcb14a6e 100644 --- a/src/dev/odyssey/tests/mod.rs +++ b/src/dev/odyssey/tests/mod.rs @@ -225,13 +225,13 @@ pub(super) fn screen_field(s: &Board, buffer: u32) -> crate::dev::odyssey::view: /// Byte `b` (0 the window id, 1 the overlay) of the window-id buffer /// Board::test_screen() sets up, at linear screen index `i` (y * 1280 + x). pub(super) fn aux_at(s: &Board, b: u32, i: u32) -> u8 { - s.raster_wait(); + s.gpu_sync(); let w = crate::dev::odyssey::vram::Surface::new(0x14_2000, 80, 1); s.vram.ptr_shared().load(w.offset(i % 1280, i / 1280) + b as u64, 1) as u8 } pub(super) fn set_aux(s: &mut Board, b: u32, i: u32, v: u8) { - s.raster_wait(); + s.gpu_sync(); let w = crate::dev::odyssey::vram::Surface::new(0x14_2000, 80, 1); let at = w.offset(i % 1280, i / 1280) + b as u64; s.vram.ptr().store(at, 1, v as u64); @@ -242,20 +242,27 @@ pub(super) fn screen_stencil(s: &Board, x: i32, y: i32) -> u32 { use crate::dev::odyssey::field::Layout; use crate::dev::odyssey::view::{Rows, View, MAX_DIM}; use crate::dev::odyssey::vram::{Format, Surface}; - s.raster_wait(); + s.gpu_sync(); let f = Format::decode(0x0040_6d50); let (_, st) = View::depth_stencil(&Surface::new(0x2000, 80, f.fb_log2), &Layout::of(f), MAX_DIM); Rows::new(s.vram_ptr(), st).load1(x, y).unwrap_or(0) as u32 } /// The pixel at (x, y) of colour buffer `buffer`, as 0xRRGGBBAA. +/// Whether the test boards run on the GPU raster unit (`VPRO_GPU=1` with +/// the `wgpu` feature): the raster thread's own mechanics are not there +/// to test then. +pub(super) fn gpu_unit_tests() -> bool { + cfg!(feature = "wgpu") && std::env::var("VPRO_GPU").is_ok() +} + pub(super) fn pixel(s: &Board, buffer: u32, x: i32, y: i32) -> u32 { - s.raster_wait(); + s.gpu_sync(); screen_field(s, buffer).load_word(x, y).unwrap_or(0) } /// Store a pixel into colour buffer 0, as a draw would. pub(super) fn pixel_write(s: &mut Board, x: i32, y: i32, value: u32) { - s.raster_wait(); + s.gpu_sync(); screen_field(s, 0).store_word(x, y, value); } diff --git a/src/dev/odyssey/tests/monitor.rs b/src/dev/odyssey/tests/monitor.rs index b8be0d23a..01c33271c 100644 --- a/src/dev/odyssey/tests/monitor.rs +++ b/src/dev/odyssey/tests/monitor.rs @@ -22,7 +22,7 @@ const DISPMODE_RGB: u32 = 0x0096_55a6; fn board() -> Arc { let heart = Arc::new(Heart::new(HeartConfig::octane([128 << 20, 0, 0, 0]), host_clock())); let ram = Arc::new(HeartRam::try_new([128, 0, 0, 0]).expect("ram")); - let v = Odyssey::new(WIDGET, "v12", heart, ram.dma_view(), crate::disp::StatusSource::detached(), (2, 32)).expect("board"); + let v = Odyssey::new(WIDGET, "v12", heart, ram.dma_view(), crate::disp::StatusSource::detached(), (2, 32), (crate::config::RasterUnit::Cpu, "auto")).expect("board"); v.state.test_board().test_screen(); v } diff --git a/src/dev/odyssey/tests/pixels.rs b/src/dev/odyssey/tests/pixels.rs index 4f5cf284f..ac2962966 100644 --- a/src/dev/odyssey/tests/pixels.rs +++ b/src/dev/odyssey/tests/pixels.rs @@ -532,7 +532,7 @@ fn a_write_code_without_a_pixel_form_is_refused() { /// A scratch surface at `base`: `pitch` tiles a row, 1 << `log2` bytes a /// pixel, its field at byte 0 packed as `codec`. fn scratch(s: &Board, base: u32, pitch: u32, log2: u8, codec: Codec) -> Rows { - s.raster_wait(); + s.gpu_sync(); Rows::new(s.vram_ptr(), View::colour(&Surface::new(base, pitch, log2), Field { at: 0, codec }, MAX_DIM)) } @@ -606,7 +606,7 @@ fn a_copy_from_a_10_10_10_2_window_into_a_texture_converts_the_colour() { // The texture the copy lands in, selected first as the library does. s.pixel_mode = [0x39_de80, 0x10, 0x0800_24d0, 0]; copy_block(&mut s, [0x3000_0019, y << 16 | x, 0x8010_0050, 1 << 16 | 1, 0x1263_3002, 0x39_de80, 0, 0xf801_210b]); - s.raster_wait(); + s.gpu_sync(); let tex = Surface::new(0x39_de80, 0x10, 2); assert_eq!(s.vram.ptr_shared().load(tex.offset(0, 0), 4), 0xc040_20ff); } @@ -933,7 +933,7 @@ fn bind_read_drawable(s: &mut Board, word: u32) { fn a_pixel_buffers_front_is_the_buffer_a_front_draw_mask_writes() { let (mut s, ram) = read_window(); colour_destination(&mut s); - s.raster_wait(); + s.gpu_sync(); screen_field(&s, 1).store_word(30, 40, 0x0102_0304); let front = read_word4(2, 8); let back = front & !dma::FMT_FRONT; @@ -1252,7 +1252,7 @@ fn a_row_that_ends_inside_a_word_drops_the_rest_of_the_word() { #[test] fn a_window_move_copies_every_byte_of_both_halves() { let (mut s, _, _) = board_with_ram(); - s.raster_wait(); + s.gpu_sync(); let screen = Surface::new(0x2000, 0x50, 4); s.pixel_mode = [0x2000, 0x50, 0x6950, 0]; let src = screen.offset(10, 10); @@ -1263,7 +1263,7 @@ fn a_window_move_copies_every_byte_of_both_halves() { for (w2, w4) in [(0x0050_0050u32, 0x0003_4000u32), (0x8050_4050, 0x8043_4000)] { copy_block(&mut s, [0x2000, sy << 16 | sx, w2, 1 << 16 | 1, w4, 0x2000, dy << 16 | dx, 0xc003_4000]); } - s.raster_wait(); + s.gpu_sync(); let dst = screen.offset(dx, dy); let got: Vec = (0..16).map(|k| s.vram.ptr_shared().load(dst + k, 1) as u8).collect(); assert_eq!(got, (0..16).map(|k| 0x40 + k as u8).collect::>()); diff --git a/src/dev/odyssey/tests/raster.rs b/src/dev/odyssey/tests/raster.rs index a890cceac..d8f695824 100644 --- a/src/dev/odyssey/tests/raster.rs +++ b/src/dev/odyssey/tests/raster.rs @@ -116,6 +116,10 @@ fn eventually(what: &str, mut done: impl FnMut() -> bool) { /// does not wait answers at once. #[test] fn a_waiting_read_behind_a_queued_draw_sees_it_drawn() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, st) = slow_raster(300); red_triangle(&mut h.test_board(), &st); @@ -136,6 +140,10 @@ fn a_waiting_read_behind_a_queued_draw_sees_it_drawn() { /// (`a_read_of_a_register_the_work_may_change_finishes_the_work_first`). #[test] fn the_memory_word_answers_while_a_draw_is_queued() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, st) = slow_raster(300); red_triangle(&mut h.test_board(), &st); @@ -156,6 +164,10 @@ fn the_memory_word_answers_while_a_draw_is_queued() { /// written in the moment the test looks. #[test] fn a_fence_behind_a_queued_draw_lands_after_the_draw() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, st) = slow_raster(0); h.raster.set_budget(u64::MAX); let hold = Arc::new(AtomicBool::new(true)); @@ -277,7 +289,7 @@ fn the_stipple_phase_carries_across_draws_on_the_raster_thread() { phase = d.line_phase; } let s = h.test_board(); - s.raster_wait(); + s.gpu_sync(); (0..64).map(|x| pixel(&s, 0, x, 20)).collect() } @@ -304,12 +316,12 @@ fn a_reused_draw_job_draws_only_its_new_primitive() { let quad = vertices(&[(0.0, 0.0), (64.0, 0.0), (0.0, 64.0), (64.0, 0.0), (64.0, 64.0), (0.0, 64.0)], RED); let mut d = LibDraw::new(GL_TRIANGLES, 6, &quad, st, &stipple); h.test_board().lib_draw(&mut d); - h.test_board().raster_wait(); + h.test_board().gpu_sync(); let tri = vertices(&[(w - 64.0, ht - 64.0), (w, ht - 64.0), (w - 64.0, ht)], BLUE); let mut d = LibDraw::new(GL_TRIANGLES, 3, &tri, st, &stipple); h.test_board().lib_draw(&mut d); let s = h.test_board(); - s.raster_wait(); + s.gpu_sync(); let mut px = Vec::new(); for y in (0..ODSY_HEIGHT as i32).step_by(8) { for x in (0..ODSY_WIDTH as i32).step_by(8) { @@ -336,6 +348,10 @@ fn a_reused_draw_job_draws_only_its_new_primitive() { /// draw is in the planes, its snapshot already in place. #[test] fn a_query_behind_a_queued_draw_syncs_after_the_draw_without_holding_the_front_end() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, st) = slow_raster(0); let release = hold_raster(&h); let ram = Ram::new(1 << 20); @@ -365,6 +381,10 @@ fn a_query_behind_a_queued_draw_syncs_after_the_draw_without_holding_the_front_e /// read; the memory word answers at once. #[test] fn a_waiting_bus_read_is_busy_until_the_raster_unit_has_drawn() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, st) = slow_raster(300); h.work_start(); @@ -566,6 +586,10 @@ fn tiny_triangle(s: &mut Board, st: &LibState) { /// sum of theirs. #[test] fn many_small_jobs_queue_without_the_front_end_waiting() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, st) = slow_raster(0); let release = hold_raster(&h); let mut s = h.test_board(); @@ -588,6 +612,10 @@ fn many_small_jobs_queue_without_the_front_end_waiting() { /// until the raster unit has drawn it. #[test] fn one_big_job_makes_the_front_end_wait() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, st) = slow_raster(0); let release = hold_raster(&h); // The held raster unit runs nothing, so the first job returning shows it @@ -612,6 +640,10 @@ fn one_big_job_makes_the_front_end_wait() { /// runs, in order. #[test] fn the_bound_releases_as_the_raster_unit_works() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, _st) = slow_raster(200); let ran = std::sync::Arc::new(Mutex::new(Vec::new())); let job = |i: u32| { @@ -638,6 +670,10 @@ fn the_bound_releases_as_the_raster_unit_works() { /// raster unit's slice; the stop still finishes every job, in order. #[test] fn stop_releases_a_waiting_front_end() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, _st) = slow_raster(1500); let ran = std::sync::Arc::new(Mutex::new(Vec::new())); let job = |i: u32| { @@ -811,6 +847,10 @@ fn stress(seed: u64, last_is_lut: bool) { #[test] fn the_frame_ends_as_a_whole_resolve_under_live_writers() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } for (seed, lut) in [(0x5a17, false), (0x77, true), (0xbeef, false), (0x31, true), (0x42, false), (0x99, true)] { stress(seed, lut); } @@ -913,6 +953,10 @@ fn swap_atomicity() { #[test] fn a_swap_after_a_fence_shows_the_whole_new_frame_with_no_lock() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } swap_atomicity(); } @@ -923,6 +967,10 @@ fn a_swap_after_a_fence_shows_the_whole_new_frame_with_no_lock() { /// that waited for the slice would never end. #[test] fn a_frame_waits_for_no_slice_and_takes_no_lock() { + // The raster thread's own behaviour: nothing to test on the GPU unit. + if super::gpu_unit_tests() { + return; + } let (h, st) = slow_raster(0); let display = Arc::new(Display::new(crate::disp::StatusSource::detached())); let hold = Arc::new(AtomicBool::new(true)); diff --git a/src/dev/odyssey/tests/stop.rs b/src/dev/odyssey/tests/stop.rs index 24deb384b..c6e996aec 100644 --- a/src/dev/odyssey/tests/stop.rs +++ b/src/dev/odyssey/tests/stop.rs @@ -50,7 +50,7 @@ impl Renderer for MainBoundRenderer { fn board() -> Arc { let heart = Arc::new(Heart::new(HeartConfig::octane([128 << 20, 0, 0, 0]), host_clock())); let ram = Arc::new(HeartRam::try_new([128, 0, 0, 0]).expect("ram")); - Odyssey::new(11, "v12", heart, ram.dma_view(), crate::disp::StatusSource::detached(), (2, 32)).expect("board") + Odyssey::new(11, "v12", heart, ram.dma_view(), crate::disp::StatusSource::detached(), (2, 32), (crate::config::RasterUnit::Cpu, "auto")).expect("board") } /// Queue the self-test preamble twice behind the board's thread. diff --git a/src/dev/odyssey/view.rs b/src/dev/odyssey/view.rs index f738ac6fe..e54b52bc7 100644 --- a/src/dev/odyssey/view.rs +++ b/src/dev/odyssey/view.rs @@ -92,6 +92,14 @@ impl View { self.base.wrapping_add((tile << 8 | (y & 15) << 4 | (x & 15)) << self.log2).wrapping_add(self.at as u64) } + /// The view's placement: the surface's first byte, its pitch in tiles, + /// log2 of its pixel size and the field's byte inside the pixel (for + /// the GPU instantiation's slots). + #[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "used by the GPU raster unit"))] + pub fn parts(&self) -> (u64, u32, u8, u8) { + (self.base, self.pitch, self.log2, self.at) + } + /// The view `by` bytes further into the pixel. pub fn shifted(mut self, by: i64) -> View { self.at = (self.at as i64 + by) as u8; diff --git a/src/dev/odyssey/vram.rs b/src/dev/odyssey/vram.rs index 3a990edfc..7268fd395 100644 --- a/src/dev/odyssey/vram.rs +++ b/src/dev/odyssey/vram.rs @@ -19,6 +19,9 @@ //! format word's decode: `ignore/odsy/docs/measured.md` sections 4 and 5, //! `clanker/plans/odyssey-board-memory.md`. +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + /// Bytes in one unit of a base address. pub(super) const UNIT: u64 = 16; /// Pixels on a side of a tile. @@ -119,17 +122,83 @@ impl Surface { } +/// Bytes in one block of the dirty map, log2. +pub(super) const DIRTY_BLOCK_SHIFT: u32 = 14; + +/// Which blocks of the memory the CPU has stored to since the map was last +/// taken: kept when the memory is the CPU's mirror of the GPU's +/// (`wgpu::GpuRaster`), so an upload moves only what changed. Marked by +/// `VramPtr::store` from any thread. +#[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "used by the GPU raster unit"))] +pub(super) struct DirtyBlocks { + bits: Box<[AtomicU64]>, + blocks: u64, +} + +#[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "used by the GPU raster unit"))] +impl DirtyBlocks { + fn new(size: u64) -> DirtyBlocks { + let blocks = (size >> DIRTY_BLOCK_SHIFT).max(1); + DirtyBlocks { bits: (0..blocks.div_ceil(64)).map(|_| AtomicU64::new(0)).collect(), blocks } + } + + /// Mark the block holding word index w. + #[inline(always)] + fn mark_word(&self, w: usize) { + let b = w >> (DIRTY_BLOCK_SHIFT - 3); + self.bits[b >> 6].fetch_or(1 << (b & 63), Ordering::Relaxed); + } + + pub fn clear(&self) { + for w in self.bits.iter() { + w.store(0, Ordering::Relaxed); + } + } + + /// The marked blocks as runs of bytes (offset, length), unmarked. + pub fn take_runs(&self) -> Vec<(u64, u64)> { + let mut runs: Vec<(u64, u64)> = Vec::new(); + let block = 1u64 << DIRTY_BLOCK_SHIFT; + for (i, w) in self.bits.iter().enumerate() { + let mut bits = w.swap(0, Ordering::Relaxed); + while bits != 0 { + let b = (i as u64) * 64 + bits.trailing_zeros() as u64; + bits &= bits - 1; + if b >= self.blocks { + break; + } + let off = b * block; + match runs.last_mut() { + Some((o, l)) if *o + *l == off => *l += block, + _ => runs.push((off, block)), + } + } + } + runs + } +} + /// The board's memory. #[derive(Clone, Default)] pub(super) struct Vram { words: Vec, + /// The dirty map, once the memory mirrors the GPU's. + dirty: Option>, } impl Vram { /// Memory of at least size bytes, a power of two, so an address masks /// into it. pub fn new(size: u64) -> Vram { - Vram { words: vec![0; (size.next_power_of_two() / 8).max(1) as usize] } + Vram { words: vec![0; (size.next_power_of_two() / 8).max(1) as usize], dirty: None } + } + + /// Start keeping the dirty map (the GPU instantiation's mirror). + #[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "used by the GPU raster unit"))] + pub fn track_dirty(&mut self) { + if self.dirty.is_none() { + self.dirty = Some(Arc::new(DirtyBlocks::new(self.size()))); + } } /// Its size in bytes. @@ -141,13 +210,17 @@ impl Vram { /// sized once, at allocation, and never moves, so the pointer stays /// good for the board's life. pub fn ptr(&mut self) -> VramPtr { - VramPtr { ptr: self.words.as_mut_ptr(), mask: self.words.len() - 1 } + VramPtr { ptr: self.words.as_mut_ptr(), mask: self.words.len() - 1, dirty: self.dirty_ptr() } } /// ptr() from a shared borrow: the memory is written through the pointer /// by units that race as the board's do (see VramPtr). pub fn ptr_shared(&self) -> VramPtr { - VramPtr { ptr: self.words.as_ptr() as *mut u64, mask: self.words.len() - 1 } + VramPtr { ptr: self.words.as_ptr() as *mut u64, mask: self.words.len() - 1, dirty: self.dirty_ptr() } + } + + fn dirty_ptr(&self) -> *const DirtyBlocks { + self.dirty.as_ref().map_or(std::ptr::null(), |d| Arc::as_ptr(d)) } pub fn words_mut(&mut self) -> &mut [u64] { @@ -165,6 +238,8 @@ pub(super) struct VramPtr { ptr: *mut u64, /// Words less one: a word index masks into the memory. mask: usize, + /// The dirty map, or null when none is kept (Vram::track_dirty). + dirty: *const DirtyBlocks, } // SAFETY: the array outlives every user (Board owns it and stops the raster @@ -204,9 +279,56 @@ impl VramPtr { let m = ((1u64 << (8 * n)) - 1) << shift; *p = (*p & !m) | ((v << shift) & m); } + if !self.dirty.is_null() { + (*self.dirty).mark_word((a >> 3) as usize & self.mask); + } } } + /// The same memory with no dirty map: for a writer that uploads what + /// it wrote itself (`GpuRaster::upload_region`). + pub fn untracked(&self) -> VramPtr { + VramPtr { ptr: self.ptr, mask: self.mask, dirty: std::ptr::null() } + } + + /// The dirty map's marked blocks as byte runs, unmarked; none without a + /// map. + #[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "used by the GPU raster unit"))] + pub fn take_dirty_runs(&self) -> Vec<(u64, u64)> { + if self.dirty.is_null() { + return Vec::new(); + } + // SAFETY: the map lives as long as the memory (Vram::dirty). + unsafe { (*self.dirty).take_runs() } + } + + #[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "used by the GPU raster unit"))] + pub fn clear_dirty(&self) { + if !self.dirty.is_null() { + // SAFETY: as take_dirty_runs(). + unsafe { (*self.dirty).clear() } + } + } + + /// The words at byte offset `off` (8-aligned, in range, not wrapping), + /// `len` bytes of them, for an upload. + #[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "used by the GPU raster unit"))] + pub fn words_at(&self, off: u64, len: u64) -> &[u64] { + debug_assert!(off % 8 == 0 && len % 8 == 0 && off + len <= self.size()); + // SAFETY: in range; readers and writers race as the board's units do. + unsafe { std::slice::from_raw_parts(self.ptr.add((off >> 3) as usize), (len >> 3) as usize) } + } + + /// Store `words` at byte offset `off` (8-aligned, in range) without + /// marking the dirty map: a readback from the GPU, which the GPU + /// already holds. + #[cfg_attr(not(feature = "wgpu"), allow(dead_code, reason = "used by the GPU raster unit"))] + pub fn write_words_untracked(&self, off: u64, words: &[u64]) { + debug_assert!(off % 8 == 0 && off + 8 * words.len() as u64 <= self.size()); + // SAFETY: in range; as above. + unsafe { std::ptr::copy_nonoverlapping(words.as_ptr(), self.ptr.add((off >> 3) as usize), words.len()) } + } + /// The memory's size in bytes. pub fn size(&self) -> u64 { (self.mask as u64 + 1) * 8 @@ -226,7 +348,7 @@ impl VramPtr { /// No memory: a view of a field the surface lacks, which nothing reads. pub fn null() -> VramPtr { - VramPtr { ptr: std::ptr::null_mut(), mask: 0 } + VramPtr { ptr: std::ptr::null_mut(), mask: 0, dirty: std::ptr::null() } } } diff --git a/src/dev/odyssey/wgpu/mod.rs b/src/dev/odyssey/wgpu/mod.rs new file mode 100644 index 000000000..b302a9e12 --- /dev/null +++ b/src/dev/odyssey/wgpu/mod.rs @@ -0,0 +1,1303 @@ +//! The VPro raster unit on the host GPU (`clanker/plans/wgpu-raster.md`, +//! the VPro section). +//! +//! The quad ring's three commands (pipe/ring.rs: a triangle's record and +//! box, a quad the raster thread shaded, a clear's rectangle) become +//! records of a `crate::wgpu::Batch`, and the back-end state a draw's +//! slots name becomes a batch slot: the back-end words as the command FIFO +//! delivered them, the destination's views, the window's clip list and id +//! test, and the polygon stipple. Board memory is a `GpuMemory`; the +//! kernels in `shaders/` store into it with the model's arithmetic +//! (pipe/raster.rs, interp.rs, fixed.rs, backend.rs), and every CPU read of +//! it is a readback after a flush. +//! +//! This is the raster unit alone: `GpuRaster` records, flushes, waits and +//! reads back. Wiring it into the board in place of `RasterQueue` and the +//! front end's own memory accesses is the next milestone. + +// The board does not use the unit yet: it is wired in, in place of +// RasterQueue, in the next milestone; until then only the tests drive it. +#![allow(dead_code, reason = "wired into the board in the next milestone")] + +pub(crate) mod scanout; +#[cfg(test)] +mod tests; + +use std::sync::Arc; + +use bytemuck::{Pod, Zeroable}; + +use crate::wgpu::{library, ArchShaders, Batch, BatchPipelines, Fragment, Full, Gpu, GpuMemory, TileGrid}; + +use super::consts::write; +use super::field::Codec; +use super::gpu_unit::{BlitKey, BlitView}; +use super::pipe::backend::{ClearDraw, DestMem}; +use super::pipe::ring::BackQuad; +use super::pipe::setup::Tri; +use super::pipe::state::{BackendWords, RasterWords}; +use super::pipe::texture; +use super::view::{Kind, Rows, MAX_DIM}; +use super::vram::{VramPtr, DIRTY_BLOCK_SHIFT}; + +/// The plane the batch grid covers, per axis, in pixels: records beyond +/// it are dropped and counted. +pub(crate) const PLANE_SPAN: u32 = 2048; + +pub(crate) const VRAM: Fragment = Fragment::new("vram.wgsl", include_str!("shaders/vram.wgsl")); +pub(crate) const VIEWS: Fragment = Fragment::new("views.wgsl", include_str!("shaders/views.wgsl")); +const BACKEND: Fragment = Fragment::new("backend.wgsl", include_str!("shaders/backend.wgsl")); +const TEXTURE: Fragment = Fragment::new("texture.wgsl", include_str!("shaders/texture.wgsl")); +const VPRO: Fragment = Fragment::new("vpro.wgsl", include_str!("shaders/vpro.wgsl")); + +/// `fixed::log2_q8`'s table: log2(1 + i/256) in 1/256, for the kernels' +/// level of detail (a small buffer of their own). +fn log2_table() -> [u32; 256] { + std::array::from_fn(|i| ((1.0 + i as f64 / 256.0).log2() * 256.0).round() as u16 as u32) +} + +/// Record kinds (`shaders/vpro.wgsl`). +const REC_TRI: u32 = 1; +const REC_FRAG: u32 = 2; +const REC_RECT: u32 = 3; +const REC_BLIT: u32 = 4; + +const TRI_SMOOTH: u32 = 1; +const TRI_TEXTURED: u32 = 2; +const TRI_SPECULAR: u32 = 4; + +/// Payload words of a record. +const REC_WORDS: usize = 84; + +// Triangle word offsets. +const T_E0: usize = 0; +const T_DX: usize = 6; +const T_DY: usize = 12; +const T_NEED: usize = 18; +const T_INV_AREA: usize = 19; +const T_Z: usize = 20; +const T_D: usize = 23; +const T_IW: usize = 26; +const T_SLACK: usize = 29; +const T_RGBA12: usize = 32; +const T_FLAT12: usize = 44; +const T_STEPS: usize = 48; +const T_BOX: usize = 49; +const T_ORIGIN: usize = 53; +const T_ALPHA: usize = 55; +const T_MODE: usize = 56; +/// S/W, T/W, R/W, Q/W at the three vertices (TriTex::values), 12 f32. +const T_TEX: usize = 57; +const T_TEXSLOT: usize = 69; +/// The separate specular colours at the vertices (3 x 3) and the +/// provoking one (3), in 12.12. +const T_SPEC12: usize = 70; +const T_FLAT_SPEC: usize = 79; + +// Fragment word offsets. +const F_X: usize = 0; +const F_Y: usize = 1; +const F_COV: usize = 2; +const F_ZQ: usize = 3; +const F_RGBA: usize = 7; + +// Rectangle word offsets and flags. +const R_BOX: usize = 0; +const R_COLOUR: usize = 4; +const R_COLOUR_MASK: usize = 5; +const R_DEPTH: usize = 6; +const R_STENCIL: usize = 7; +const R_ID: usize = 8; +// Blit word offsets and flags. +const B_BOX: usize = 0; +const B_OFF: usize = 4; +const B_MASK: usize = 5; +const B_VIEW: usize = 6; +const B_OP: usize = 7; +const B_AND: usize = 8; +const B_OR: usize = 9; +const B_OWN: usize = 10; +const BLIT_WINDOW: u32 = 1; +const BLIT_OWN_VIEW: u32 = 2; + +/// Words the stage buffer holds (the pixels of a batch's blits). +const STAGE_WORDS: usize = 1 << 20; + +/// How long a batch stays open with the board busy before `tick` submits it. +const MAX_OPEN: std::time::Duration = std::time::Duration::from_millis(4); + +const CLEAR_COLOUR: u32 = 1; +const CLEAR_DEPTH: u32 = 2; +const CLEAR_STENCIL: u32 = 4; +const CLEAR_TARGET0: u32 = 8; + +// Slot flags. +const SLOT_HAS_STENCIL: u32 = 1; +const SLOT_RGBA: u32 = 2; +const SLOT_TO_AUX: u32 = 4; +const SLOT_STIPPLE: u32 = 8; +const SLOT_BACKGROUND: u32 = 16; + +// View kinds. +const KIND_NONE: u32 = 0; +const KIND_RGBA8: u32 = 1; +const KIND_Z24: u32 = 2; +const KIND_BYTE: u32 = 3; +/// A texel field: the byte count in bits 15:8, FIELD_FMT in 23:16. +const KIND_TEXEL: u32 = 4; +const KIND_LANES: u32 = 5; + +/// A setup record, as `shaders/vpro.wgsl` reads it. +#[repr(C)] +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +pub(crate) struct Rec { + kind: u32, + slot: u32, + flags: u32, + one: u32, + w: [u32; REC_WORDS], +} + +const _: () = assert!(size_of::() == 352); + +impl Rec { + fn new(kind: u32, flags: u32) -> Rec { + Rec { kind, slot: 0, flags, one: 1.0f32.to_bits(), w: [0; REC_WORDS] } + } + + fn set_i(&mut self, i: usize, v: i32) { + self.w[i] = v as u32; + } + + fn set_f(&mut self, i: usize, v: f32) { + self.w[i] = v.to_bits(); + } + + fn set_i64(&mut self, i: usize, v: i64) { + self.w[i] = v as u32; + self.w[i + 1] = (v >> 32) as u32; + } + + /// A filled triangle: its setup record `t` from a draw whose window + /// origin is (ox, oy), walked within plane box `bounds` (inclusive; + /// `walk_bounds` plus the origin), with `steps` of polygon offset, + /// under depth range and slack `depth`, with the alpha test's words. + #[allow(clippy::too_many_arguments, reason = "the record's sources")] + pub fn tri(t: &Tri, ox: i32, oy: i32, bounds: (i32, i32, i32, i32), steps: i64, depth_range: [f64; 2], depth_slack: f64, mode: u32, alpha: u32, textured: bool, specular: bool) -> Rec { + let mut flags = if t.smooth { TRI_SMOOTH } else { 0 }; + if textured { + flags |= TRI_TEXTURED; + } + if specular { + flags |= TRI_SPECULAR; + } + let mut r = Rec::new(REC_TRI, flags); + for (a, axis) in t.tex.values().iter().enumerate() { + for (v, value) in axis.iter().enumerate() { + r.set_f(T_TEX + 3 * a + v, *value); + } + } + for v in 0..3 { + for k in 0..3 { + r.set_i(T_SPEC12 + 3 * v + k, t.spec12[v][k]); + } + r.set_i(T_FLAT_SPEC + v, t.flat_spec12[v]); + } + let tri = &t.tri; + for k in 0..3 { + // The edge values at the centre of plane pixel (0, 0): window + // pixel (-ox, -oy). + let e0 = tri.e00[k].wrapping_sub(tri.dx[k].wrapping_mul(ox as i64)).wrapping_sub(tri.dy[k].wrapping_mul(oy as i64)); + r.set_i64(T_E0 + 2 * k, e0); + r.set_i64(T_DX + 2 * k, tri.dx[k]); + r.set_i64(T_DY + 2 * k, tri.dy[k]); + } + r.w[T_NEED] = (0..3).fold(0, |m, k| m | ((tri.need[k] != 0) as u32) << k); + r.set_f(T_INV_AREA, tri.inv_area); + // The planes as depth4_lanes() takes them. + r.set_f(T_Z, tri.z[0] as f32); + r.set_f(T_Z + 1, (tri.z[1] - tri.z[0]) as f32); + r.set_f(T_Z + 2, (tri.z[2] - tri.z[0]) as f32); + r.set_f(T_D, tri.d[0] as f32); + r.set_f(T_D + 1, (tri.d[1] - tri.d[0]) as f32); + r.set_f(T_D + 2, (tri.d[2] - tri.d[0]) as f32); + for k in 0..3 { + r.set_f(T_IW + k, t.inv_w[k] as f32); + } + let (n, f) = (depth_range[0] as f32, depth_range[1] as f32); + r.set_f(T_SLACK, depth_slack as f32); + r.set_f(T_SLACK + 1, n); + r.set_f(T_SLACK + 2, f - n); + for v in 0..3 { + for k in 0..4 { + r.set_i(T_RGBA12 + 4 * v + k, t.rgba12[v][k]); + } + } + for k in 0..4 { + r.set_i(T_FLAT12 + k, t.flat12[k]); + } + r.set_i(T_STEPS, steps.clamp(-(1 << 25), 1 << 25) as i32); + r.set_i(T_BOX, bounds.0); + r.set_i(T_BOX + 1, bounds.1); + r.set_i(T_BOX + 2, bounds.2); + r.set_i(T_BOX + 3, bounds.3); + r.set_i(T_ORIGIN, ox); + r.set_i(T_ORIGIN + 1, oy); + r.w[T_ALPHA] = alpha; + r.w[T_MODE] = mode; + r + } + + /// A quad shaded on the CPU (`QuadRing::push_shaded`). + pub fn frag(q: &BackQuad) -> Rec { + let mut r = Rec::new(REC_FRAG, 0); + r.set_i(F_X, q.x); + r.set_i(F_Y, q.y); + r.w[F_COV] = q.cov as u32; + for i in 0..4 { + r.w[F_ZQ + i] = q.zq[i]; + let c = q.rgba[i]; + r.w[F_RGBA + 2 * i] = c[0] as u32 | (c[1] as u32) << 16; + r.w[F_RGBA + 2 * i + 1] = c[2] as u32 | (c[3] as u32) << 16; + } + r + } + + /// A clear's rectangle, (x0, y0)-(x1, y1) inclusive in plane pixels + /// (`QuadRing::push_rect`). + pub fn rect(x0: i32, y0: i32, x1: i32, y1: i32, c: &ClearDraw) -> Rec { + let mut flags = 0; + let mut r = Rec::new(REC_RECT, 0); + r.set_i(R_BOX, x0); + r.set_i(R_BOX + 1, y0); + r.set_i(R_BOX + 2, x1); + r.set_i(R_BOX + 3, y1); + if let Some(colour) = c.colour { + flags |= CLEAR_COLOUR; + r.w[R_COLOUR] = colour; + } + r.w[R_COLOUR_MASK] = c.colour_mask; + if let Some(d) = c.depth { + flags |= CLEAR_DEPTH; + r.w[R_DEPTH] = d; + } + if let Some((v, m)) = c.stencil { + flags |= CLEAR_STENCIL; + r.w[R_STENCIL] = v as u32 | (m as u32) << 8; + } + r.w[R_ID] = c.id_value as u32 | (c.id_mask as u32) << 8; + for b in 0..2 { + if c.targets[b] { + flags |= CLEAR_TARGET0 << b; + } + } + r.flags = flags; + r + } + + /// A row run of pixels at (x0, y) to (x1, y) from stage word `off`, + /// stored as `key` says. + fn blit(x0: i32, y: i32, x1: i32, off: u32, key: &BlitKey) -> Rec { + let mut flags = if key.window { BLIT_WINDOW } else { 0 }; + let mut r = Rec::new(REC_BLIT, 0); + r.set_i(B_BOX, x0); + r.set_i(B_BOX + 1, y); + r.set_i(B_BOX + 2, x1); + r.set_i(B_BOX + 3, y); + r.w[B_OFF] = off; + r.w[B_MASK] = key.mask; + r.w[B_OP] = key.op; + r.w[B_AND] = key.and; + r.w[B_OR] = key.or; + match key.view { + BlitView::Slot(v) => r.w[B_VIEW] = v, + BlitView::Own(v) => { + flags |= BLIT_OWN_VIEW; + r.w[B_OWN..B_OWN + 4].copy_from_slice(&v); + } + } + r.flags = flags; + r + } + + /// The box a record of this kind touches, from its own words. + fn pixel_box(&self) -> (i32, i32, i32, i32) { + match self.kind { + REC_TRI => (self.w[T_BOX] as i32, self.w[T_BOX + 1] as i32, self.w[T_BOX + 2] as i32, self.w[T_BOX + 3] as i32), + REC_FRAG => { + let (x, y) = (self.w[F_X] as i32, self.w[F_Y] as i32); + (x, y, x + 1, y + 1) + } + // The clear's and the blit's boxes sit at the same words. + _ => (self.w[R_BOX] as i32, self.w[R_BOX + 1] as i32, self.w[R_BOX + 2] as i32, self.w[R_BOX + 3] as i32), + } + } +} + +/// A state slot, as `shaders/backend.wgsl` reads it: words 0 to 15 the +/// back-end words and flags, then the six views, the clip rectangles, the +/// stipple and the opaque stipple's background. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Pod, Zeroable)] +pub(crate) struct Slot { + words: [u32; 16], + views: [[u32; 4]; 6], + clip: [[i32; 4]; 4], + pattern: [u32; 32], + bg12: [i32; 4], +} + +const _: () = assert!(size_of::() == 368); + +/// A plane the kernels cannot address yet. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Unsupported(pub String); + +/// A coded field's kind word (`views.wgsl`). +pub(crate) fn codec_kind(c: Codec) -> Result { + Ok(match c { + Codec::Byte => KIND_BYTE, + Codec::Lanes => KIND_LANES, + Codec::Texel { bytes, field } => { + if !matches!(bytes, 1 | 2 | 4) { + return Err(Unsupported(format!("a {bytes}-byte texel field"))); + } + KIND_TEXEL | (bytes as u32) << 8 | (field as u32) << 16 + } + }) +} + +fn pack_view(r: &Rows) -> Result<[u32; 4], Unsupported> { + let v = &r.v; + let kind = match v.kind { + Kind::None => KIND_NONE, + Kind::Rgba8 => KIND_RGBA8, + Kind::Z24 => KIND_Z24, + Kind::Byte => KIND_BYTE, + Kind::Coded(c) => codec_kind(c)?, + }; + let (base, pitch, log2, at) = v.parts(); + if base >= 1 << 32 || pitch >= 1 << 16 || v.width >= 1 << 16 || v.height > MAX_DIM { + return Err(Unsupported(format!("a view at {base:#x} of pitch {pitch}, {} x {}", v.width, v.height))); + } + Ok([base as u32, pitch | (log2 as u32) << 16 | (at as u32) << 24, kind, v.width | v.height << 16]) +} + +impl Slot { + /// The slot of a draw under back-end words `w` into `planes`, in index + /// mode or not, for an overlay context or not; `raster` for the window's + /// clip list and id test; the polygon stipple's rows where it is on, + /// and an opaque stipple's background colour. + pub fn new(w: &BackendWords, planes: &DestMem, index: bool, to_aux: bool, raster: &RasterWords, stipple: Option<&[u32; 32]>, background: Option<[f64; 4]>) -> Result { + let mut flags = 0; + if planes.stencil.v.kind != Kind::None { + flags |= SLOT_HAS_STENCIL; + } + if !index { + flags |= SLOT_RGBA; + } + if to_aux { + flags |= SLOT_TO_AUX; + } + let mut s = Slot::zeroed(); + s.words[..9].copy_from_slice(&[w.mode, w.depth_func, w.write_mask, w.stencil, w.blend[0], w.blend[1], w.blend[2], w.logic_op, w.draw_mask]); + s.words[10] = raster.clip_kept; + s.words[11] = raster.id_compare; + for (i, r) in [&planes.colour[0], &planes.colour[1], &planes.depth, &planes.stencil, &planes.did, &planes.overlay].iter().enumerate() { + s.views[i] = pack_view(r)?; + } + for (i, c) in raster.clip_rect.iter().enumerate().take(4) { + s.clip[i] = [c.x0, c.y0, c.x1, c.y1]; + } + if let Some(rows) = stipple { + flags |= SLOT_STIPPLE; + s.pattern = *rows; + if let Some(bg) = background { + flags |= SLOT_BACKGROUND; + s.bg12 = bg.map(super::pipe::fixed::from_f64); + } + } + s.words[9] = flags; + Ok(s) + } + + /// The slot of a clear: the planes and the window's id test, no + /// back-end words. + pub fn clear(planes: &DestMem, raster: &RasterWords) -> Result { + Slot::new(&BackendWords { write_mask: write::INDEX_MASK, ..Default::default() }, planes, false, false, raster, None, None) + } + + /// No state at all: for a record that carries its own view and tests + /// nothing (a DMA put's blits). + pub fn none() -> Slot { + Slot::zeroed() + } + + /// A view the slot holds, for a blit of its own view into a surface + /// the kernels address (`BlitView::Own`). + pub fn own_view(rows: &Rows) -> Result<[u32; 4], Unsupported> { + pack_view(rows) + } +} + +impl Slot { + /// The byte span view `i` may be written in over plane box `b` + /// (inclusive): the tiles from the box's first to its last, which the + /// tile-major layout keeps contiguous. None for a view the slot lacks. + fn view_span(&self, i: usize, b: (i32, i32, i32, i32)) -> Option<(u64, u64)> { + let v = self.views[i]; + if v[2] == KIND_NONE { + return None; + } + let (width, height) = ((v[3] & 0xffff) as i32, (v[3] >> 16) as i32); + let (x0, y0, x1, y1) = (b.0.max(0), b.1.max(0), b.2.min(width - 1), b.3.min(height - 1)); + if x0 > x1 || y0 > y1 { + return None; + } + let (base, pitch, log2) = (v[0] as u64, (v[1] & 0xffff) as u64, (v[1] >> 16) & 0xff); + let tile = |x: i32, y: i32| (y as u64 >> 4) * pitch + (x as u64 >> 4); + let lo = base + ((tile(x0, y0) << 8) << log2); + let hi = base + (((tile(x1, y1) + 1) << 8) << log2); + Some((lo, hi - lo)) + } +} + +/// A draw's texture on the GPU (`shaders/texture.wgsl` `TexSlot`): the +/// registers the unit reads at sample time and what the CPU latches from +/// them once (`texture::Latch`), in the kernel's layout. +#[repr(C)] +#[derive(Clone, Copy, PartialEq, Eq, Pod, Zeroable)] +pub(crate) struct TexSlot { + head: [u32; 44], + levels: [[u32; 12]; 32], + detail: [u32; 12], + colour_table: [u32; 512], + detail_fn: [u32; 256], +} + +const _: () = assert!(size_of::() == 4 * (44 + 384 + 12 + 512 + 256)); + +const TEX_LOD: u32 = 1; +const TEX_POST: u32 = 2; +const TEX_STORED_BORDER: u32 = 4; +const TEX_TABLE: u32 = 8; +const TEX_TABLE_ALPHA: u32 = 16; +const NO_CLAMP: i32 = 0x7fff_ffff; + +fn pack_level(l: &texture::Level) -> [u32; 12] { + let bytes = 1u32 << l.bpp; + [l.addr as u32, l.pitch, l.bpp as u32, l.field as u32, bytes, l.ox, l.oy, l.wl, l.hl, l.dl, l.slice as u32, 0] +} + +impl TexSlot { + pub fn new(r: &texture::TexRegs, l: &texture::Latch) -> TexSlot { + let mut t = TexSlot::zeroed(); + let mut flags = 0; + if l.lod { + flags |= TEX_LOD; + } + if l.post { + flags |= TEX_POST; + } + if r.stored_border() { + flags |= TEX_STORED_BORDER; + } + let (table, alpha) = r.colour_table(); + if table { + flags |= TEX_TABLE; + } + if alpha { + flags |= TEX_TABLE_ALPHA; + } + let h = &mut t.head; + h[0] = r.sampler[0]; + h[1] = r.sampler[4]; + h[2] = r.target().axes() as u32; + h[3] = flags; + h[4] = l.count; + h[5] = l.min_lod as u32; + h[6] = l.max_lod as u32; + h[7] = l.mag_below as u32; + for a in 0..3 { + h[8 + a] = l.lod_scale[a].to_bits(); + h[12 + a] = l.scale[a].to_bits(); + h[20 + a] = if l.clamp[a] == i64::MAX { NO_CLAMP as u32 } else { l.clamp[a] as i32 as u32 }; + } + h[16] = l.detail_scale[0].to_bits(); + h[17] = l.detail_scale[1].to_bits(); + h[23] = NO_CLAMP as u32; + h[24..28].copy_from_slice(&l.border); + for k in 0..8 { + h[28 + k] = super::bits::sextract32(r.scale_bias[k], 0, super::consts::tex::SCALE_BIAS_BITS) as u32; + } + h[36] = match r.env() { + texture::Env::Replace => 0, + texture::Env::Modulate => 1, + texture::Env::Decal => 2, + texture::Env::Blend => 3, + texture::Env::Add => 4, + }; + let ec = r.env_colour(); + for k in 0..4 { + h[37 + k] = super::pipe::fixed::field12_full(ec[k]) as u32; + } + for i in 0..texture::LEVELS { + t.levels[i] = pack_level(&l.level[i]); + } + t.detail = pack_level(&l.detail); + t.colour_table[..256].copy_from_slice(&r.colour_table[0]); + t.colour_table[256..].copy_from_slice(&r.colour_table[1]); + t.detail_fn = r.detail; + t + } + + fn hash(&self) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for w in bytemuck::cast_slice::(std::slice::from_ref(self)) { + h = (h ^ *w as u64).wrapping_mul(0x0100_0000_01b3); + } + h + } +} + +/// Texture slots a batch may hold. +const MAX_TEX_SLOTS: u32 = 256; + +/// A run of consecutive pixels of one row being collected into one blit. +struct BlitRun { + slot: Slot, + key: BlitKey, + y: i32, + x0: i32, + next_x: i32, + off: u32, +} + +/// The raster unit: the batch recorder over the board's memory on the +/// GPU. The front end records in command order and flushes when it would +/// observe memory; `wait` is `Board::raster_wait`'s equivalent. +pub(crate) struct GpuRaster { + gpu: Arc, + pipelines: BatchPipelines, + batch: Batch, + vram: GpuMemory, + group: wgpu::BindGroup, + /// The batch's texture slots, their buffer, and their indices by hash. + tex_slots: Vec, + tex_buf: wgpu::Buffer, + tex_map: std::collections::HashMap, + /// The batch's blit pixels and their buffer, and the run of + /// consecutive pixels being collected (`blit_pixel`): its slot, view, + /// mask and window flag, its row, first and next x, and where its + /// pixels start in the stage. + stage: Vec, + stage_buf: wgpu::Buffer, + run: Option, + /// A rectangle of a view the front end is storing into the mirror + /// untracked (a scratch image through the FIFO), to upload when it is + /// done: the view and its inclusive pixel box. + region: Option<(super::view::View, i32, i32, i32, i32)>, + /// The slot last issued in this batch and its index. + last: Option<(Slot, u32)>, + pending: Option, + /// When the open batch took its first record or blit pixel; None while + /// it is empty. `tick` submits a batch that has been open too long. + opened: Option, + /// The command stream position the board's thread had retired at the + /// last `tick`, and where it is published once everything recorded up + /// to it is submitted (`DisplayShared::covered`). + stream_now: u64, + stream: Option>, + /// The CPU's mirror of the memory (`Vram` with its dirty map): what + /// the front end's own paths read and write. Its dirty blocks go up + /// before each batch; the blocks the batches wrote come back on + /// `read_back`. + mirror: VramPtr, + /// Blocks (`DIRTY_BLOCK_SHIFT`) the batches since the last readback may + /// have written, from the records' boxes and their slots' views. + gpu_dirty: Vec, + gpu_dirty_any: bool, + /// Records whose box missed the grid. + pub dropped: u64, + /// Bytes moved each way so far; the uploads split into the mirror's + /// dirty blocks and the regions writers named. + pub uploaded: u64, + pub uploaded_blocks: u64, + pub uploaded_tiles: u64, + pub read_back_bytes: u64, + pub readbacks: u64, + /// Waits for a submission (`wait`), and batches `tick` submitted + /// because the board went idle or the batch had been open too long. + pub waits: u64, + pub flushes_idle: u64, + pub flushes_timed: u64, + /// Measurement only: `read_back` flushes and returns without waiting + /// or reading, so the front end's paths see a stale mirror and the + /// GPU path's own time shows; `read_back_now` still does the work. + pub lazy: bool, + /// `VPRO_GPU_TRACE_WAITS=1`: readbacks counted by the board function + /// that caused them (the first `odyssey::` frame outside this module + /// and the raster unit's plumbing). + pub waits_by: Option>, +} + + +/// Slots a batch may hold. +const MAX_SLOTS: u32 = 1024; + +impl GpuRaster { + /// A raster unit over `vram_bytes` of board memory (a power of two) + /// whose plane coordinates span `width` x `height` pixels. + pub fn new(gpu: &Arc, vram_bytes: u64, width: u32, height: u32) -> Result { + if !vram_bytes.is_power_of_two() || vram_bytes < 8 { + return Err(format!("vram size {vram_bytes:#x} is not a power of two")); + } + let vram = GpuMemory::new(gpu, "vpro vram", vram_bytes)?; + let entry = |binding, read_only| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Storage { read_only }, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }; + let layout = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("vpro vram"), + entries: &[entry(0, false), entry(1, true), entry(2, true), entry(3, true)], + }); + let log2_buf = gpu.buffer("vpro log2 table", 256 * 4, wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST); + gpu.queue.write_buffer(&log2_buf, 0, bytemuck::cast_slice(&log2_table())); + let tex_buf = gpu.buffer("vpro texture slots", MAX_TEX_SLOTS as u64 * size_of::() as u64, wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST); + let stage_buf = gpu.buffer("vpro blit stage", STAGE_WORDS as u64 * 4, wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST); + let words = vram_bytes / 8; + let pipelines = BatchPipelines::new( + gpu, + &ArchShaders { + label: "vpro", + raster: &[VPRO], + bin_accept: None, + library: &[library::I64, library::FLOAT, VRAM, VIEWS, BACKEND, TEXTURE], + defines: &[("VRAM_WORD_MASK", (words - 1) as u32), ("MAX_DIM", MAX_DIM)], + layouts: &[&layout], + }, + )?; + let group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("vpro vram"), + layout: &layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: vram.buffer.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: tex_buf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: stage_buf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: log2_buf.as_entire_binding() }, + ], + }); + let batch = Batch::new(gpu, &pipelines, TileGrid::new(width, height), MAX_SLOTS); + let blocks = (vram_bytes >> DIRTY_BLOCK_SHIFT).max(1) as usize; + Ok(GpuRaster { + gpu: Arc::clone(gpu), + pipelines, + batch, + vram, + group, + tex_slots: Vec::new(), + tex_buf, + tex_map: std::collections::HashMap::new(), + stage: Vec::new(), + stage_buf, + run: None, + region: None, + last: None, + pending: None, + opened: None, + stream_now: 0, + stream: None, + mirror: VramPtr::null(), + gpu_dirty: vec![0; blocks.div_ceil(64)], + gpu_dirty_any: false, + dropped: 0, + uploaded: 0, + uploaded_blocks: 0, + uploaded_tiles: 0, + read_back_bytes: 0, + readbacks: 0, + waits: 0, + flushes_idle: 0, + flushes_timed: 0, + lazy: false, + waits_by: std::env::var("VPRO_GPU_TRACE_WAITS").is_ok().then(std::collections::BTreeMap::new), + }) + } + + /// Attribute a readback to the board function that asked for it. + fn note_wait(&mut self) { + let Some(by) = &mut self.waits_by else { return }; + let bt = std::backtrace::Backtrace::force_capture().to_string(); + let mut who = String::from("?"); + for line in bt.lines() { + let Some(i) = line.find("odyssey::") else { continue }; + let name = line[i..].trim_end(); + if name.contains("::wgpu::") || name.contains("raster_unit::") || name.contains("::raster_wait") || name.contains("::gpu_sync") || name.contains("::dest_mem") || name.contains("::run_now") { + continue; + } + who = name.split(" at ").next().unwrap_or(name).to_string(); + break; + } + *by.entry(who).or_insert(0) += 1; + } + + + /// Mirror the memory in `m` (a `Vram` with its dirty map on): upload + /// all of it now, and from here on its dirty blocks before each batch. + pub fn attach_mirror(&mut self, m: VramPtr) { + assert_eq!(m.size(), self.vram.size(), "the mirror is the memory's size"); + self.mirror = m; + self.vram.write(0, bytemuck::cast_slice(m.words_at(0, m.size()))); + self.uploaded += m.size(); + m.clear_dirty(); + } + + /// Note that the batch may write `s`'s views over plane box `b`. + fn mark_gpu(&mut self, s: &Slot, b: (i32, i32, i32, i32)) { + let size = self.vram.size(); + for i in 0..6 { + let Some((lo, len)) = s.view_span(i, b) else { continue }; + if lo + len > size { + // Past the end (an address that wraps): everything. + for w in self.gpu_dirty.iter_mut() { + *w = u64::MAX; + } + self.gpu_dirty_any = true; + return; + } + let (b0, b1) = ((lo >> DIRTY_BLOCK_SHIFT) as usize, ((lo + len - 1) >> DIRTY_BLOCK_SHIFT) as usize); + for blk in b0..=b1 { + self.gpu_dirty[blk >> 6] |= 1 << (blk & 63); + } + self.gpu_dirty_any = true; + } + } + + /// The marked blocks as byte runs, unmarked. + fn take_gpu_dirty(&mut self) -> Vec<(u64, u64)> { + let mut runs: Vec<(u64, u64)> = Vec::new(); + let block = 1u64 << DIRTY_BLOCK_SHIFT; + let blocks = self.vram.size() >> DIRTY_BLOCK_SHIFT; + for i in 0..self.gpu_dirty.len() { + let mut bits = std::mem::take(&mut self.gpu_dirty[i]); + while bits != 0 { + let b = (i as u64) * 64 + bits.trailing_zeros() as u64; + bits &= bits - 1; + if b >= blocks { + break; + } + let off = b * block; + match runs.last_mut() { + Some((o, l)) if *o + *l == off => *l += block, + _ => runs.push((off, block)), + } + } + } + self.gpu_dirty_any = false; + runs + } + + /// Upload the mirror's bytes of view `v` over its pixel box (inclusive) + /// now, tile row by tile row, ahead of what is submitted next: for a + /// writer that stored into memory no pending drawing touches (a + /// texture or scratch upload) through an untracked pointer, so the + /// upload is the pixels it wrote and not their whole 16 KB blocks, + /// which may hold GPU-written bytes the mirror has not read back. + pub fn upload_region(&mut self, v: &super::view::View, x0: i32, y0: i32, x1: i32, y1: i32) { + // An upload lands ahead of the batch: the pixels recorded so far + // must be in a batch of their own first. + self.flush(); + self.upload_tiles(v, x0, y0, x1, y1); + } + + /// `upload_region`'s copies, with nothing submitted first. + fn upload_tiles(&mut self, v: &super::view::View, x0: i32, y0: i32, x1: i32, y1: i32) { + let (x0, y0, x1, y1) = (x0.max(0), y0.max(0), x1.min(v.width as i32 - 1), y1.min(v.height as i32 - 1)); + if v.kind == Kind::None || x0 > x1 || y0 > y1 { + return; + } + let (base, pitch, log2, _) = v.parts(); + let (pitch, log2) = (pitch as u64, log2 as u32); + let size = self.vram.size(); + let (tx0, tx1) = ((x0 >> 4) as u64, (x1 >> 4) as u64); + for ty in (y0 >> 4) as u64..=(y1 >> 4) as u64 { + let lo = base + (((ty * pitch + tx0) << 8) << log2); + let hi = base + (((ty * pitch + tx1 + 1) << 8) << log2); + if hi > size { + // Wrapping addresses: let the block map carry it. + self.mirror.take_dirty_runs(); + continue; + } + self.vram.write(lo, bytemuck::cast_slice(self.mirror.words_at(lo, hi - lo))); + self.uploaded += hi - lo; + self.uploaded_tiles += hi - lo; + } + } + + /// The end of a run of the board's stores. The mirror's dirty blocks go + /// up so the scanout, which composes from the GPU's memory, sees what + /// the CPU paths drew; and the open batch is submitted when the board + /// is going idle (`idle`: no store waits) or when it has been open for + /// `MAX_OPEN`, so what the guest drew reaches the screen about when it + /// would on the board. A busy guest (a GL game) fills batches faster + /// than that and is not slowed. + pub fn tick(&mut self, idle: bool, retired: u64) { + self.stream_now = retired; + let open = self.opened.is_some_and(|t| idle || t.elapsed() >= MAX_OPEN); + if open { + if idle { + self.flushes_idle += 1; + } else { + self.flushes_timed += 1; + } + self.flush(); + } else { + self.upload_dirty(); + if self.opened.is_none() { + // Nothing recorded since the last submission: it covers + // everything retired. + self.publish_covered(); + } + } + } + + /// Publish the stream position the submitted work covers through `at` + /// (`DisplayShared::covered`), from now on. + pub fn publish_stream(&mut self, at: std::sync::Arc) { + at.store(self.stream_now, std::sync::atomic::Ordering::Release); + self.stream = Some(at); + } + + fn publish_covered(&self) { + if let Some(s) = &self.stream { + s.store(self.stream_now, std::sync::atomic::Ordering::Release); + } + } + + /// Count readbacks by the board function that causes them (`waits_by`) + /// from now on, or stop and drop the counts. + pub fn trace_waits(&mut self, on: bool) { + self.waits_by = on.then(std::collections::BTreeMap::new); + } + + /// The unit's counters as text for the monitor (`vpro status`, `vpro + /// gpu`): batches and records, how the batches were submitted, bytes + /// each way, waits and readbacks, and the readbacks by cause when + /// traced. + pub fn report(&self) -> String { + let (flushes, records) = self.counts(); + let mut out = format!( + " gpu unit: {records} records in {flushes} batches ({} on idle, {} timed, {} dropped), {} open now; {} MiB up ({} MiB blocks, {} MiB regions), {} waits, {} readbacks ({} MiB)\n", + self.flushes_idle, + self.flushes_timed, + self.dropped, + self.batch.len(), + self.uploaded >> 20, + self.uploaded_blocks >> 20, + self.uploaded_tiles >> 20, + self.waits, + self.readbacks, + self.read_back_bytes >> 20, + ); + if let Some(by) = &self.waits_by { + let mut v: Vec<_> = by.iter().collect(); + v.sort_by(|a, b| b.1.cmp(a.1)); + for (who, n) in v { + out.push_str(&format!(" {n:>8} readbacks from {who}\n")); + } + } + out + } + + /// Upload the mirror's dirty blocks, ahead of what is submitted next. + fn upload_dirty(&mut self) { + for (off, len) in self.mirror.take_dirty_runs() { + self.vram.write(off, bytemuck::cast_slice(self.mirror.words_at(off, len))); + self.uploaded += len; + self.uploaded_blocks += len; + } + } + + /// Submit and wait, then bring every block the batches may have written + /// back into the mirror: `Board::raster_wait` for the GPU unit. Cheap + /// when nothing was recorded since the last readback. + pub fn read_back(&mut self) -> Result<(), String> { + if self.lazy { + self.flush(); + return self.gpu.check(); + } + self.read_back_now() + } + + /// `read_back` whether lazy or not. + pub fn read_back_now(&mut self) -> Result<(), String> { + self.flush(); + if !self.gpu_dirty_any { + return self.gpu.check(); + } + self.note_wait(); + let runs = self.take_gpu_dirty(); + let bytes = self.vram.read_ranges(&runs)?; + self.pending = None; + let words: &[u64] = bytemuck::cast_slice(&bytes); + let mut at = 0usize; + for (off, len) in runs { + let n = (len >> 3) as usize; + self.mirror.write_words_untracked(off, &words[at..at + n]); + at += n; + } + self.read_back_bytes += bytes.len() as u64; + self.readbacks += 1; + self.gpu.check() + } + + /// Bring the mirror current over view `v`'s pixel box (inclusive): the + /// blocks of its tile rows the batches may have written, and only + /// those. Nothing waits when none of them is marked, which is the + /// common case for a front end reading pixels one at a time. + pub fn read_back_view(&mut self, v: &super::view::View, x0: i32, y0: i32, x1: i32, y1: i32) -> Result<(), String> { + // A blit run still being collected has marked nothing yet: close + // it, so its blocks count. + self.end_pending(); + let (x0, y0, x1, y1) = (x0.max(0), y0.max(0), x1.min(v.width as i32 - 1), y1.min(v.height as i32 - 1)); + if v.kind == Kind::None || x0 > x1 || y0 > y1 || !self.gpu_dirty_any { + return Ok(()); + } + let (base, pitch, log2, _) = v.parts(); + let (pitch, log2) = (pitch as u64, log2 as u32); + let size = self.vram.size(); + let (tx0, tx1) = ((x0 >> 4) as u64, (x1 >> 4) as u64); + // The marked blocks under the box, as runs. + let mut runs: Vec<(u64, u64)> = Vec::new(); + let block = 1u64 << DIRTY_BLOCK_SHIFT; + for ty in (y0 >> 4) as u64..=(y1 >> 4) as u64 { + let lo = base + (((ty * pitch + tx0) << 8) << log2); + let hi = base + (((ty * pitch + tx1 + 1) << 8) << log2); + if hi > size { + // Wrapping addresses: everything. + return self.read_back_now(); + } + for b in (lo >> DIRTY_BLOCK_SHIFT)..=((hi - 1) >> DIRTY_BLOCK_SHIFT) { + let (w, bit) = ((b >> 6) as usize, 1u64 << (b & 63)); + if self.gpu_dirty[w] & bit == 0 { + continue; + } + self.gpu_dirty[w] &= !bit; + let off = b * block; + match runs.last_mut() { + Some((o, l)) if *o + *l == off => *l += block, + _ => runs.push((off, block)), + } + } + } + if runs.is_empty() { + return Ok(()); + } + // Those blocks were marked by records: submitted or not, they are + // drawn once the flush below has run. + self.flush(); + self.note_wait(); + let bytes = self.vram.read_ranges(&runs)?; + let words: &[u64] = bytemuck::cast_slice(&bytes); + let mut at = 0usize; + for (off, len) in &runs { + let n = (len >> 3) as usize; + self.mirror.write_words_untracked(*off, &words[at..at + n]); + at += n; + } + self.read_back_bytes += bytes.len() as u64; + self.readbacks += 1; + self.gpu_dirty_any = self.gpu_dirty.iter().any(|w| *w != 0); + self.gpu.check() + } + + // The quad ring's commands (pipe/ring.rs) as records. + + /// A filled triangle (`QuadRing::push_tri`): see `Rec::tri`. With a + /// texture slot the kernel textures it; `specular` adds the separate + /// specular colour after. + #[allow(clippy::too_many_arguments, reason = "the record's sources")] + pub fn push_tri(&mut self, t: &Tri, ox: i32, oy: i32, bounds: (i32, i32, i32, i32), steps: i64, depth_range: [f64; 2], depth_slack: f64, mode: u32, alpha: u32, slot: &Slot, tex: Option<&TexSlot>, specular: bool) { + let r = Rec::tri(t, ox, oy, bounds, steps, depth_range, depth_slack, mode, alpha, tex.is_some(), specular); + self.push_with_tex(r, slot, tex); + } + + /// The batch's index for texture slot `t`, or Full. + fn tex_index(&mut self, t: &TexSlot) -> Result { + let h = t.hash(); + if let Some(&i) = self.tex_map.get(&h) { + if self.tex_slots[i as usize] == *t { + return Ok(i); + } + } + if self.tex_slots.len() as u32 >= MAX_TEX_SLOTS { + return Err(Full); + } + self.tex_slots.push(*t); + let i = self.tex_slots.len() as u32 - 1; + self.tex_map.insert(h, i); + Ok(i) + } + + /// `push` with the record's texture slot resolved in the same batch. + fn push_with_tex(&mut self, mut r: Rec, s: &Slot, tex: Option<&TexSlot>) { + self.end_pending(); + loop { + let ok = match tex { + Some(t) => match self.tex_index(t) { + Ok(i) => { + r.w[T_TEXSLOT] = i; + true + } + Err(Full) => false, + }, + None => true, + }; + if ok && self.push_once(r, s) { + return; + } + self.flush(); + } + } + + /// A quad shaded on the CPU (`QuadRing::push_shaded`). + pub fn push_frag(&mut self, q: &BackQuad, slot: &Slot) { + self.push(Rec::frag(q), slot); + } + + /// A clear's rectangle (`QuadRing::push_rect`). + pub fn push_rect(&mut self, x0: i32, y0: i32, x1: i32, y1: i32, c: &ClearDraw, slot: &Slot) { + self.push(Rec::rect(x0, y0, x1, y1, c), slot); + } + + pub fn gpu(&self) -> &Arc { + &self.gpu + } + + /// The memory buffer and its size in words, for the scanout's own + /// bind group. + pub fn vram_buffer(&self) -> (wgpu::Buffer, u64) { + (self.vram.buffer.clone(), self.vram.size() / 8) + } + + /// Record `r` under slot `s`, flushing a full batch first. + pub fn push(&mut self, r: Rec, s: &Slot) { + self.end_pending(); + self.push_record(r, s); + } + + /// Close whatever the front end was collecting: the blit run into its + /// record, the untracked region into its upload. + fn end_pending(&mut self) { + self.end_run(); + self.close_region(); + } + + /// One pixel the front end stored into the mirror untracked at (x, y) + /// of view `v` (a scratch image: memory no record of the batch + /// touches): the rectangle grows and goes up when it closes, after the + /// records before it and before any after it. + pub fn note_region(&mut self, v: &super::view::View, x: i32, y: i32) { + match &mut self.region { + Some((rv, x0, y0, x1, y1)) if *rv == *v => { + *x0 = (*x0).min(x); + *y0 = (*y0).min(y); + *x1 = (*x1).max(x); + *y1 = (*y1).max(y); + } + _ => { + self.end_pending(); + // The records so far go first; the upload lands between + // their batch and the next. + self.flush(); + self.region = Some((*v, x, y, x, y)); + } + } + } + + /// Upload the region being collected, if any. + fn close_region(&mut self) { + if let Some((v, x0, y0, x1, y1)) = self.region.take() { + self.upload_tiles(&v, x0, y0, x1, y1); + } + } + + fn push_record(&mut self, r: Rec, s: &Slot) { + while !self.push_once(r, s) { + self.flush(); + } + } + + /// One pixel of a CPU path that stores pixels in rows (an image put, + /// an integer primitive, a window-id write): `word` at plane (x, y), + /// stored as `key` says under slot `s`. Consecutive pixels of a row + /// with one key join one blit record; anything else the unit does + /// first closes the run, so the pixels keep their place in the order. + pub fn blit_pixel(&mut self, s: &Slot, key: &BlitKey, x: i32, y: i32, word: u32) { + let joins = self.run.as_ref().is_some_and(|r| r.y == y && r.next_x == x && r.key == *key && r.slot == *s); + if !joins { + self.end_pending(); + if self.stage.len() + 1 > STAGE_WORDS { + self.flush(); + } + self.run = Some(BlitRun { slot: *s, key: *key, y, x0: x, next_x: x, off: self.stage.len() as u32 }); + } else if self.stage.len() + 1 > STAGE_WORDS { + // The stage is full mid-row: close the run and start the rest + // of the row in the next batch. + self.end_run(); + self.flush(); + self.run = Some(BlitRun { slot: *s, key: *key, y, x0: x, next_x: x, off: 0 }); + } + self.stage.push(word); + self.run.as_mut().expect("the run").next_x = x + 1; + self.opened.get_or_insert_with(std::time::Instant::now); + } + + /// Close the blit run being collected, if any, into its record. + fn end_run(&mut self) { + let Some(r) = self.run.take() else { return }; + let rec = Rec::blit(r.x0, r.y, r.next_x - 1, r.off, &r.key); + // The record and its pixels go in the same batch: the stage is + // uploaded by the flush that submits them. + if !self.push_once(rec, &r.slot) { + // The batch is full of records: the pixels must move to the + // next batch's stage with the record. + let words: Vec = self.stage.drain(r.off as usize..).collect(); + self.flush(); + let rec = Rec::blit(r.x0, r.y, r.next_x - 1, 0, &r.key); + self.stage.extend_from_slice(&words); + self.push_record(rec, &r.slot); + } + } + + /// Record `r` under slot `s` in the current batch; false when the batch + /// is full. + fn push_once(&mut self, mut r: Rec, s: &Slot) -> bool { + let slot = match &self.last { + Some((last, i)) if last == s => *i, + _ => match self.batch.slot(*s) { + Ok(i) => { + self.last = Some((*s, i)); + i + } + Err(Full) => return false, + }, + }; + r.slot = slot; + let (x0, y0, x1, y1) = r.pixel_box(); + match self.batch.prim(r, x0, y0, x1, y1) { + Ok(Some(_)) => { + self.opened.get_or_insert_with(std::time::Instant::now); + if r.kind == REC_BLIT && r.flags & BLIT_OWN_VIEW != 0 { + let mut own = Slot::zeroed(); + own.views[0].copy_from_slice(&r.w[B_OWN..B_OWN + 4]); + self.mark_gpu(&own, (x0, y0, x1, y1)); + } else { + self.mark_gpu(s, (x0, y0, x1, y1)); + } + true + } + Ok(None) => { + self.dropped += 1; + true + } + Err(Full) => false, + } + } + + /// Submit what was recorded, the mirror's changes ahead of it; nothing + /// waits. + pub fn flush(&mut self) { + self.end_pending(); + // The mirror's changes go up whether or not records follow: the + // scanout reads the GPU's memory directly. + self.upload_dirty(); + if self.batch.is_empty() { + self.publish_covered(); + return; + } + if !self.stage.is_empty() { + self.gpu.queue.write_buffer(&self.stage_buf, 0, bytemuck::cast_slice(&self.stage)); + self.stage.clear(); + } + if !self.tex_slots.is_empty() { + self.gpu.queue.write_buffer(&self.tex_buf, 0, bytemuck::cast_slice(&self.tex_slots)); + self.tex_slots.clear(); + self.tex_map.clear(); + } + if let Some(sub) = self.batch.flush(&self.pipelines, &[&self.group]) { + self.pending = Some(sub); + } + self.last = None; + self.opened = None; + // Everything recorded from the stores retired by the last tick is + // now submitted; the display may latch flips armed after them. + self.publish_covered(); + } + + /// Submit and wait until the GPU has stored everything recorded, so + /// memory can be read. + pub fn wait(&mut self) -> Result<(), String> { + self.flush(); + // Nothing submitted since the last wait: nothing to wait for, and + // a poll would also wait for the display's scanout. + let Some(sub) = self.pending.take() else { + return self.gpu.check(); + }; + self.waits += 1; + let r = self.gpu.wait(Some(sub)); + self.gpu.check()?; + r + } + + /// Batches flushed and records drawn so far. + pub fn counts(&self) -> (u64, u64) { + (self.batch.flushes, self.batch.prims_total) + } + + /// Write `words` at byte offset `offset` of board memory, in the + /// model's word order (vram.rs: a `Vram`'s words as bytes). + pub fn write_words(&mut self, offset: u64, words: &[u64]) { + self.flush(); + self.vram.write(offset, bytemuck::cast_slice(words)); + self.uploaded += 8 * words.len() as u64; + } + + /// Zero all of board memory (the mirror has been zeroed by the caller). + pub fn clear_all(&mut self) { + self.flush(); + self.vram.clear(0, self.vram.size()); + for w in self.gpu_dirty.iter_mut() { + *w = 0; + } + self.gpu_dirty_any = false; + self.mirror.clear_dirty(); + } + + /// Board memory as the model's words, after a wait. + pub fn read_words(&mut self) -> Result, String> { + self.wait()?; + let bytes = self.vram.read_all()?; + Ok(bytemuck::cast_slice(&bytes).to_vec()) + } + + /// blake3 of board memory in address order, the lowest address the + /// highest byte of each word (`VramPtr::address_bytes`, what + /// `debug::memory_hash_bytes` hashes). + pub fn hash(&mut self) -> Result<[u8; 32], String> { + let words = self.read_words()?; + let mut h = blake3::Hasher::new(); + let mut buf = Vec::with_capacity(1 << 16); + for w in words { + buf.extend_from_slice(&w.to_be_bytes()); + if buf.len() == buf.capacity() { + h.update(&buf); + buf.clear(); + } + } + h.update(&buf); + Ok(*h.finalize().as_bytes()) + } +} diff --git a/src/dev/odyssey/wgpu/scanout.rs b/src/dev/odyssey/wgpu/scanout.rs new file mode 100644 index 000000000..205b5d980 --- /dev/null +++ b/src/dev/odyssey/wgpu/scanout.rs @@ -0,0 +1,152 @@ +//! The VPro scan-out on the GPU (`crate::wgpu::scanout`): the display +//! slot the kernel reads, resolved on the CPU from the registers as +//! `Frame::snapshot` resolves them, and the unit the display thread runs. + +use std::sync::Arc; + +use bytemuck::{Pod, Zeroable}; + +use crate::wgpu::{Gpu, Scanout, ScanoutPipeline, ScanoutShaders}; + +use super::super::consts::{cursor, dispmode, lut, overlay}; +use super::super::field::Codec; +use super::super::frame::Frame; +use super::super::reader::DisplayRegs; +use super::{codec_kind, Unsupported, VIEWS, VRAM}; + +const SCANOUT: crate::wgpu::Fragment = crate::wgpu::Fragment::new("vpro_scanout.wgsl", include_str!("shaders/scanout.wgsl")); + +/// What `shaders/scanout.wgsl` reads of a frame. +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +pub(crate) struct DisplaySlot { + head: [u32; 16], + ids: [u32; 8 * dispmode::COUNT as usize], + overlay: [u32; 2 * overlay::IDS as usize], + gamma: [u32; 768], + cursor: [u32; cursor::ENTRIES as usize], + rgb: [u32; lut::SIZE as usize], +} + +const _: () = assert!(size_of::() == 4 * (16 + 256 + 16 + 768 + 256 + 8192)); + +impl DisplaySlot { + /// The slot of `frame` (Frame::snapshot) under registers `regs`. + pub fn of(frame: &Frame, regs: &DisplayRegs) -> Result { + let mut d = DisplaySlot::zeroed(); + let (fb, wid) = frame.surfaces(); + let fb_base = fb.base as u64 * super::super::vram::UNIT; + let wid_base = wid.base as u64 * super::super::vram::UNIT; + if fb_base >= 1 << 32 || wid_base >= 1 << 32 { + return Err(Unsupported("a scan-out base past 4 GB".into())); + } + d.head[0] = fb_base as u32; + d.head[1] = fb.pitch; + d.head[2] = fb.bpp_log2 as u32; + d.head[3] = fb.bytes_per_pixel(); + d.head[4] = wid_base as u32; + d.head[5] = wid.pitch; + d.head[6] = frame.width; + d.head[7] = frame.height; + d.head[8] = regs.cursor_x() as u32; + d.head[9] = regs.cursor_y() as u32; + d.head[10] = regs.cursor_shown() as u32 | if regs.cursor[cursor::CONTROL as usize] & cursor::CTL_CROSSHAIR != 0 { 2 } else { 0 }; + d.head[11] = frame.wid_layout.front.at as u32; + d.head[12] = frame.wid_layout.back.at as u32; + let ctx = &frame.ctx; + for id in 0..dispmode::COUNT as usize { + let show = &ctx.ids[id]; + let i = id * 8; + d.ids[i] = show.buf as u32; + d.ids[i + 1] = show.rgb as u32; + d.ids[i + 2] = show.keep; + d.ids[i + 3] = show.down; + d.ids[i + 4] = show.base; + d.ids[i + 5] = show.mask; + for b in 0..2 { + let f = &frame.fields[id][b]; + let kind = match f.codec { + Codec::Byte => super::KIND_BYTE, + c => codec_kind(c)?, + }; + d.ids[i + 6 + b] = kind | (f.at as u32) << 24; + } + } + for (o, (enabled, base)) in ctx.overlay.iter().enumerate() { + d.overlay[o * 2] = *enabled as u32; + d.overlay[o * 2 + 1] = *base; + } + for c in 0..3 { + d.gamma[c * 256..(c + 1) * 256].copy_from_slice(&ctx.gamma[c]); + } + d.cursor = regs.cursor; + d.rgb.copy_from_slice(&ctx.rgb[..]); + Ok(d) + } +} + +/// The frame's pixels the scanout can hold. +const MAX_PIXELS: u32 = 2048 * 2048; + +/// The display thread's scan-out on the GPU over the board's memory. +pub(crate) struct VproScanout { + pipeline: ScanoutPipeline, + scanout: Scanout, + vram_group: wgpu::BindGroup, +} + +impl VproScanout { + /// The device the scan-out (and the raster unit) runs on. + pub fn gpu(&self) -> Arc { + Arc::clone(self.scanout.gpu()) + } + + /// Over the raster unit's memory buffer. + pub fn new(gpu: &Arc, vram: &wgpu::Buffer, words: u64) -> Result { + let layout = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("vpro scanout vram"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Storage { read_only: false }, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }], + }); + let pipeline = ScanoutPipeline::new( + gpu, + &ScanoutShaders { + label: "vpro", + body: &[SCANOUT], + library: &[VRAM, VIEWS], + defines: &[("VRAM_WORD_MASK", (words - 1) as u32)], + layouts: &[&layout], + }, + )?; + let vram_group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("vpro scanout vram"), + layout: &layout, + entries: &[wgpu::BindGroupEntry { binding: 0, resource: vram.as_entire_binding() }], + }); + let scanout = Scanout::new(gpu, &pipeline, MAX_PIXELS, size_of::() as u64); + Ok(VproScanout { pipeline, scanout, vram_group }) + } + + /// Compose `frame` under `regs` into `out` (rows of `width`). + pub fn run(&mut self, frame: &Frame, regs: &DisplayRegs, width: u32, height: u32, out: &mut [u32]) -> Result<(), String> { + let slot = DisplaySlot::of(frame, regs).map_err(|e| e.0)?; + self.scanout.run(&self.pipeline, width, height, bytemuck::bytes_of(&slot), &[&self.vram_group], out) + } + + /// Compose `frame` under `regs` (rows of `width`) and submit it with no + /// readback; the buffer it is in, for a renderer on this device. + pub fn compose(&mut self, frame: &Frame, regs: &DisplayRegs, width: u32, height: u32) -> Result { + let slot = DisplaySlot::of(frame, regs).map_err(|e| e.0)?; + self.scanout.compose(&self.pipeline, width, height, bytemuck::bytes_of(&slot), &[&self.vram_group])?; + Ok(self.scanout.output().clone()) + } + + /// Frames composed and bytes read back. + pub fn counts(&self) -> (u64, u64) { + (self.scanout.frames, self.scanout.read_back_bytes) + } +} diff --git a/src/dev/odyssey/wgpu/shaders/backend.wgsl b/src/dev/odyssey/wgpu/shaders/backend.wgsl new file mode 100644 index 000000000..1a1858475 --- /dev/null +++ b/src/dev/odyssey/wgpu/shaders/backend.wgsl @@ -0,0 +1,509 @@ +// The back end (pipe/backend.rs) from the board's words: the stencil and +// depth tests and their operations, the destination read, the blend or +// logical operation, the colour mask, the stores; and a clear's pixel. +// The colour arithmetic is integer at 12 bits a component, as there. +// +// A slot holds the back-end words as the command FIFO delivered them +// (pipe/state.rs `BackendWords`), the facts about the draw the words do +// not carry (stencil planes, index mode, an overlay context), the +// destination's views, the window's clip rectangles and id test, and the +// polygon stipple. `backend_of` decodes the words as `BackendState::of` +// does; `fragment_q` is the generic fragment path, which the +// monomorphised stores on the CPU equal for the pixels inside a window. + +#include "views.wgsl" + +// Slot flags. +const SLOT_HAS_STENCIL: u32 = 1u; +const SLOT_RGBA: u32 = 2u; +const SLOT_TO_AUX: u32 = 4u; +const SLOT_STIPPLE: u32 = 8u; +const SLOT_BACKGROUND: u32 = 16u; + +// View indices. +const V_COLOUR0: u32 = 0u; +const V_COLOUR1: u32 = 1u; +const V_DEPTH: u32 = 2u; +const V_STENCIL: u32 = 3u; +const V_DID: u32 = 4u; +const V_OVERLAY: u32 = 5u; + +struct Slot { + mode: u32, // 0x404 + depth_func: u32, // 0x420 + write_mask: u32, // 0x452 + stencil: u32, // 0x41f + blend0: u32, // 0xc21 + blend1: u32, + blend2: u32, + logic_op: u32, // 0x422 + draw_mask: u32, // 0x424 + flags: u32, + clip_kept: u32, + id_compare: u32, // 0x840 operand 1 + pad0: u32, + pad1: u32, + pad2: u32, + pad3: u32, + views: array, + clip: array, 4>, + pattern: array, + bg12: vec4, +} + +@group(1) @binding(1) var slots: array; + +// Register bits (consts.rs). +const MODE_DEPTH_TEST: u32 = 1u << 16u; +const MODE_STENCIL_TEST: u32 = 1u << 15u; +const MODE_BLEND: u32 = 1u << 17u; +const MODE_INDEX_LOGIC_OP: u32 = 1u << 19u; +const MODE_COLOR_LOGIC_OP: u32 = 1u << 20u; +const WRITE_DEPTH: u32 = 1u << 24u; +const WRITE_COLOUR_SHIFT: u32 = 20u; +const WRITE_STENCIL_SHIFT: u32 = 12u; +const WRITE_INDEX_MASK: u32 = 0xfffu; +const STENCIL_FUNC_SHIFT: u32 = 25u; +const STENCIL_VALUE_MASK_SHIFT: u32 = 8u; +const STENCIL_OP_FAIL_SHIFT: u32 = 22u; +const STENCIL_OP_ZFAIL_SHIFT: u32 = 19u; +const STENCIL_OP_ZPASS_SHIFT: u32 = 16u; +const BLEND_DST_MASK: u32 = 0x3fu; +const BLEND_SRC_SHIFT: u32 = 6u; +const BLEND_SRC_MASK: u32 = 0x7fu; +const BLEND_EQ_SHIFT: u32 = 13u; +const DRAW_MASK_BUFFER0: u32 = 0x3u; +const DRAW_MASK_BUFFER1: u32 = 0xcu; +const ID_COMPARE_ENABLE: u32 = 1u << 16u; +const ID_COMPARE_MASK_SHIFT: u32 = 8u; +const DEPTH24_MAX: u32 = 0xffffffu; + +// Blend factors, in the table order of backend.rs FACTORS. +const BF_ZERO: u32 = 0u; +const BF_ONE: u32 = 1u; +const BF_OTHER: u32 = 2u; +const BF_SRC_ALPHA: u32 = 3u; +const BF_DST_ALPHA: u32 = 4u; +const BF_CONST: u32 = 5u; +const BF_CONST_ALPHA: u32 = 6u; +const BF_ONE_MINUS_OTHER: u32 = 7u; +const BF_ONE_MINUS_SRC_ALPHA: u32 = 8u; +const BF_ONE_MINUS_DST_ALPHA: u32 = 9u; +const BF_ONE_MINUS_CONST: u32 = 10u; +const BF_ONE_MINUS_CONST_ALPHA: u32 = 11u; +const BF_SRC_ALPHA_SATURATE: u32 = 12u; +const BF_UNKNOWN: u32 = 13u; + +// Blend equations (BlendEquation::of). +const BE_ADD: u32 = 0u; +const BE_MIN: u32 = 1u; +const BE_MAX: u32 = 2u; +const BE_LOGIC_OP: u32 = 5u; +const BE_SUBTRACT: u32 = 6u; +const BE_REVERSE_SUBTRACT: u32 = 7u; + +// Stencil operations (StencilOp::of). +const SO_KEEP: u32 = 0u; +const SO_ZERO: u32 = 1u; +const SO_REPLACE: u32 = 2u; +const SO_INCR: u32 = 3u; +const SO_DECR: u32 = 4u; +const SO_INVERT: u32 = 5u; + +// The board's blend factor code to the table index (BlendFactor::of). +fn blend_factor(code: u32) -> u32 { + var r: u32; + switch code { + case 0x00u: { r = BF_ZERO; } + case 0x08u: { r = BF_ONE; } + case 0x10u: { r = BF_OTHER; } + case 0x18u: { r = BF_SRC_ALPHA; } + case 0x20u: { r = BF_DST_ALPHA; } + case 0x28u: { r = BF_CONST; } + case 0x30u: { r = BF_CONST_ALPHA; } + case 0x38u: { r = BF_ONE_MINUS_OTHER; } + case 0x39u: { r = BF_ONE_MINUS_SRC_ALPHA; } + case 0x3au: { r = BF_ONE_MINUS_DST_ALPHA; } + case 0x3bu: { r = BF_ONE_MINUS_CONST; } + case 0x3cu: { r = BF_ONE_MINUS_CONST_ALPHA; } + case 0x40u: { r = BF_SRC_ALPHA_SATURATE; } + default: { r = BF_UNKNOWN; } + } + return r; +} + +// Compare::test for the 3-bit code. +fn compare(code: u32, a: u32, b: u32) -> bool { + var r: bool; + switch code & 7u { + case 0u: { r = false; } + case 1u: { r = a < b; } + case 2u: { r = a == b; } + case 3u: { r = a <= b; } + case 4u: { r = a > b; } + case 5u: { r = a != b; } + case 6u: { r = a >= b; } + default: { r = true; } + } + return r; +} + +// logic.rs logic_op. +fn logic_op(op: u32, s: u32, d: u32) -> u32 { + var r: u32; + switch op & 0xfu { + case 0u: { r = 0u; } + case 1u: { r = s & d; } + case 2u: { r = s & ~d; } + case 3u: { r = s; } + case 4u: { r = ~s & d; } + case 5u: { r = d; } + case 6u: { r = s ^ d; } + case 7u: { r = s | d; } + case 8u: { r = ~(s | d); } + case 9u: { r = ~(s ^ d); } + case 10u: { r = ~d; } + case 11u: { r = s | ~d; } + case 12u: { r = ~s; } + case 13u: { r = ~s | d; } + case 14u: { r = ~(s & d); } + default: { r = 0xffffffffu; } + } + return r; +} + +// What the back end decides once per draw (BackendState). +struct Backend { + depth_test: bool, + depth_func: u32, + depth_write: bool, + stencil: bool, + stencil_func: u32, + stencil_ref: u32, + stencil_value_mask: u32, + stencil_ops: vec3, + stencil_write_mask: u32, + blend: bool, + blend_src: u32, + blend_dst: u32, + blend_eq: u32, + const_colour: vec4, + has_logic: bool, + logic: u32, + colour_mask: u32, + index_mask: u32, + rgba: bool, + target0: bool, + target1: bool, + reads: bool, +} + +fn field12(word: u32, shift: u32) -> u32 { + return (word >> shift) & 0xfffu; +} + +// BackendState::of(words, has_stencil, rgba) with aux_targets. +fn backend_of(si: u32) -> Backend { + var b: Backend; + let has_stencil = (slots[si].flags & SLOT_HAS_STENCIL) != 0u; + let rgba = (slots[si].flags & SLOT_RGBA) != 0u; + let blend_on = (slots[si].mode & MODE_BLEND) != 0u; + let eq = (slots[si].blend0 >> BLEND_EQ_SHIFT) & 7u; + let index_mask = slots[si].write_mask & WRITE_INDEX_MASK; + + b.depth_test = (slots[si].mode & MODE_DEPTH_TEST) != 0u; + b.depth_func = slots[si].depth_func & 7u; + b.depth_write = b.depth_test && (slots[si].write_mask & WRITE_DEPTH) != 0u; + b.stencil = has_stencil && (slots[si].mode & MODE_STENCIL_TEST) != 0u; + b.stencil_func = (slots[si].stencil >> STENCIL_FUNC_SHIFT) & 7u; + b.stencil_ref = min(slots[si].stencil & 0xffu, 0xffu); + b.stencil_value_mask = (slots[si].stencil >> STENCIL_VALUE_MASK_SHIFT) & 0xffu; + b.stencil_ops = vec3((slots[si].stencil >> STENCIL_OP_FAIL_SHIFT) & 7u, (slots[si].stencil >> STENCIL_OP_ZFAIL_SHIFT) & 7u, (slots[si].stencil >> STENCIL_OP_ZPASS_SHIFT) & 7u); + b.stencil_write_mask = (slots[si].write_mask >> WRITE_STENCIL_SHIFT) & 0xffu; + b.has_logic = (rgba && (slots[si].mode & MODE_COLOR_LOGIC_OP) != 0u) || (!rgba && (slots[si].mode & MODE_INDEX_LOGIC_OP) != 0u) || (blend_on && eq == BE_LOGIC_OP); + b.logic = slots[si].logic_op & 0xfu; + b.blend = rgba && blend_on && !b.has_logic; + b.blend_src = blend_factor((slots[si].blend0 >> BLEND_SRC_SHIFT) & BLEND_SRC_MASK); + b.blend_dst = blend_factor(slots[si].blend0 & BLEND_DST_MASK); + b.blend_eq = eq; + b.const_colour = vec4(min(field12(slots[si].blend1, 12u), 4095u), min(field12(slots[si].blend1, 0u), 4095u), min(field12(slots[si].blend2, 12u), 4095u), min(field12(slots[si].blend2, 0u), 4095u)); + if rgba { + b.colour_mask = (slots[si].write_mask >> WRITE_COLOUR_SHIFT) & 0xfu; + } else { + b.colour_mask = select(0u, 0xfu, index_mask != 0u); + } + b.index_mask = index_mask; + b.rgba = rgba; + b.target0 = (slots[si].draw_mask & DRAW_MASK_BUFFER0) != 0u; + b.target1 = (slots[si].draw_mask & DRAW_MASK_BUFFER1) != 0u; + if (slots[si].flags & SLOT_TO_AUX) != 0u { + // An overlay context's draw: an index goes to buffer B; RGBA has + // no home there. + b.target0 = false; + b.target1 = !rgba && b.target1; + } + if rgba { + b.reads = b.blend || b.has_logic || b.colour_mask != 0xfu; + } else { + b.reads = b.has_logic || index_mask != WRITE_INDEX_MASK; + } + return b; +} + +// The window's ownership of plane pixel (x, y): the destination's width, +// the clip list, and the id test (RasterState::owned). +fn owned(si: u32, x: i32, y: i32) -> bool { + let width = view_width(slots[si].views[V_COLOUR0]); + if x < 0 || y < 0 || u32(x) >= width || u32(y) >= MAX_DIM { + return false; + } + let n = min(slots[si].clip_kept, 4u); + var inside = false; + for (var i = 0u; i < n; i++) { + let r = slots[si].clip[i]; + inside = inside || (x >= r.x && x <= r.z && y >= r.y && y <= r.w); + } + if !inside { + return false; + } + if (slots[si].id_compare & ID_COMPARE_ENABLE) != 0u { + let id_mask = (slots[si].id_compare >> ID_COMPARE_MASK_SHIFT) & 0xffu; + if id_mask != 0u { + let id = view_load(slots[si].views[V_DID], x, y); + let value = slots[si].id_compare & 0xffu; + if ((id.v ^ value) & id_mask) != 0u { + return false; + } + } + } + return true; +} + +// Packers. +fn pack_rgba8(c: vec4) -> u32 { + return ((c.x & 0xffu) << 24u) | ((c.y & 0xffu) << 16u) | ((c.z & 0xffu) << 8u) | (c.w & 0xffu); +} + +fn unpack_rgba8(w: u32) -> vec4 { + return vec4(w >> 24u, (w >> 16u) & 0xffu, (w >> 8u) & 0xffu, w & 0xffu); +} + +fn index_of(src: vec4) -> u32 { + return (((src.y >> 4u) & 0xfu) << 8u) | (src.z >> 4u); +} + +fn widen8(v: u32) -> u32 { + return (v << 4u) | (v >> 4u); +} + +fn mul12(a: u32, b: u32) -> u32 { + let p = a * b; + return (p + (p >> 12u) + 2048u) >> 12u; +} + +// A blend factor's 12-bit value for component k. +fn factor(f: u32, k: u32, other_is_dst: bool, s: vec4, d: vec4, c: vec4) -> u32 { + let other = select(s, d, other_is_dst); + var r: u32; + switch f { + case BF_ZERO: { r = 0u; } + case BF_ONE: { r = 4095u; } + case BF_OTHER: { r = other[k]; } + case BF_SRC_ALPHA: { r = s.w; } + case BF_DST_ALPHA: { r = d.w; } + case BF_CONST: { r = c[k]; } + case BF_CONST_ALPHA: { r = c.w; } + case BF_ONE_MINUS_OTHER: { r = 4095u - other[k]; } + case BF_ONE_MINUS_SRC_ALPHA: { r = 4095u - s.w; } + case BF_ONE_MINUS_DST_ALPHA: { r = 4095u - d.w; } + case BF_ONE_MINUS_CONST: { r = 4095u - c[k]; } + case BF_ONE_MINUS_CONST_ALPHA: { r = 4095u - c.w; } + case BF_SRC_ALPHA_SATURATE: { r = select(min(s.w, 4095u - d.w), 4095u, k == 3u); } + default: { r = 0u; } + } + return r; +} + +// The blend of the fragment's 12-bit components over the destination's. +fn blend(b: Backend, s: vec4, d: vec4) -> vec4 { + var out: vec4; + for (var k = 0u; k < 4u; k++) { + let sf = factor(b.blend_src, k, true, s, d, b.const_colour); + let df = factor(b.blend_dst, k, false, s, d, b.const_colour); + let a = i32(mul12(s[k], sf)); + let bb = i32(mul12(d[k], df)); + var v: i32; + switch b.blend_eq { + case BE_MIN: { v = i32(min(s[k], d[k])); } + case BE_MAX: { v = i32(max(s[k], d[k])); } + case BE_SUBTRACT: { v = a - bb; } + case BE_REVERSE_SUBTRACT: { v = bb - a; } + default: { v = a + bb; } + } + out[k] = u32(clamp(v, 0, 4095)); + } + return out; +} + +// The 8-bit colour a fragment of 12-bit components stores over the 8-bit +// destination. +fn colour_result(b: Backend, src: vec4, dst: vec4) -> vec4 { + var out: vec4; + if b.blend { + out = blend(b, src, vec4(widen8(dst.x), widen8(dst.y), widen8(dst.z), widen8(dst.w))); + } else { + out = src; + } + out = out >> vec4(4u); + for (var k = 0u; k < 4u; k++) { + if b.has_logic { + out[k] = logic_op(b.logic, out[k], dst[k]) & 0xffu; + } + if (b.colour_mask & (1u << k)) == 0u { + out[k] = dst[k]; + } + } + return out; +} + +// The stencil operation op on old under the write mask; `ok` false keeps. +fn stencil_value(b: Backend, op: u32, old: u32) -> Loaded { + var value: u32; + switch op { + case SO_ZERO: { value = 0u; } + case SO_REPLACE: { value = b.stencil_ref; } + case SO_INCR: { value = select(0xffu, old + 1u, old < 0xffu); } + case SO_DECR: { value = select(old - 1u, 0u, old == 0u); } + case SO_INVERT: { value = ~old & 0xffu; } + default: { return Loaded(false, 0u); } + } + let mask = b.stencil_write_mask & 0xffu; + return Loaded(true, (old & ~mask) | (value & mask)); +} + +fn store_index(b: Backend, si: u32, x: i32, y: i32, index: u32) { + for (var t = 0u; t < 2u; t++) { + if !select(b.target0, b.target1, t == 1u) { + continue; + } + let v = slots[si].views[V_COLOUR0 + t]; + if !b.reads { + view_store(v, x, y, index); + continue; + } + let old = view_load(v, x, y); + if !old.ok { + continue; + } + var val = index; + if b.has_logic { + val = logic_op(b.logic, index, old.v); + } + view_store(v, x, y, (val & b.index_mask) | (old.v & ~b.index_mask)); + } +} + +// One fragment at plane pixel (x, y) with depth word zq and colour src in +// 12 bits a component, through the tests and into memory +// (BackendState::fragment_q). +fn fragment_q(b: Backend, si: u32, x: i32, y: i32, zq: u32, src: vec4) { + var outcome = 2u; + var old_stencil = 0u; + if b.stencil { + old_stencil = view_load(slots[si].views[V_STENCIL], x, y).v; + if !compare(b.stencil_func, b.stencil_ref & b.stencil_value_mask, old_stencil & b.stencil_value_mask) { + outcome = 0u; + } + } + if outcome == 2u && b.depth_test { + let stored = view_load(slots[si].views[V_DEPTH], x, y).v; + if !compare(b.depth_func, zq, stored) { + outcome = 1u; + } + } + if outcome != 2u { + if b.stencil { + let v = stencil_value(b, b.stencil_ops[outcome], old_stencil); + if v.ok { + view_store(slots[si].views[V_STENCIL], x, y, v.v); + } + } + return; + } + if b.depth_write { + view_store(slots[si].views[V_DEPTH], x, y, zq & DEPTH24_MAX); + } + if b.stencil { + let v = stencil_value(b, b.stencil_ops[2], old_stencil); + if v.ok { + view_store(slots[si].views[V_STENCIL], x, y, v.v); + } + } + if b.colour_mask == 0u { + return; + } + if !b.rgba { + store_index(b, si, x, y, index_of(src)); + return; + } + for (var t = 0u; t < 2u; t++) { + if !select(b.target0, b.target1, t == 1u) { + continue; + } + let v = slots[si].views[V_COLOUR0 + t]; + if !b.reads { + view_store(v, x, y, pack_rgba8(src >> vec4(4u))); + continue; + } + let d = view_load(v, x, y); + if !d.ok { + continue; + } + view_store(v, x, y, pack_rgba8(colour_result(b, src, unpack_rgba8(d.v)))); + } +} + +// A clear's pixel (ClearDraw::span). The record carries the values; the +// slot the planes and the id test. +const CLEAR_COLOUR: u32 = 1u; +const CLEAR_DEPTH: u32 = 2u; +const CLEAR_STENCIL: u32 = 4u; +const CLEAR_TARGET0: u32 = 8u; +const CLEAR_TARGET1: u32 = 16u; + +fn clear_pixel(si: u32, flags: u32, x: i32, y: i32, colour: u32, colour_mask: u32, depth: u32, stencil_vm: u32, id_vm: u32) { + let id_mask = id_vm >> 8u; + if id_mask != 0u { + let id = view_load(slots[si].views[V_DID], x, y); + if !id.ok || ((id.v ^ (id_vm & 0xffu)) & id_mask) != 0u { + return; + } + } + if (flags & CLEAR_DEPTH) != 0u { + view_store(slots[si].views[V_DEPTH], x, y, depth & DEPTH24_MAX); + } + if (flags & CLEAR_STENCIL) != 0u { + let old = view_load(slots[si].views[V_STENCIL], x, y); + if old.ok { + let m = stencil_vm >> 8u; + view_store(slots[si].views[V_STENCIL], x, y, (old.v & ~m) | (stencil_vm & m)); + } + } + if (flags & CLEAR_COLOUR) != 0u { + for (var t = 0u; t < 2u; t++) { + if (flags & (CLEAR_TARGET0 << t)) == 0u { + continue; + } + let v = slots[si].views[V_COLOUR0 + t]; + var out = colour; + if colour_mask != 0xffffffffu { + let d = view_load(v, x, y); + if !d.ok { + continue; + } + out = (colour & colour_mask) | (d.v & ~colour_mask); + } + view_store(v, x, y, out); + } + } +} diff --git a/src/dev/odyssey/wgpu/shaders/scanout.wgsl b/src/dev/odyssey/wgpu/shaders/scanout.wgsl new file mode 100644 index 000000000..1348ba5f3 --- /dev/null +++ b/src/dev/odyssey/wgpu/shaders/scanout.wgsl @@ -0,0 +1,111 @@ +// The VPro scan-out for one screen pixel (frame.rs `Frame::pixel` and +// reader.rs `DisplayRegs::cursor_row`): the window-id byte picks the +// overlay or the window's display mode, the colour field of the buffer the +// window shows is decoded and sent through the gamma tables or the colour +// lookup RAM, and the cursor goes over it. The display slot +// (wgpu/scanout.rs `DisplaySlot`) is what the CPU resolves once per +// frame from the registers. + +#include "views.wgsl" + +struct DisplaySlot { + // fb base (bytes), fb pitch (tiles), fb log2 bytes a pixel, fb bytes a + // pixel, wid base, wid pitch, width, height, cursor x, cursor y, + // cursor control (bit 0 shown, bit 1 crosshair), wid front byte, wid + // back byte, pad. + head: array, + // Per main window id, 8 words: buffer shown, rgb, keep, down, base, + // mask, field A, field B (a field is its kind word with the byte + // offset in bits 31:24). + ids: array, + // Per overlay id: enabled, entry base. + overlay: array, + // Each 8-bit channel value through the gamma table, in its byte of + // the pixel: red, green, blue. + gamma: array, + // The cursor's backend entries 0x2500-0x25ff. + cursor: array, + // Every lookup-RAM entry as 0x00RRGGBB. + rgb: array, +} + +@group(1) @binding(0) var disp: DisplaySlot; + +const DID_MAIN_SHIFT: u32 = 3u; +const DID_OVERLAY_MASK: u32 = 7u; +const CURSOR_SIZE: i32 = 32; +const CURSOR_GLYPH: u32 = 0x00u; +const CURSOR_COLOURS: u32 = 0x90u; +const CURSOR_PLANE: u32 = 0xc0u; + +// The byte address of pixel (x, y) of a tiled surface (vram.rs +// Surface::offset). +fn surface_offset(base: u32, pitch: u32, log2: u32, x: u32, y: u32) -> u32 { + let tile = (y >> 4u) * pitch + (x >> 4u); + return base + (((tile << 8u) | ((y & 15u) << 4u) | (x & 15u)) << log2); +} + +fn cursor_rgb(word: u32) -> u32 { + return (((word >> 22u) & 0xffu) << 16u) | (((word >> 12u) & 0xffu) << 8u) | ((word >> 2u) & 0xffu); +} + +fn scanout_pixel(x: u32, y: u32) -> u32 { + let fb_base = disp.head[0]; + let fb_pitch = disp.head[1]; + let fb_log2 = disp.head[2]; + let fb_bytes = disp.head[3]; + let w = surface_offset(disp.head[4], disp.head[5], 1u, x, y); + let p = surface_offset(fb_base, fb_pitch, fb_log2, x, y); + let did = vram_load(w + disp.head[11], 1u); + let ov = vram_load(w + disp.head[12], 1u); + var rgb = 0u; + let o = did & DID_OVERLAY_MASK; + if disp.overlay[o * 2u] != 0u && ov != 0u { + rgb = disp.rgb[(disp.overlay[o * 2u + 1u] + ov) & 8191u]; + } else { + let id = (did >> DID_MAIN_SHIFT) & 31u; + let i = id * 8u; + let buf = disp.ids[i] & 1u; + let field = disp.ids[i + 6u + buf]; + let at = field >> 24u; + if at + 2u <= fb_bytes { + let word = load_kind(p + at, field & 0xffffffu); + if disp.ids[i + 1u] != 0u { + let keep = disp.ids[i + 2u]; + let down = disp.ids[i + 3u]; + let r = ((((word >> 24u) & keep) | ((word >> 24u) >> down))) & 0xffu; + let g = (((((word >> 16u) & 0xffu) & keep) | (((word >> 16u) & 0xffu) >> down))) & 0xffu; + let b = (((((word >> 8u) & 0xffu) & keep) | (((word >> 8u) & 0xffu) >> down))) & 0xffu; + rgb = disp.gamma[r] | disp.gamma[256u + g] | disp.gamma[512u + b]; + } else { + rgb = disp.rgb[(disp.ids[i + 4u] + (word & disp.ids[i + 5u])) & 8191u]; + } + } + } + // The cursor over it. + let ctl = disp.head[10]; + if (ctl & 1u) == 0u { + return rgb; + } + let cx = bitcast(disp.head[8]); + let cy = bitcast(disp.head[9]); + let row = i32(y) - cy; + if (ctl & 2u) != 0u { + if row == 0 || i32(x) == cx { + return cursor_rgb(disp.cursor[CURSOR_COLOURS + 1u]); + } + return rgb; + } + let col = i32(x) - cx; + if row < 0 || row >= CURSOR_SIZE || col < 0 || col >= CURSOR_SIZE { + return rgb; + } + let ur = u32(row); + let uc = u32(col); + let plane = (disp.cursor[CURSOR_PLANE + ur * 2u + uc / 16u] >> ((uc % 16u) * 2u)) & 3u; + let glyph = (disp.cursor[CURSOR_GLYPH + ur * 4u + uc / 8u] >> ((uc % 8u) * 4u)) & 0xfu; + if plane != 3u || glyph == 0xfu { + return rgb; + } + return cursor_rgb(disp.cursor[CURSOR_COLOURS + glyph]); +} diff --git a/src/dev/odyssey/wgpu/shaders/texture.wgsl b/src/dev/odyssey/wgpu/shaders/texture.wgsl new file mode 100644 index 000000000..76349305b --- /dev/null +++ b/src/dev/odyssey/wgpu/shaders/texture.wgsl @@ -0,0 +1,635 @@ +// The texture unit (pipe/texture.rs) for one pixel of a quad: the quad's +// positions in each axis, its level of detail from the diagonals, the +// filter and the levels lambda chooses, the taps gathered from board +// memory and filtered with 8-bit weights to 12 bits with 16 fraction +// bits, the detail image, the post-filter, and the texel in 12.12. The +// arithmetic is the CPU unit's operation for operation; where the CPU +// uses 64-bit products they are split here. +// +// A texture slot (wgpu/mod.rs `TexSlot`) is the draw's texture registers +// and what the CPU latches from them once (`Latch`): the levels, the LOD +// clamps and scales, the border colour. Slots are in their own buffer. + +#include "float.wgsl" +#include "views.wgsl" + +struct TexLevel { + addr: u32, + pitch: u32, + bpp: u32, // log2 of the texel size + field: u32, + bytes: u32, + ox: u32, + oy: u32, + wl: u32, + hl: u32, + dl: u32, + slice: u32, + pad: u32, +} + +struct TexSlot { + sampler0: u32, + sampler4: u32, + axes: u32, + flags: u32, + count: u32, + min_lod: i32, + max_lod: i32, + mag_below: i32, + lod_scale: array, + scale: array, + detail_scale: array, + clamp: array, + border: array, + scale_bias: array, + // The environment (ENV_*) and its colour in 12.12. + env: u32, + env_colour: array, + pad: array, + levels: array, + detail: TexLevel, + colour_table: array, + detail_fn: array, +} + +@group(2) @binding(1) var tex_slots: array; +// log2(1 + i/256) in 1/256 (fixed::log2_q8's table), i in 0..256. +@group(2) @binding(3) var log2_table: array; + +const TEX_LOD: u32 = 1u; +const TEX_POST: u32 = 2u; +const TEX_STORED_BORDER: u32 = 4u; +const TEX_TABLE: u32 = 8u; +const TEX_TABLE_ALPHA: u32 = 16u; +// The clamp word's "no clamp". +const NO_CLAMP: i32 = 0x7fffffff; +const LAMBDA_MIN: i32 = -(1 << 24); +const FIXED_ONE: i32 = 1 << 24; + +const WRAP_REPEAT: u32 = 0u; +const WRAP_CLAMP_TO_EDGE: u32 = 1u; +const WRAP_CLAMP: u32 = 2u; +const WRAP_CLAMP_TO_BORDER: u32 = 3u; + +const ENV_REPLACE: u32 = 0u; +const ENV_MODULATE: u32 = 1u; +const ENV_DECAL: u32 = 2u; +const ENV_BLEND: u32 = 3u; +const ENV_ADD: u32 = 4u; + +const CLASS_ALPHA: u32 = 0u; +const CLASS_LUMINANCE: u32 = 1u; +const CLASS_LUMINANCE_ALPHA: u32 = 2u; +const CLASS_INTENSITY: u32 = 3u; +const CLASS_RGB: u32 = 4u; +const CLASS_RGBA: u32 = 5u; + +// Sampler word 4's fields. +fn tex_wrap(ti: u32, axis: u32) -> u32 { + return (tex_slots[ti].sampler4 >> (12u - 2u * axis)) & 3u; +} + +fn tex_min(ti: u32) -> u32 { + return (tex_slots[ti].sampler4 >> 5u) & 7u; +} + +fn tex_mag(ti: u32) -> u32 { + let m = (tex_slots[ti].sampler4 >> 1u) & 0xfu; + return select(m, 1u, m > 4u); +} + +fn tex_class(ti: u32) -> u32 { + let c = (tex_slots[ti].sampler4 >> 23u) & 7u; + switch c { + case 1u: { return CLASS_ALPHA; } + case 2u: { return CLASS_LUMINANCE; } + case 3u: { return CLASS_LUMINANCE_ALPHA; } + case 4u: { return CLASS_INTENSITY; } + case 5u: { return CLASS_RGB; } + default: { return CLASS_RGBA; } + } +} + +fn min_mipmap(m: u32) -> bool { + return m >= 2u; +} + +fn min_linear(m: u32) -> bool { + return m == 1u || m == 3u || m == 5u; +} + +fn min_nearest_level(m: u32) -> bool { + return m == 2u || m == 3u; +} + +fn mag_detail(m: u32) -> bool { + return m >= 2u; +} + +// A texel's components as the cls fills them. +fn expand(cls: u32, c: vec4) -> vec4 { + let one = 0xfffu; + switch cls { + case CLASS_ALPHA: { return vec4(one, one, one, c.w); } + case CLASS_LUMINANCE: { return vec4(c.x, c.x, c.x, one); } + case CLASS_LUMINANCE_ALPHA: { return vec4(c.x, c.x, c.x, c.w); } + case CLASS_INTENSITY: { return vec4(c.x, c.x, c.x, c.x); } + case CLASS_RGB: { return vec4(c.x, c.y, c.z, one); } + default: { return c; } + } +} + +// Rust's `as i32` then the step down for a value below it: floor for |x| +// below 2^31, saturating past. +fn floor_i32(x: f32) -> i32 { + let i = sat_i32(x); + return i - select(0, 1, x < f32(i)); +} + +fn sat_u32(x: f32) -> u32 { + let c = clamp(x, 0.0, 4294967040.0); + return select(u32(c), 0u, x != x); +} + +// log2(v) in 1/256 for v > 0 (fixed::log2_q8): the leading one's position +// and a table of the eight bits after it (LOG2_TABLE comes with the +// pipeline's defines). +fn log2_q8(v: u32) -> i32 { + let n = 31u - countLeadingZeros(v); + var f: u32; + if n >= 8u { + f = (v >> (n - 8u)) & 0xffu; + } else { + f = (v << (8u - n)) & 0xffu; + } + return i32(n << 8u) + i32(log2_table[f]); +} + +// (a * (256 - f) + b * f) >> 8 for a, b below 2^28, exactly. +fn blend8(a: u32, b: u32, f: u32) -> u32 { + let g = 256u - f; + let lo = (a & 0xffu) * g + (b & 0xffu) * f; + let hi = (a >> 8u) * g + (b >> 8u) * f; + return hi + (lo >> 8u); +} + +// (v * 4097) >> 16 for v below 2^28: 12 bits with 16 fraction bits over +// 4095 as 12.12 over 4096. +fn to_fixed(v: u32) -> i32 { + let vl = v & 0xffffu; + let vh = v >> 16u; + return i32(vh * 4097u + ((vl * 4097u) >> 16u)); +} + +// The position along one axis in 1/256 of a texel of a level 2^l texels +// wide: the first tap's index before wrapping and the second tap's weight +// (LINEAR), or the tap's index (NEAREST). `big` levels (2^(l + 8) past +// i32) make the clamps no-ops. +struct Pos { + at: i32, + w: u32, +} + +fn pos(u_in: i32, l: u32, wrap: u32, linear: bool) -> Pos { + let big = l + 8u >= 31u; + let whole = select(1 << (l + 8u), 0x7fffffff, big); + let last = i32((1u << min(l, 31u)) - 1u); + var u = u_in; + if linear { + switch wrap { + case WRAP_REPEAT: { u = u - 128; } + case WRAP_CLAMP_TO_EDGE: { + u = max(u - 128, 0); + if !big { + u = min(u, whole - 256); + } + } + case WRAP_CLAMP_TO_BORDER: { + u = max(u, -128); + if !big { + u = min(u, whole + 128); + } + u = u - 128; + } + default: { + u = max(u, 0); + if !big { + u = min(u, whole); + } + u = u - 128; + } + } + return Pos(u >> 8u, u32(u & 255)); + } + var x = u >> 8u; + switch wrap { + case WRAP_REPEAT: {} + case WRAP_CLAMP_TO_BORDER: { x = min(max(x, -1), last + 1); } + default: { x = min(max(x, 0), last); } + } + return Pos(x, 0u); +} + +// One tap's part of the texel's byte offset along an axis, wrapped, and +// whether it lies outside the level (the border colour). +struct Tap { + off: u32, + outside: bool, +} + +fn axis_tap(lv: TexLevel, axis: u32, x_in: i32, wrap: u32, border: bool) -> Tap { + var l: u32; + var inset: u32; + if axis == 0u { + l = lv.wl; + inset = lv.ox; + } else if axis == 1u { + l = lv.hl; + inset = lv.oy; + } else { + l = lv.dl; + inset = 0u; + } + let mask = i32((1u << min(l, 31u)) - 1u); + var x = x_in; + var outside = false; + if wrap == WRAP_REPEAT { + x = x & mask; + } else if wrap == WRAP_CLAMP_TO_EDGE { + x = min(max(x, 0), mask); + } else if !border && (x < 0 || x > mask) { + outside = true; + } + let ux = u32(x) + inset; + var off: u32; + if axis == 0u { + off = (((ux >> 4u) << 8u) | (ux & 15u)) << lv.bpp; + } else if axis == 1u { + off = (((ux >> 4u) * lv.pitch << 8u) | ((ux & 15u) << 4u)) << lv.bpp; + } else { + off = ux * lv.slice; + } + return Tap(off, outside); +} + +// The texel at byte address a of a level, as 12-bit components of the +// cls. +fn texel_at(lv: TexLevel, cls: u32, a: u32) -> vec4 { + var w: u32; + if lv.bytes >= 8u { + w = word_of_lanes(vram_load64(a, 6u)); + } else { + w = texel_unpack(vram_load(a, lv.bytes), lv.bytes, lv.field); + } + var c: vec4; + for (var k = 0u; k < 4u; k++) { + let v = (w >> (24u - 8u * k)) & 0xffu; + c[k] = (v << 4u) | (v >> 4u); + } + if cls == CLASS_ALPHA { + c = vec4(0u, 0u, 0u, c.w); + } + return expand(cls, c); +} + +// A pixel's texel from level lv at first taps `at` with weights w: LINEAR +// over 2 (4, 8 for a volume) taps or NEAREST; 12-bit components with 16 +// fraction bits. +fn filter_level(ti: u32, lv: TexLevel, cls: u32, axes: u32, at: vec3, w: vec3, linear: bool, wraps: vec3, border: bool) -> vec4 { + let three = axes == 3u; + let bc = vec4(tex_slots[ti].border[0], tex_slots[ti].border[1], tex_slots[ti].border[2], tex_slots[ti].border[3]); + let nx = select(1u, 2u, linear); + let ny = select(1u, 2u, linear && axes >= 2u); + let nz = select(1u, 2u, linear && three); + // Every element initialised: a tap or texel a filter does not use is + // still read into a product with a zero weight, and an uninitialised + // read is not a zero to the compiler. + var tx = array(Tap(0u, false), Tap(0u, false)); + var ty = array(Tap(0u, false), Tap(0u, false)); + var tz = array(Tap(0u, false), Tap(0u, false)); + for (var i = 0u; i < nx; i++) { + tx[i] = axis_tap(lv, 0u, at.x + i32(i), wraps.x, border); + } + for (var i = 0u; i < ny; i++) { + if axes >= 2u { + ty[i] = axis_tap(lv, 1u, at.y + i32(i), wraps.y, border); + } else { + ty[i] = Tap(0u, false); + } + } + for (var i = 0u; i < nz; i++) { + if three { + tz[i] = axis_tap(lv, 2u, at.z + i32(i), wraps.z, border); + } else { + tz[i] = Tap(0u, false); + } + } + // The window of taps, texel (x, y, z) at z * 4 + y * 2 + x. + var win = array, 8>(); + for (var z = 0u; z < nz; z++) { + for (var y = 0u; y < ny; y++) { + let row = lv.addr + tz[z].off + ty[y].off; + let edge = tz[z].outside || ty[y].outside; + for (var x = 0u; x < nx; x++) { + if edge || tx[x].outside { + win[z * 4u + y * 2u + x] = bc; + } else { + win[z * 4u + y * 2u + x] = texel_at(lv, cls, row + tx[x].off); + } + } + } + } + if !linear { + return win[0] << vec4(16u); + } + let a = w.x; + let b = select(0u, w.y, axes >= 2u); + var planes = array, 2>(); + for (var z = 0u; z < nz; z++) { + let c00 = win[z * 4u]; + let c10 = win[z * 4u + 1u]; + var c01 = c00; + var c11 = c10; + if ny == 2u { + c01 = win[z * 4u + 2u]; + c11 = win[z * 4u + 3u]; + } + var out: vec4; + for (var k = 0u; k < 4u; k++) { + let top = c00[k] * (256u - a) + c10[k] * a; + let bot = c01[k] * (256u - a) + c11[k] * a; + out[k] = top * (256u - b) + bot * b; + } + planes[z] = out; + } + if !three { + return planes[0]; + } + var out: vec4; + for (var k = 0u; k < 4u; k++) { + out[k] = blend8(planes[0][k], planes[1][k], w.z); + } + return out; +} + +// The quad's texture coordinates: c[a] holds S, T, R or Q at pixels 0 to +// 3 (pixel i at x + (i & 1), y + (i >> 1)). +struct TexQuad { + c: array, 4>, +} + +// Each pixel's position in each of the first `axes` axes in 1/256 of a +// texel: S, T or R over Q times the axis's scale, floored. +fn positions(q: TexQuad, scale: array, axes: u32) -> array, 3> { + var p = array, 3>(); + for (var i = 0u; i < 4u; i++) { + let rq = div_exact(1.0, q.c[3][i]); + for (var a = 0u; a < axes; a++) { + let v = (q.c[a][i] * rq) * scale[a]; + p[a][i] = floor_i32(v); + } + } + return p; +} + +// The post-filter: the scale and bias, then the colour table. +fn tex_post(ti: u32, c_in: vec4) -> vec4 { + var c = c_in; + for (var k = 0u; k < 4u; k++) { + let scale = tex_slots[ti].scale_bias[k * 2u]; + let bias = tex_slots[ti].scale_bias[k * 2u + 1u]; + if scale != 4096 || bias != 0 { + // c * scale >> 12 in 64 bits on the CPU: c is below 2^24 and + // scale a signed 16-bit value, so split c. + let ch = c[k] >> 12u; + let cl = c[k] & 0xfff; + let v = ch * scale + ((cl * scale) >> 12u); + c[k] = clamp(v + (bias << 12u), 0, FIXED_ONE - 1); + } + } + if (tex_slots[ti].flags & TEX_TABLE) == 0u { + return c; + } + for (var k = 0u; k < 3u; k++) { + let idx = table_index(c[k]); + let word = tex_slots[ti].colour_table[select(0u, 256u, k == 2u) + idx]; + c[k] = field12_full(select(word >> 12u, word, k == 1u)); + } + if (tex_slots[ti].flags & TEX_TABLE_ALPHA) != 0u { + c.w = field12_full(tex_slots[ti].colour_table[256u + table_index(c.w)]); + } + return c; +} + +// (v * 255 + 2^23) >> 24 clamped to 0..255, for v in 12.12 (below 2^24 +// after the clamp above, so the product fits with a split). +fn table_index(v: i32) -> u32 { + let vv = clamp(v, 0, FIXED_ONE - 1); + let vh = vv >> 12u; + let vl = vv & 0xfff; + // (vh * 2^12 + vl) * 255 + 2^23, over 2^24. + let low = vl * 255 + (1 << 23); + let total = vh * 255 + (low >> 12u); + return u32(clamp(total >> 12u, 0, 255)); +} + +fn field12_full(v: u32) -> i32 { + let x = i32(v & 0xfffu); + return (x << 12u) + x; +} + +fn fixed_clamp(v: i32) -> i32 { + return clamp(v, 0, FIXED_ONE - 1); +} + +// fixed::smul: a signed difference times a fraction in [0, ONE]. +fn smul(d: i32, b: i32) -> i32 { + return ((d >> 9u) * (b >> 9u)) >> 6u; +} + +// The detail function at lambda, in 12.12. +fn detail_weight(ti: u32, lambda: i32) -> i32 { + if lambda >= 0 { + return i32((tex_slots[ti].detail_fn[0] >> 12u) & 0xfffu) << 12u; + } + let x = u32(-lambda); + let i = x >> 3u; + if i >= 255u { + return i32((tex_slots[ti].detail_fn[255] >> 12u) & 0xfffu) << 12u; + } + let w0 = i32((tex_slots[ti].detail_fn[i] >> 12u) & 0xfffu); + let w1 = i32((tex_slots[ti].detail_fn[i + 1u] >> 12u) & 0xfffu); + let f = i32(x & 7u); + return (w0 << 12u) + ((((w1 - w0) << 12u) * f) >> 3u); +} + +// Sample the quad for pixel i: the texel in 12.12. +fn sample_quad(ti: u32, q: TexQuad, i: u32) -> vec4 { + let axes = tex_slots[ti].axes; + var p = positions(q, tex_slots[ti].scale, axes); + let has_lod = (tex_slots[ti].flags & TEX_LOD) != 0u; + var lambda = 0; + if has_lod { + // The diagonal rule: the largest difference along the quad's two + // diagonals, in 1/256 of a base texel, scaled by the bias. + var rho = 0.0; + for (var k = 0u; k < 3u; k++) { + // The magnitude of a difference of two i32, which can pass + // i32: the larger less the smaller in u32. + let d0 = u32(max(p[k][3], p[k][0])) - u32(min(p[k][3], p[k][0])); + let d1 = u32(max(p[k][2], p[k][1])) - u32(min(p[k][2], p[k][1])); + rho = max(rho, i64_to_f32(I64(d0, 0u)) * tex_slots[ti].lod_scale[k]); + rho = max(rho, i64_to_f32(I64(d1, 0u)) * tex_slots[ti].lod_scale[k]); + } + let r = sat_u32(min(rho, 4294967295.0)); + lambda = select(log2_q8(r) - (8 << 8u), LAMBDA_MIN, r == 0u); + } + let mn = tex_min(ti); + let mg = tex_mag(ti); + var linear = min_linear(mn); + var first = 0u; + var second = 0u; + var frac = 0u; + var detail = false; + let range = tex_slots[ti].count - 1u; + if has_lod { + lambda = min(max(lambda, tex_slots[ti].min_lod), tex_slots[ti].max_lod); + if lambda <= tex_slots[ti].mag_below { + linear = mg != 0u; + detail = mag_detail(mg); + } else if min_mipmap(mn) { + if min_nearest_level(mn) { + first = select(min((u32(lambda) + 127u) >> 8u, range), 0u, lambda <= 128); + } else { + let rel = u32(min(max(lambda, 0), i32(range << 8u))); + first = rel >> 8u; + frac = rel & 255u; + second = min(first + 1u, range); + } + } + } + for (var a = 0u; a < 3u; a++) { + if tex_slots[ti].clamp[a] != NO_CLAMP { + p[a] = min(p[a], vec4(tex_slots[ti].clamp[a])); + } + } + let cls = tex_class(ti); + let three = axes == 3u; + let wraps = vec3(tex_wrap(ti, 0u), select(tex_wrap(ti, 1u), WRAP_CLAMP_TO_EDGE, axes == 1u), tex_wrap(ti, 2u)); + let l0 = tex_slots[ti].levels[0]; + // This pixel's taps on a level: the base position shifted down to the + // level, through pos(). + var acc = level_sample(ti, first, l0, cls, p, i, linear, wraps, three); + if frac != 0u { + let other = level_sample(ti, second, l0, cls, p, i, linear, wraps, three); + for (var k = 0u; k < 4u; k++) { + acc[k] = blend8(acc[k], other[k], frac); + } + } + var c = vec4(to_fixed(acc.x), to_fixed(acc.y), to_fixed(acc.z), to_fixed(acc.w)); + if detail { + let dscale = array(tex_slots[ti].detail_scale[0], tex_slots[ti].detail_scale[1], 0.0, 0.0); + let d = positions(q, dscale, 2u); + let lv = tex_slots[ti].detail; + let px = pos(d[0][i], lv.wl, WRAP_REPEAT, true); + let py = pos(d[1][i], lv.hl, WRAP_REPEAT, true); + let dt = filter_level(ti, lv, cls, 2u, vec3(px.at, py.at, 0), vec3(px.w, py.w, 0u), true, vec3(WRAP_REPEAT, WRAP_REPEAT, WRAP_REPEAT), false); + let weight = detail_weight(ti, lambda); + let add = (tex_slots[ti].sampler0 & 0x80000000u) == 0u; + for (var k = 0u; k < 4u; k++) { + if (k == 3u && mg == 4u) || (k != 3u && mg == 3u) { + continue; + } + let o = to_fixed(dt[k]); + let signed = smul(2 * o - FIXED_ONE, weight); + var v: i32; + if add { + v = c[k] + signed; + } else { + v = c[k] + smul(signed, c[k]); + } + c[k] = fixed_clamp(v); + } + } + if (tex_slots[ti].flags & TEX_POST) != 0u { + c = tex_post(ti, c); + } + return c; +} + +// Level `li` at pixel i's base position. +fn level_sample(ti: u32, li: u32, l0: TexLevel, cls: u32, p: array, 3>, i: u32, linear: bool, wraps: vec3, three: bool) -> vec4 { + let lv = tex_slots[ti].levels[li]; + let n = select(select(2u, 1u, tex_slots[ti].axes == 1u), 3u, three); + var at = vec3(0); + var w = vec3(0u); + let shifts = vec3(l0.wl - lv.wl, l0.hl - lv.hl, l0.dl - lv.dl); + let sizes = vec3(lv.wl, lv.hl, lv.dl); + for (var a = 0u; a < n; a++) { + let r = pos(p[a][i] >> shifts[a], sizes[a], wraps[a], linear); + at[a] = r.at; + w[a] = r.w; + } + return filter_level(ti, lv, cls, tex_slots[ti].axes, at, w, linear, wraps, (tex_slots[ti].flags & TEX_STORED_BORDER) != 0u); +} + +// fixed::mul: two colours' product, 4096.0 times 4096.0 is 4096.0. +fn fixed_mul(a: i32, b: i32) -> i32 { + return i32(((u32(a) >> 8u) * (u32(b) >> 8u)) >> 8u); +} + +fn fixed_lerp(a: i32, b: i32, f: i32) -> i32 { + return a + smul(b - a, f); +} + +// Which of a texel's components the environment takes. +fn takes(cls: u32, k: u32) -> bool { + let alpha = k == 3u; + return !((!alpha && cls == CLASS_ALPHA) || (alpha && (cls == CLASS_RGB || cls == CLASS_LUMINANCE))); +} + +// ShaderState::apply12x4: the texel by the environment into the colour, +// in 12.12. +fn tex_env(ti: u32, texel: vec4, rgba_in: vec4) -> vec4 { + let cls = tex_class(ti); + var rgba = rgba_in; + for (var k = 0u; k < 4u; k++) { + if !takes(cls, k) { + continue; + } + let alpha = k == 3u; + let tx = texel[k]; + let c = rgba[k]; + var r: i32; + switch tex_slots[ti].env { + case ENV_MODULATE: { r = fixed_mul(c, tx); } + case ENV_ADD: { + if alpha && cls != CLASS_INTENSITY { + r = fixed_mul(c, tx); + } else { + r = fixed_clamp(c + tx); + } + } + case ENV_DECAL: { + if alpha { + r = c; + } else if cls == CLASS_RGB { + r = tx; + } else { + r = fixed_lerp(c, tx, texel.w); + } + } + case ENV_BLEND: { + if alpha && cls != CLASS_INTENSITY { + r = fixed_mul(c, tx); + } else { + r = fixed_lerp(c, tex_slots[ti].env_colour[k], tx); + } + } + default: { r = tx; } + } + rgba[k] = r; + } + return rgba; +} diff --git a/src/dev/odyssey/wgpu/shaders/views.wgsl b/src/dev/odyssey/wgpu/shaders/views.wgsl new file mode 100644 index 000000000..45cc35608 --- /dev/null +++ b/src/dev/odyssey/wgpu/shaders/views.wgsl @@ -0,0 +1,235 @@ +// A field of a surface in board memory (view.rs): the surface's base, its +// pitch in 16 x 16 tiles, its pixel size and the field's byte inside the +// pixel, and the field's kind. Tiles are row-major, pixels row-major +// inside a tile. A view arrives packed in four words (wgpu/mod.rs +// `View::pack`). + +#include "vram.wgsl" + +const KIND_NONE: u32 = 0u; +const KIND_RGBA8: u32 = 1u; +const KIND_Z24: u32 = 2u; +const KIND_BYTE: u32 = 3u; +// A texel field (field.rs): the byte count in bits 15:8 of the kind word +// and FIELD_FMT in 23:16. +const KIND_TEXEL: u32 = 4u; +// Four 12-bit lanes in the field's first 6 bytes. +const KIND_LANES: u32 = 5u; + +const FIELD_INDEX: u32 = 0u; +const FIELD_ONE: u32 = 2u; +const FIELD_TWO: u32 = 3u; +const FIELD_FOUR: u32 = 5u; +const FIELD_PACKED: u32 = 6u; + +struct View { + base: u32, + // pitch in bits 15:0, log2 of the pixel size in 23:16, the field's + // byte in 31:24. + geom: u32, + kind: u32, + // width in bits 15:0, height in 31:16. + dims: u32, +} + +fn view_width(v: View) -> u32 { + return v.dims & 0xffffu; +} + +fn view_height(v: View) -> u32 { + return v.dims >> 16u; +} + +// Whether (x, y) is in the view. +fn view_has(v: View, x: i32, y: i32) -> bool { + return v.kind != KIND_NONE && x >= 0 && y >= 0 && u32(x) < view_width(v) && u32(y) < view_height(v); +} + +// The byte address of the field at (x, y), unchecked. +fn view_addr(v: View, x: i32, y: i32) -> u32 { + let ux = u32(x); + let uy = u32(y); + let pitch = v.geom & 0xffffu; + let log2 = (v.geom >> 16u) & 0xffu; + let at = v.geom >> 24u; + let tile = (uy >> 4u) * pitch + (ux >> 4u); + return v.base + (((tile << 8u) | ((uy & 15u) << 4u) | (ux & 15u)) << log2) + at; +} + +// Bytes the field's value is held in. +fn view_len(v: View) -> u32 { + var r: u32; + switch v.kind & 0xffu { + case KIND_RGBA8: { r = 4u; } + case KIND_Z24: { r = 3u; } + case KIND_BYTE: { r = 1u; } + case KIND_TEXEL: { r = (v.kind >> 8u) & 0xffu; } + case KIND_LANES: { r = 6u; } + default: { r = 0u; } + } + return r; +} + +// Texel codecs (field.rs texel_pack / texel_unpack): a 0xRRGGBBAA word +// to and from the field's value, the field's first byte the value's top. + +fn narrow(c: u32, n: u32) -> u32 { + return c >> (8u - n); +} + +// An n-bit value widened to 8 bits by repeating its bits. +fn widen(v: u32, n: u32) -> u32 { + var out = 0u; + var have = 0u; + while have < 8u { + out = (out << n) | v; + have += n; + } + return out >> (have - 8u); +} + +fn chan(word: u32, k: u32) -> u32 { + return (word >> (24u - 8u * k)) & 0xffu; +} + +fn word4(c: vec4) -> u32 { + return (c.x << 24u) | (c.y << 16u) | (c.z << 8u) | c.w; +} + +fn texel_pack(word: u32, bytes: u32, field: u32) -> u32 { + let c0 = chan(word, 0u); + let c1 = chan(word, 1u); + let c2 = chan(word, 2u); + let c3 = chan(word, 3u); + if bytes == 2u { + switch field { + case FIELD_INDEX: { return word & 0xffffu; } + case FIELD_ONE: { return (c0 << 8u) | c0; } + case FIELD_TWO: { return (c0 << 8u) | c3; } + case FIELD_FOUR: { return (narrow(c0, 4u) << 12u) | (narrow(c1, 4u) << 8u) | (narrow(c2, 4u) << 4u) | narrow(c3, 4u); } + case FIELD_PACKED: { return (narrow(c0, 5u) << 11u) | (narrow(c1, 5u) << 6u) | (narrow(c2, 5u) << 1u) | narrow(c3, 1u); } + default: {} + } + } else if bytes == 4u { + switch field { + case FIELD_INDEX: { return word; } + case FIELD_TWO: { return (((c0 << 8u) | c0) << 16u) | (c3 << 8u) | c3; } + case FIELD_PACKED: { return (((c0 << 2u) | (c0 >> 6u)) << 22u) | (((c1 << 2u) | (c1 >> 6u)) << 12u) | (((c2 << 2u) | (c2 >> 6u)) << 2u) | narrow(c3, 2u); } + default: {} + } + } + // The word's top bytes. + return word >> (32u - 8u * min(bytes, 4u)); +} + +fn texel_unpack(w: u32, bytes: u32, field: u32) -> u32 { + if bytes == 2u { + switch field { + case FIELD_INDEX: { return w; } + case FIELD_ONE: { + let v = (w >> 8u) & 0xffu; + return word4(vec4(v, v, v, v)); + } + case FIELD_TWO: { + let b0 = (w >> 8u) & 0xffu; + return word4(vec4(b0, b0, b0, w & 0xffu)); + } + case FIELD_FOUR: { return word4(vec4(widen((w >> 12u) & 0xfu, 4u), widen((w >> 8u) & 0xfu, 4u), widen((w >> 4u) & 0xfu, 4u), widen(w & 0xfu, 4u))); } + case FIELD_PACKED: { return word4(vec4(widen((w >> 11u) & 0x1fu, 5u), widen((w >> 6u) & 0x1fu, 5u), widen((w >> 1u) & 0x1fu, 5u), select(0u, 0xffu, (w & 1u) != 0u))); } + default: {} + } + } else if bytes == 4u { + switch field { + case FIELD_INDEX: { return w; } + case FIELD_TWO: { + let b0 = (w >> 24u) & 0xffu; + return word4(vec4(b0, b0, b0, (w >> 8u) & 0xffu)); + } + case FIELD_PACKED: { return word4(vec4((w >> 24u) & 0xffu, (w >> 14u) & 0xffu, (w >> 4u) & 0xffu, widen(w & 3u, 2u))); } + default: {} + } + } + // The value as the word's top bytes. + return w << (32u - 8u * min(bytes, 4u)); +} + +// Four 12-bit lanes (48 bits, R highest) from a word: each byte at the top +// of its lane, its top bits repeated below; and back. +fn lanes_of(word: u32) -> vec2 { + var l: vec4; + for (var k = 0u; k < 4u; k++) { + let c = chan(word, k); + l[k] = (c << 4u) | (c >> 4u); + } + // bits 47:36 l0, 35:24 l1, 23:12 l2, 11:0 l3. + let lo = (l.y << 24u) | (l.z << 12u) | l.w; + let hi = (l.x << 4u) | (l.y >> 8u); + return vec2(lo, hi); +} + +fn word_of_lanes(v: vec2) -> u32 { + let l0 = (v.y >> 4u) & 0xfffu; + let l1 = ((v.y << 8u) | (v.x >> 24u)) & 0xfffu; + let l2 = (v.x >> 12u) & 0xfffu; + let l3 = v.x & 0xfffu; + return word4(vec4((l0 >> 4u) & 0xffu, (l1 >> 4u) & 0xffu, (l2 >> 4u) & 0xffu, (l3 >> 4u) & 0xffu)); +} + +// The field's word at (x, y), when it is in the view; `ok` says so. +struct Loaded { + ok: bool, + v: u32, +} + +// A field's word (see the module) from its value at byte address a, for +// a field of kind `kind` (the View kind word). +fn load_kind(a: u32, kind: u32) -> u32 { + let k = kind & 0xffu; + if k == KIND_LANES { + return word_of_lanes(vram_load64(a, 6u)); + } + var n: u32; + switch k { + case KIND_RGBA8: { n = 4u; } + case KIND_Z24: { n = 3u; } + case KIND_BYTE: { n = 1u; } + case KIND_TEXEL: { n = (kind >> 8u) & 0xffu; } + default: { n = 0u; } + } + if n == 0u { + return 0u; + } + let raw = vram_load(a, n); + if k == KIND_TEXEL { + return texel_unpack(raw, n, (kind >> 16u) & 0xffu); + } + return raw; +} + +fn view_load(v: View, x: i32, y: i32) -> Loaded { + if !view_has(v, x, y) { + return Loaded(false, 0u); + } + return Loaded(true, load_kind(view_addr(v, x, y), v.kind)); +} + +fn view_store(v: View, x: i32, y: i32, w: u32) { + if !view_has(v, x, y) { + return; + } + let a = view_addr(v, x, y); + let kind = v.kind & 0xffu; + if kind == KIND_LANES { + vram_store64(a, 6u, lanes_of(w)); + return; + } + var val = w; + if kind == KIND_Z24 { + val = w & 0xffffffu; + } else if kind == KIND_BYTE { + val = w & 0xffu; + } else if kind == KIND_TEXEL { + val = texel_pack(w, (v.kind >> 8u) & 0xffu, (v.kind >> 16u) & 0xffu); + } + vram_store(a, view_len(v), val); +} diff --git a/src/dev/odyssey/wgpu/shaders/vpro.wgsl b/src/dev/odyssey/wgpu/shaders/vpro.wgsl new file mode 100644 index 000000000..ffb543904 --- /dev/null +++ b/src/dev/odyssey/wgpu/shaders/vpro.wgsl @@ -0,0 +1,388 @@ +// The VPro raster unit's hooks for the common raster pass: the quad ring's +// three commands (pipe/ring.rs) as records. A triangle record is the +// walk's setup (raster::Triangle) and the shading state of its record +// (setup::Tri); a fragment record is a quad the CPU shaded itself; a +// rectangle record is a clear's. The arithmetic follows pipe/raster.rs, +// pipe/interp.rs, pipe/fixed.rs and pipe/backend.rs operation for +// operation; where a product feeds a sum, it is multiplied by a one the +// compiler cannot see through (`nc`), so no driver fuses it into a +// multiply-add the CPU does not do, and quotients go through +// `div_exact` (src/wgpu/shaders/float.wgsl), since a GPU's division need +// not be correctly rounded. +// +// Record layout: src/dev/odyssey/wgpu/mod.rs `Rec` writes it. + +#include "i64.wgsl" +#include "float.wgsl" +#include "backend.wgsl" +#include "texture.wgsl" + +const REC_TRI: u32 = 1u; +const REC_FRAG: u32 = 2u; +const REC_RECT: u32 = 3u; +const REC_BLIT: u32 = 4u; + +const REC_WORDS: u32 = 84u; + +struct Rec { + kind: u32, + slot: u32, + flags: u32, + // 1.0 as bits: the one that stops contraction. + one: u32, + w: array, +} + +@group(1) @binding(0) var recs: array; + +// Triangle flags. +const TRI_SMOOTH: u32 = 1u; +const TRI_TEXTURED: u32 = 2u; +const TRI_SPECULAR: u32 = 4u; + +// Triangle word offsets. +const T_E0: u32 = 0u; // 3 x I64: e0, e1, e2 at the centre of plane pixel (0, 0) +const T_DX: u32 = 6u; // 3 x I64 +const T_DY: u32 = 12u; // 3 x I64 +const T_NEED: u32 = 18u; // bit k: edge k needs 1 (not owning) +const T_INV_AREA: u32 = 19u; +const T_Z: u32 = 20u; // z0, z1 - z0, z2 - z0 +const T_D: u32 = 23u; // d0, d1 - d0, d2 - d0 +const T_IW: u32 = 26u; // 1/w at the vertices +const T_SLACK: u32 = 29u; // depth slack, near, far - near +const T_RGBA12: u32 = 32u; // 3 x 4 in 12.12 +const T_FLAT12: u32 = 44u; // 4 +const T_STEPS: u32 = 48u; // polygon offset, clamped to +-2^25 +const T_BOX: u32 = 49u; // x0, y0, x1, y1 in plane pixels, inclusive +const T_ORIGIN: u32 = 53u; // ox, oy +const T_ALPHA: u32 = 55u; // 0x41e: the alpha test's word +const T_MODE: u32 = 56u; // 0x404: its enable +const T_TEX: u32 = 57u; // S/W, T/W, R/W, Q/W at the three vertices +const T_TEXSLOT: u32 = 69u; +const T_SPEC12: u32 = 70u; // 3 x 3 in 12.12 +const T_FLAT_SPEC: u32 = 79u; + +// Fragment word offsets. +const F_X: u32 = 0u; +const F_Y: u32 = 1u; +const F_COV: u32 = 2u; +const F_ZQ: u32 = 3u; // 4 +const F_RGBA: u32 = 7u; // 4 pixels x 2 words: r | g << 16, b | a << 16 + +// Rectangle word offsets. +const R_BOX: u32 = 0u; // x0, y0, x1, y1 +const R_COLOUR: u32 = 4u; +const R_COLOUR_MASK: u32 = 5u; +const R_DEPTH: u32 = 6u; +const R_STENCIL: u32 = 7u; // value | mask << 8 +const R_ID: u32 = 8u; // value | mask << 8 + +// Blit word offsets: a row run of pixels from the stage buffer into one +// of the slot's views, under a mask, through the window's tests or not. +const B_BOX: u32 = 0u; // x0, y0, x1, y1 (one row) +const B_OFF: u32 = 4u; // the first pixel's word in the stage +const B_MASK: u32 = 5u; // the bits stored (all ones: a plain store) +const B_VIEW: u32 = 6u; // the slot's view index +const B_OP: u32 = 7u; // 0, or 0x100 | the logical operation's code +const B_AND: u32 = 8u; // the operation's result is and-ed and or-ed +const B_OR: u32 = 9u; +const B_OWN: u32 = 10u; // the record's own view, with BLIT_OWN_VIEW +const BLIT_WINDOW: u32 = 1u; +const BLIT_OWN_VIEW: u32 = 2u; + +// Pixels the blits of the batch store (wgpu/mod.rs `GpuRaster::stage`). +@group(2) @binding(2) var stage: array; + +const MODE_ALPHA_TEST: u32 = 1u << 14u; +const MODE_FLAT_SHADE: u32 = 1u << 12u; + +const W_ONE: i32 = 1 << 15; +const FIXED_MAX: i32 = (1 << 24) - 1; +const DEPTH_2S: f32 = 8589901824.0; +const DEPTH_MIN_VALUE: f32 = 1.0 / 2147483648.0; + +var nc: f32; + +// Record words, read in place: a record or slot copied by value sits in +// each lane's private memory, and enough of that gave wrong results on +// RADV (clanker/wgpu/read-storage-in-place.md). +fn ru(pi: u32, i: u32) -> u32 { + return recs[pi].w[i]; +} + +fn rf(pi: u32, i: u32) -> f32 { + return bitcast(recs[pi].w[i]); +} + +fn ri(pi: u32, i: u32) -> i32 { + return bitcast(recs[pi].w[i]); +} + +fn r64(pi: u32, i: u32) -> I64 { + return I64(recs[pi].w[i], recs[pi].w[i + 1u]); +} + +// x * 1.0 through a value the compiler cannot fold: the product is +// rounded on its own before it joins a sum. +fn p(x: f32) -> f32 { + return x * nc; +} + +// Rust's `as i32`: truncation, saturating, NaN to 0. +fn sat_i32(x: f32) -> i32 { + let c = clamp(x, -2147483648.0, 2147483520.0); + return select(i32(c), 0, x != x); +} + +// word_for_fraction32: the 24-bit depth word of a window depth. +fn depth_word(z_in: f32) -> u32 { + let z = select(select(0.0, z_in, z_in > 0.0), 1.0, z_in > 1.0); + let v = z * DEPTH_2S; + let bits = bitcast(v); + let exp = (((bits >> 23u) & 0xffu) + 31u) - 127u; + let mantissa = (bits >> 5u) & 0x3ffffu; + let word = ((exp << 18u) | mantissa) & DEPTH24_MAX; + return select(0u, word, v >= DEPTH_MIN_VALUE); +} + +// fixed::interpolate: weights p (0.15) over vertex values c (12.12). +fn interpolate(p0: i32, p1: i32, p2: i32, c0: i32, c1: i32, c2: i32) -> i32 { + let s = (c0 >> 8u) * p0 + (c1 >> 8u) * p1 + (c2 >> 8u) * p2; + return s >> 7u; +} + +fn colour12(c: i32) -> u32 { + return u32(clamp(c, 0, FIXED_MAX) >> 12u); +} + +fn tri(pi: u32, si: u32, px: i32, py: i32) { + // The walk's bounds, every clip already folded in. + if px < ri(pi, T_BOX) || px > ri(pi, T_BOX + 2u) || py < ri(pi, T_BOX + 1u) || py > ri(pi, T_BOX + 3u) { + return; + } + // The three edge values at the pixel centre, exact. + var e: array; + let need = ru(pi, T_NEED); + for (var k = 0u; k < 3u; k++) { + var v = r64(pi, T_E0 + 2u * k); + v = i64_add(v, i64_mul_u16(r64(pi, T_DX + 2u * k), u32(px))); + v = i64_add(v, i64_mul_u16(r64(pi, T_DY + 2u * k), u32(py))); + if !i64_ge_need(v, (need >> k) & 1u) { + return; + } + e[k] = v; + } + if !owned(si, px, py) { + return; + } + let inv_area = rf(pi, T_INV_AREA); + let l1 = p(i64_to_f32(e[1]) * inv_area); + let l2 = p(i64_to_f32(e[2]) * inv_area); + // The pixel's quad (2x2, even window coordinates) and its index in + // it: the texture unit reads the whole quad's positions. + let wx = px - ri(pi, T_ORIGIN); + let wy = py - ri(pi, T_ORIGIN + 1u); + let qi = u32(wx & 1) | (u32(wy & 1) << 1u); + + // Triangle::depth4: the planes, the ratio, the slack clamps, the range + // and the word with the polygon offset. + let zz = (rf(pi, T_Z) + p(l1 * rf(pi, T_Z + 1u))) + p(l2 * rf(pi, T_Z + 2u)); + let dd = (rf(pi, T_D) + p(l1 * rf(pi, T_D + 1u))) + p(l2 * rf(pi, T_D + 2u)); + var z = div_exact(zz, dd); + let slack = rf(pi, T_SLACK); + if z < 0.0 && z >= -slack { + z = 0.0; + } else if z > 1.0 && z <= 1.0 + slack { + z = 1.0; + } + if !(z >= 0.0 && z <= 1.0) { + return; + } + var zw = rf(pi, T_SLACK + 1u) + p(rf(pi, T_SLACK + 2u) * z); + zw = select(select(0.0, zw, zw > 0.0), 1.0, zw >= 1.0); + let zq = u32(clamp(i32(depth_word(zw)) + ri(pi, T_STEPS), 0, i32(DEPTH24_MAX))); + + // The polygon stipple, in window pixels. + var background = false; + if (slots[si].flags & SLOT_STIPPLE) != 0u { + let wx = u32(px - ri(pi, T_ORIGIN)); + let wy = u32(py - ri(pi, T_ORIGIN + 1u)); + if (slots[si].pattern[wy & 31u] & (1u << (wx & 31u))) == 0u { + if (slots[si].flags & SLOT_BACKGROUND) == 0u { + return; + } + background = true; + } + } + + // The colour: the provoking vertex's, or interpolated by the + // perspective-correct weights (interp::weights4); the separate + // specular likewise. + var c: vec4; + var spec = vec3(ri(pi, T_FLAT_SPEC), ri(pi, T_FLAT_SPEC + 1u), ri(pi, T_FLAT_SPEC + 2u)); + let specular = (recs[pi].flags & TRI_SPECULAR) != 0u; + if background { + c = slots[si].bg12; + spec = vec3(0); + } else if (recs[pi].flags & TRI_SMOOTH) != 0u { + let iw0 = rf(pi, T_IW); + let w1 = p(l1 * rf(pi, T_IW + 1u)); + let w2 = p(l2 * rf(pi, T_IW + 2u)); + let sum = (p(((1.0 - l1) - l2) * iw0) + w1) + w2; + if sum == 0.0 || !(abs(sum) < bitcast(0x7f800000u)) { + return; + } + let rr = div_exact(f32(W_ONE), sum); + let p1 = clamp(sat_i32(p(w1 * rr) + 0.5), 0, W_ONE); + let rest = W_ONE - p1; + let p2 = clamp(sat_i32(p(w2 * rr) + 0.5), 0, rest); + let p0 = rest - p2; + for (var k = 0u; k < 4u; k++) { + c[k] = interpolate(p0, p1, p2, ri(pi, T_RGBA12 + k), ri(pi, T_RGBA12 + 4u + k), ri(pi, T_RGBA12 + 8u + k)); + } + if specular { + for (var k = 0u; k < 3u; k++) { + spec[k] = interpolate(p0, p1, p2, ri(pi, T_SPEC12 + k), ri(pi, T_SPEC12 + 3u + k), ri(pi, T_SPEC12 + 6u + k)); + } + } + } else { + c = vec4(ri(pi, T_FLAT12), ri(pi, T_FLAT12 + 1u), ri(pi, T_FLAT12 + 2u), ri(pi, T_FLAT12 + 3u)); + } + + // The texture: the quad's coordinates from its four pixels' + // barycentrics (interp::TriTex::quad), sampled for this pixel, then + // the environment. + if (recs[pi].flags & TRI_TEXTURED) != 0u { + var tq: TexQuad; + for (var j = 0u; j < 4u; j++) { + let dx = i32(j & 1u) - i32(qi & 1u); + let dy = i32(j >> 1u) - i32(qi >> 1u); + var l1j = l1; + var l2j = l2; + if j != qi { + var e1 = e[1]; + var e2 = e[2]; + if dx > 0 { + e1 = i64_add(e1, r64(pi, T_DX + 2u)); + e2 = i64_add(e2, r64(pi, T_DX + 4u)); + } else if dx < 0 { + e1 = i64_add(e1, i64_neg(r64(pi, T_DX + 2u))); + e2 = i64_add(e2, i64_neg(r64(pi, T_DX + 4u))); + } + if dy > 0 { + e1 = i64_add(e1, r64(pi, T_DY + 2u)); + e2 = i64_add(e2, r64(pi, T_DY + 4u)); + } else if dy < 0 { + e1 = i64_add(e1, i64_neg(r64(pi, T_DY + 2u))); + e2 = i64_add(e2, i64_neg(r64(pi, T_DY + 4u))); + } + l1j = p(i64_to_f32(e1) * inv_area); + l2j = p(i64_to_f32(e2) * inv_area); + } + for (var a = 0u; a < 4u; a++) { + let v0 = rf(pi, T_TEX + 3u * a); + let v1 = rf(pi, T_TEX + 3u * a + 1u); + let v2 = rf(pi, T_TEX + 3u * a + 2u); + tq.c[a][j] = (v0 + p(l1j * (v1 - v0))) + p(l2j * (v2 - v0)); + } + } + let ti = ru(pi, T_TEXSLOT); + let texel = sample_quad(ti, tq, qi); + c = tex_env(ti, texel, c); + } + if specular { + for (var k = 0u; k < 3u; k++) { + c[k] = fixed_clamp(c[k] + spec[k]); + } + } + + // The alpha test on the 12-bit alpha. + if (ru(pi, T_MODE) & MODE_ALPHA_TEST) != 0u { + let a = ru(pi, T_ALPHA); + if !compare((a >> 12u) & 7u, colour12(c.w), a & 0xfffu) { + return; + } + } + let src = vec4(colour12(c.x), colour12(c.y), colour12(c.z), colour12(c.w)); + fragment_q(backend_of(si), si, px, py, zq, src); +} + +fn frag(pi: u32, si: u32, px: i32, py: i32) { + let x = ri(pi, F_X); + let y = ri(pi, F_Y); + let dx = px - x; + let dy = py - y; + if dx < 0 || dx > 1 || dy < 0 || dy > 1 { + return; + } + let i = u32(dx) + 2u * u32(dy); + if (ru(pi, F_COV) & (1u << i)) == 0u { + return; + } + let a = ru(pi, F_RGBA + 2u * i); + let b = ru(pi, F_RGBA + 2u * i + 1u); + let src = vec4(a & 0xffffu, a >> 16u, b & 0xffffu, b >> 16u); + fragment_q(backend_of(si), si, px, py, ru(pi, F_ZQ + i), src); +} + +fn rect(pi: u32, si: u32, px: i32, py: i32) { + if px < ri(pi, R_BOX) || px > ri(pi, R_BOX + 2u) || py < ri(pi, R_BOX + 1u) || py > ri(pi, R_BOX + 3u) { + return; + } + clear_pixel(si, recs[pi].flags, px, py, ru(pi, R_COLOUR), ru(pi, R_COLOUR_MASK), ru(pi, R_DEPTH), ru(pi, R_STENCIL), ru(pi, R_ID)); +} + +fn blit(pi: u32, si: u32, px: i32, py: i32) { + if px < ri(pi, B_BOX) || px > ri(pi, B_BOX + 2u) || py < ri(pi, B_BOX + 1u) || py > ri(pi, B_BOX + 3u) { + return; + } + if (recs[pi].flags & BLIT_WINDOW) != 0u && !owned(si, px, py) { + return; + } + var v: View; + if (recs[pi].flags & BLIT_OWN_VIEW) != 0u { + v = View(ru(pi, B_OWN), ru(pi, B_OWN + 1u), ru(pi, B_OWN + 2u), ru(pi, B_OWN + 3u)); + } else { + v = slots[si].views[ru(pi, B_VIEW)]; + } + let w = stage[ru(pi, B_OFF) + u32(px - ri(pi, B_BOX))]; + let mask = ru(pi, B_MASK); + let op = ru(pi, B_OP); + if op == 0u && mask == 0xffffffffu { + view_store(v, px, py, w); + return; + } + // The destination is read once: the operation and a byte plane's mask + // both see the same old word (Board::paint_integer, buffer_store). + let old = view_load(v, px, py); + if !old.ok { + return; + } + var val = w; + if op != 0u { + val = (logic_op(op & 0xfu, w, old.v) & ru(pi, B_AND)) | ru(pi, B_OR); + } + if mask == 0xffffffffu { + view_store(v, px, py, val); + } else { + view_store(v, px, py, (old.v & ~mask) | (val & mask)); + } +} + +fn arch_tile_begin(tile: u32, px: i32, py: i32, lane: u32) { +} + +fn arch_prim(pi: u32, px: i32, py: i32, lane: u32) { + let si = recs[pi].slot; + nc = bitcast(recs[pi].one); + switch recs[pi].kind { + case REC_TRI: { tri(pi, si, px, py); } + case REC_FRAG: { frag(pi, si, px, py); } + case REC_RECT: { rect(pi, si, px, py); } + case REC_BLIT: { blit(pi, si, px, py); } + default: {} + } +} + +fn arch_tile_end(tile: u32, px: i32, py: i32, lane: u32) { +} diff --git a/src/dev/odyssey/wgpu/shaders/vram.wgsl b/src/dev/odyssey/wgpu/shaders/vram.wgsl new file mode 100644 index 000000000..297f530e8 --- /dev/null +++ b/src/dev/odyssey/wgpu/shaders/vram.wgsl @@ -0,0 +1,144 @@ +// Board memory as the model holds it (vram.rs): 64-bit words, a field of +// n bytes at byte address a being the bits of its word that a 64-bit load +// of guest memory gives those addresses, the lowest address highest. The +// GPU buffer is the model's word array as bytes on a little-endian host, +// so word i is u32 pair (2i, 2i + 1), the low half first. +// +// Every access is atomic: two pixels can share a word (the window-id +// buffer's bytes, the overlay), and two lanes of the tile's workgroup +// then read-modify-write the same word. Relaxed atomics on one location +// are program-ordered within a lane, which is all the order the kernels +// need; the partial store is an AND then an OR, each lane touching only +// its own bits. VRAM_WORD_MASK (words less one) arrives as a define. + +@group(2) @binding(0) var vram: array>; + +// The u32 index of the low half of the word holding byte address a. +fn vram_word(a: u32) -> u32 { + return ((a >> 3u) & VRAM_WORD_MASK) * 2u; +} + +fn vram_mask(n: u32) -> u32 { + return select((1u << (8u * n)) - 1u, 0xffffffffu, n >= 4u); +} + +// The n (1 to 4) bytes at byte address a, which lie in one word. +fn vram_load(a: u32, n: u32) -> u32 { + let w = vram_word(a); + let shift = 64u - 8u * ((a & 7u) + n); + let m = vram_mask(n); + if shift >= 32u { + return (atomicLoad(&vram[w + 1u]) >> (shift - 32u)) & m; + } + let lo = atomicLoad(&vram[w]); + if shift + 8u * n <= 32u { + return (lo >> shift) & m; + } + let hi = atomicLoad(&vram[w + 1u]); + return ((lo >> shift) | (hi << (32u - shift))) & m; +} + +// Store v as the n (1 to 4) bytes at byte address a. +fn vram_store(a: u32, n: u32, v: u32) { + let w = vram_word(a); + let shift = 64u - 8u * ((a & 7u) + n); + let m = vram_mask(n); + let v_m = v & m; + if shift >= 32u { + let s = shift - 32u; + if n == 4u { + atomicStore(&vram[w + 1u], v_m); + } else { + atomicAnd(&vram[w + 1u], ~(m << s)); + atomicOr(&vram[w + 1u], v_m << s); + } + return; + } + if shift + 8u * n <= 32u { + if n == 4u { + atomicStore(&vram[w], v_m); + } else { + atomicAnd(&vram[w], ~(m << shift)); + atomicOr(&vram[w], v_m << shift); + } + return; + } + // Straddling the halves: the value's low 32 - shift bits go to the + // top of the low half, the rest to the bottom of the high half. + let lo_bits = 32u - shift; + atomicAnd(&vram[w], ~(m << shift)); + atomicOr(&vram[w], v_m << shift); + atomicAnd(&vram[w + 1u], ~(m >> lo_bits)); + atomicOr(&vram[w + 1u], v_m >> lo_bits); +} + +// The n (1 to 8) bytes at byte address a as a 64-bit value, low half +// first: the lane fields (6 bytes) need it. +fn vram_load64(a: u32, n: u32) -> vec2 { + let w = vram_word(a); + let shift = 64u - 8u * ((a & 7u) + n); + let lo = atomicLoad(&vram[w]); + let hi = atomicLoad(&vram[w + 1u]); + var r: vec2; + if shift >= 32u { + r = vec2(hi >> (shift - 32u), 0u); + } else if shift == 0u { + r = vec2(lo, hi); + } else { + r = vec2((lo >> shift) | (hi << (32u - shift)), hi >> shift); + } + if n < 4u { + r = vec2(r.x & vram_mask(n), 0u); + } else if n == 4u { + r.y = 0u; + } else { + r.y &= vram_mask(n - 4u); + } + return r; +} + +// Store the 64-bit value v (low half first) as the n (1 to 8) bytes at +// byte address a. +fn vram_store64(a: u32, n: u32, v: vec2) { + let w = vram_word(a); + let shift = 64u - 8u * ((a & 7u) + n); + // The value's mask, 64 bits, before the shift. + var m: vec2; + if n < 4u { + m = vec2(vram_mask(n), 0u); + } else if n == 4u { + m = vec2(0xffffffffu, 0u); + } else { + m = vec2(0xffffffffu, vram_mask(n - 4u)); + } + let vm = v & m; + // Shift both left by `shift`. + var sm: vec2; + var sv: vec2; + if shift >= 32u { + sm = vec2(0u, m.x << (shift - 32u)); + sv = vec2(0u, vm.x << (shift - 32u)); + } else if shift == 0u { + sm = m; + sv = vm; + } else { + sm = vec2(m.x << shift, (m.y << shift) | (m.x >> (32u - shift))); + sv = vec2(vm.x << shift, (vm.y << shift) | (vm.x >> (32u - shift))); + } + if sm.x != 0u { + if sm.x == 0xffffffffu { + atomicStore(&vram[w], sv.x); + } else { + atomicAnd(&vram[w], ~sm.x); + atomicOr(&vram[w], sv.x); + } + } + if sm.y != 0u { + if sm.y == 0xffffffffu { + atomicStore(&vram[w + 1u], sv.y); + } else { + atomicAnd(&vram[w + 1u], ~sm.y); + atomicOr(&vram[w + 1u], sv.y); + } + } +} diff --git a/src/dev/odyssey/wgpu/tests.rs b/src/dev/odyssey/wgpu/tests.rs new file mode 100644 index 000000000..7a9a11adb --- /dev/null +++ b/src/dev/odyssey/wgpu/tests.rs @@ -0,0 +1,892 @@ +//! The GPU raster unit against the CPU pipeline, word for word: the same +//! triangles, shaded quads and clears through `pipe::raster`, +//! `pipe::interp`, `pipe::fixed` and `pipe::backend` into a `Vram`, and +//! through `GpuRaster` into its GPU memory, then both memories compared. +//! They need a GPU (or lavapipe) and skip without one. + +use std::sync::Arc; + +use super::super::clip::ClipRect; +use super::super::consts::{gl_cmd, mode, write, ODSY_DEPTH24_MAX}; +use super::super::depth_word::word_for_fraction; +use super::super::pipe::backend::{colour12, BackendState, ClearDraw, DestMem}; +use super::super::pipe::aux_targets; +use super::super::pipe::fixed; +use super::super::pipe::interp::{self, TriTex}; +use super::super::pipe::raster::{triangle_quads, walk_bounds, Quad, RasterState, RasterVertex, Triangle}; +use super::super::pipe::ring::BackQuad; +use super::super::pipe::setup::Tri; +use super::super::pipe::shader::ShaderState; +use super::super::pipe::state::{BackendWords, Compare, RasterWords, ShaderWords}; +use super::super::pipe::texture::{Latch, TexRegs, Unit, Wrap}; +use super::super::consts::tex; +use super::super::view::DestViews; +use super::super::vram::Vram; +use super::{GpuRaster, Rec, Slot, TexSlot}; +use crate::wgpu::{Gpu, GpuOptions}; + +/// The test memory: 4 MB, a power of two. +const VRAM_BYTES: u64 = 1 << 22; +/// The plane the batch grid covers. +const GRID: u32 = 512; +/// The RGBA destination: 16-byte pixels, 256 wide, from byte 0x1000 +/// (the pixel mode Board::test_screen() uses, narrowed), and the +/// window-id surface at 2 MB with 2-byte pixels. +const RGBA_MODE: [u32; 4] = [0x100, 16, 0x0040_6d50, 0]; +const WID_BASE: u32 = 0x20000; +const WID_STRIDE: u32 = 16; +/// The overlay as a destination: an index visual of 2-byte pixels over +/// the window-id surface (vram.rs tests' OVERLAY select). +const OVERLAY_MODE: [u32; 4] = [WID_BASE, WID_STRIDE, 0x2008_0080, 0]; + +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 33) as u32 + } + + fn below(&mut self, n: u32) -> u32 { + self.next() % n + } + + fn range(&mut self, lo: i32, hi: i32) -> i32 { + lo + self.below((hi - lo) as u32) as i32 + } + + fn unit(&mut self) -> f64 { + self.next() as f64 / 4294967296.0 + } + + fn f(&mut self, lo: f64, hi: f64) -> f64 { + lo + (hi - lo) * self.unit() + } + + fn chance(&mut self, n: u32) -> bool { + self.below(n) == 0 + } +} + +fn gpu() -> Option> { + match Gpu::new(&GpuOptions::default()) { + Ok(g) => Some(g), + Err(e) => { + eprintln!("skipped: {e}"); + None + } + } +} + +/// Both memories and the destination they share. +struct Pair { + cpu: Vram, + gpu: GpuRaster, + dv: DestViews, + /// The clip list (plane coordinates), the window origin. + raster: RasterWords, + ox: i32, + oy: i32, +} + +impl Pair { + fn new(gpu: &Arc, rng: &mut Lcg, pixel_mode: &[u32; 4]) -> Pair { + let mut cpu = Vram::new(VRAM_BYTES); + for w in cpu.words_mut().iter_mut() { + *w = (rng.next() as u64) << 32 | rng.next() as u64; + } + let mut g = GpuRaster::new(gpu, VRAM_BYTES, GRID, GRID).unwrap_or_else(|e| panic!("{e}")); + g.write_words(0, cpu.words_mut()); + let dv = DestViews::of(pixel_mode, WID_BASE, WID_STRIDE); + let mut raster = RasterWords::default(); + // Two clip rectangles over the 256-pixel plane, a window origin + // near the corner. + raster.clip_kept = 2; + raster.clip_rect[0] = ClipRect { x0: rng.range(0, 40), y0: rng.range(0, 40), x1: rng.range(120, 255), y1: rng.range(120, 255) }; + raster.clip_rect[1] = ClipRect { x0: rng.range(100, 200), y0: rng.range(0, 100), x1: 255, y1: rng.range(150, 255) }; + raster.origin = [rng.range(-30, 30), rng.range(-30, 30)]; + let (ox, oy) = (raster.origin[0], raster.origin[1]); + Pair { cpu, gpu: g, dv, raster, ox, oy } + } + + fn planes(&mut self) -> DestMem { + DestMem::of(self.cpu.ptr(), &self.dv) + } + + /// A raster state over the destination: the clip list, no scissor + /// beyond the plane, the pipeline's 32-bit depth, no pre-test. + fn raster_state(&mut self, words: &RasterWords, depth_range: [f64; 2]) -> RasterState { + let planes = self.planes(); + let mut st = RasterState::of(words, self.dv.colour[0].width, planes.did, depth_range, gl_cmd::DEPTH_SLACK, word_for_fraction, None, false); + st.scissor = Some((-100_000, -100_000, 100_000, 100_000)); + st.depth32 = true; + st + } + + /// Both memories equal, or the first differences. + fn check(&mut self, what: &str) { + let got = self.gpu.read_words().unwrap_or_else(|e| panic!("{what}: {e}")); + let want = self.cpu.words_mut(); + assert_eq!(got.len(), want.len()); + let mut diffs = 0; + let mut msg = String::new(); + let (base, pitch, log2, _) = self.dv.colour[0].parts(); + let fields = [("colour0", self.dv.colour[0]), ("colour1", self.dv.colour[1]), ("depth", self.dv.depth), ("stencil", self.dv.stencil)]; + for (i, (g, w)) in got.iter().zip(want.iter()).enumerate() { + if g != w { + diffs += 1; + if diffs <= 12 { + msg += &format!("\n word {i:#x} (byte {:#x}): gpu {g:#018x} cpu {w:#018x}", i * 8); + // The pixel of the destination it falls in, and its fields. + let a = i as u64 * 8; + if a >= base { + let p = (a - base) >> log2; + let (tile, inner) = (p >> 8, p & 255); + let (x, y) = ((tile % pitch as u64) * 16 + (inner & 15), (tile / pitch as u64) * 16 + (inner >> 4)); + msg += &format!(" = pixel ({x}, {y}) byte {}", a & ((1 << log2) - 1)); + for (name, v) in &fields { + let (_, _, _, at) = v.parts(); + msg += &format!(" {name}@{at}"); + } + } + } + } + } + assert_eq!(diffs, 0, "{what}: {diffs} words differ{msg}"); + } +} + +/// A random triangle in window subpixels around the plane, with depth +/// over w, its divisor and 1/w as a perspective draw has them, and +/// colours. +fn vertex(rng: &mut Lcg, cx: i32, cy: i32, size: i32, ox: i32, oy: i32) -> RasterVertex { + let w = rng.f(0.5, 20.0); + let inv_w = 1.0 / w; + let fraction = rng.f(-0.05, 1.05); + RasterVertex { + x: ((cx + rng.range(-size, size + 1) - ox) as i64) * 256 + rng.range(0, 256) as i64, + y: ((cy + rng.range(-size, size + 1) - oy) as i64) * 256 + rng.range(0, 256) as i64, + z: fraction * inv_w, + inv_w, + depth_w: inv_w, + rgba: [rng.unit(), rng.unit(), rng.unit(), rng.unit()], + spec: [rng.unit(), rng.unit(), rng.unit()], + tex: [rng.f(-2.0, 3.0), rng.f(-2.0, 3.0), rng.f(0.0, 1.0), 1.0], + ..Default::default() + } +} + +fn tri_of(rng: &mut Lcg, st: &RasterState, smooth: bool, ox: i32, oy: i32, size: i32) -> Option { + tri_textured(rng, st, smooth, ox, oy, size, None) +} + +fn tri_textured(rng: &mut Lcg, st: &RasterState, smooth: bool, ox: i32, oy: i32, size: i32, regs: Option<&TexRegs>) -> Option { + let (cx, cy) = (rng.range(-40, 300), rng.range(-40, 300)); + let v = [vertex(rng, cx, cy, size, ox, oy), vertex(rng, cx, cy, size, ox, oy), vertex(rng, cx, cy, size, ox, oy)]; + let mut area = v[0].edge(&v[1], v[2].x, v[2].y); + if area == 0 { + return None; + } + let (front, culled) = st.facing(area); + if culled { + return None; + } + let mut v = v; + if area < 0 { + v.swap(1, 2); + area = -area; + } + let provoking = rng.below(3) as usize; + let flat = v[provoking].rgba; + let tex = match regs { + Some(r) => TriTex::setup(&v.map(|r| super::super::pipe::setup::texture_values(&r)), [r.wrap(0) == Wrap::Repeat, r.wrap(1) == Wrap::Repeat]), + None => TriTex::default(), + }; + Some(Tri { + tri: Triangle::setup(v.map(|r| (r.x, r.y)), v.map(|r| r.z), v.map(|r| r.depth_w), area, front), + live: std::ptr::null(), + rgba12: [0, 1, 2].map(|i| v[i].rgba.map(fixed::from_f64)), + flat12: flat.map(fixed::from_f64), + spec12: [0, 1, 2].map(|i| v[i].spec.map(fixed::from_f64)), + flat_spec12: v[provoking].spec.map(fixed::from_f64), + smooth, + tex, + inv_w: v.map(|r| r.inv_w), + clip: [[0.0; 6]; 3], + }) +} + +/// What a draw shades with, besides the triangle: the alpha test and the +/// polygon stipple. +#[derive(Clone, Copy, Default)] +struct Shade { + alpha: Option<(Compare, u32)>, + stipple: Option<[u32; 32]>, + background: Option<[f64; 4]>, +} + +impl Shade { + fn words(&self) -> (u32, u32) { + match self.alpha { + Some((f, r)) => (mode::ALPHA_TEST, (compare_code(f) << 12) | r), + None => (0, 0), + } + } +} + +fn compare_code(c: Compare) -> u32 { + (0..8).find(|&k| Compare::of(k) == c).unwrap() +} + +/// Draw::shade_quad without fog or user planes: the stipple, the colour +/// and specular weights, the texture unit, then the shader stage. +fn cpu_shade(t: &Tri, q: &Quad, sh: &Shade, unit: Option<&Unit>, shader: &ShaderState) -> BackQuad { + let mut cov = q.cov; + let mut background = 0u8; + for i in 0..4usize { + if cov & (1 << i) == 0 { + continue; + } + let (px, py) = (q.x + (i & 1) as i32, q.y + (i >> 1) as i32); + if let Some(rows) = &sh.stipple { + if rows[((py as u32) & 31) as usize] & (1u32 << ((px as u32) & 31)) == 0 { + if sh.background.is_some() { + background |= 1 << i; + } else { + cov &= !(1 << i); + } + } + } + } + background &= cov; + let mut c = t.flat12.map(|v| [v; 4]); + let mut s = t.flat_spec12.map(|v| [v; 4]); + if t.smooth { + let iw = t.inv_w.map(|w| w as f32); + let (w, kept) = interp::weights4(&q.l1, &q.l2, &iw); + let lanes = |v: [i32; 3]| -> [i32; 4] { std::array::from_fn(|i| fixed::interpolate(&[w[0][i], w[1][i], w[2][i]], v)) }; + cov &= kept | background; + c = std::array::from_fn(|k| lanes([t.rgba12[0][k], t.rgba12[1][k], t.rgba12[2][k]])); + if shader.specular { + s = std::array::from_fn(|k| lanes([t.spec12[0][k], t.spec12[1][k], t.spec12[2][k]])); + } + } + if background != 0 { + let bg = sh.background.expect("an opaque stipple").map(fixed::from_f64); + for i in (0..4).filter(|i| background & (1 << i) != 0) { + for k in 0..4 { + c[k][i] = bg[k]; + } + for sk in s.iter_mut() { + sk[i] = 0; + } + } + } + let mut texel = [[0i32; 4]; 4]; + if let Some(u) = unit { + u.sample_quad(&t.tex.quad(&q.l1, &q.l2), q.cov, &mut texel); + } + cov &= shader.shade12x4(unit.is_some(), &mut c, &s, &texel, |_| 0.0); + let mut out = BackQuad { cov, ..BackQuad::default() }; + for i in (0..4).filter(|i| cov & (1 << i) != 0) { + out.zq[i] = q.zq[i]; + out.rgba[i] = std::array::from_fn(|k| colour12(c[k][i]) as u16); + } + out +} + +/// One draw's state on both sides. +struct DrawState { + words: BackendWords, + index: bool, + to_aux: bool, + depth_range: [f64; 2], + steps: i64, + shade: Shade, + /// The draw's texture registers and their latch, when it textures. + tex: Option>, + specular: bool, +} + +impl DrawState { + fn plain() -> DrawState { + DrawState { + words: BackendWords { write_mask: write::DEPTH | (0xf << write::COLOUR_SHIFT) | (0xff << write::STENCIL_SHIFT), draw_mask: 1, ..Default::default() }, + index: false, + to_aux: false, + depth_range: [0.0, 1.0], + steps: 0, + shade: Shade::default(), + tex: None, + specular: false, + } + } + + fn depth(mut self, func: u32) -> DrawState { + self.words.mode |= mode::DEPTH_TEST; + self.words.depth_func = func; + self + } +} + +/// The codes of the blend factors the board knows. +const FACTOR_CODES: [u32; 13] = [0x00, 0x08, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x40]; + +fn random_state(rng: &mut Lcg) -> DrawState { + let mut d = DrawState::plain(); + if rng.chance(2) { + d = d.depth(rng.below(8)); + if rng.chance(4) { + d.words.write_mask &= !write::DEPTH; + } + } + if rng.chance(3) { + d.words.mode |= mode::BLEND; + let (src, dst) = (FACTOR_CODES[rng.below(13) as usize], FACTOR_CODES[rng.below(12) as usize]); + let eq = [0u32, 1, 2, 6, 7][rng.below(5) as usize]; + d.words.blend = [(eq << 13) | (src << 6) | dst, rng.next() & 0x00ff_ffff, rng.next() & 0x00ff_ffff]; + } else if rng.chance(4) { + d.words.mode |= mode::COLOR_LOGIC_OP; + d.words.logic_op = rng.below(16); + } + if rng.chance(4) { + d.words.write_mask = (d.words.write_mask & !(0xf << write::COLOUR_SHIFT)) | (rng.below(16) << write::COLOUR_SHIFT); + } + if rng.chance(3) { + d.words.mode |= mode::STENCIL_TEST; + d.words.stencil = (rng.below(8) << 25) | (rng.below(6) << 22) | (rng.below(6) << 19) | (rng.below(6) << 16) | ((rng.next() & 0xff) << 8) | (rng.next() & 0xff); + if rng.chance(2) { + d.words.write_mask = (d.words.write_mask & !(0xff << write::STENCIL_SHIFT)) | ((rng.next() & 0xff) << write::STENCIL_SHIFT); + } + } + d.words.draw_mask = [1, 4, 5][rng.below(3) as usize]; + if rng.chance(3) { + d.depth_range = [rng.f(-0.1, 0.4), rng.f(0.5, 1.1)]; + } + if rng.chance(3) { + d.steps = [0, 1, -1, 1000, -1000, 1 << 24, -(1 << 24), 1 << 30][rng.below(8) as usize]; + } + if rng.chance(3) { + d.shade.alpha = Some((Compare::of(rng.below(8)), rng.below(4096))); + } + d +} + +/// Draw `tris` under `d` on both sides. +fn draw(p: &mut Pair, d: &DrawState, smooth: bool, tris: &[Tri]) { + let planes = p.planes(); + let mut st = BackendState::of(&d.words, planes.stencil.v.kind != super::super::view::Kind::None, !d.index); + if d.to_aux { + st.targets = aux_targets(st.targets, d.index); + } + st.store = st.select(&planes); + let raster_words = RasterWords { mode: if smooth { 0 } else { mode::FLAT_SHADE }, ..p.raster }; + let mut rst = p.raster_state(&raster_words, d.depth_range); + rst.offset_fill = true; + rst.offset_steps = d.steps; + let slot = Slot::new(&d.words, &planes, d.index, d.to_aux, &raster_words, d.shade.stipple.as_ref(), d.shade.background).unwrap_or_else(|e| panic!("{e:?}")); + let (mode_word, alpha_word) = d.shade.words(); + let (ox, oy) = (p.ox, p.oy); + let default_regs = TexRegs::default(); + let regs = d.tex.as_ref().map_or(&default_regs, |t| &t.0); + let shader = ShaderState::of(&ShaderWords { mode: mode_word, alpha: alpha_word, ..Default::default() }, regs); + let mem = p.cpu.ptr(); + let unit = d.tex.as_ref().map(|t| Unit { regs: &t.0, latch: &t.1, mem }); + let tex_slot = d.tex.as_ref().map(|t| TexSlot::new(&t.0, &t.1)); + for t in tris { + // The CPU walk and store. + triangle_quads(&rst, &t.tri, None, |q| { + let mut out = cpu_shade(t, q, &d.shade, unit.as_ref(), &shader); + out.x = ox.wrapping_add(q.x); + out.y = oy.wrapping_add(q.y); + if out.cov != 0 { + (st.store)(&st, &planes, &out); + } + }); + // The GPU record. + if let Some((x0, y0, x1, y1)) = walk_bounds(&rst, &t.tri, None) { + p.gpu.push_tri(t, ox, oy, (x0 + ox, y0 + oy, x1 + ox, y1 + oy), d.steps, d.depth_range, gl_cmd::DEPTH_SLACK, mode_word, alpha_word, &slot, tex_slot.as_ref(), shader.specular); + } + } +} + +/// Random texture registers: a 1D, 2D or 3D target of a few levels in +/// memory away from the destination, any wrap, filter, class and +/// environment, sometimes a border, a detail image, LOD limits, clamps, +/// a scale and bias or a colour table. +fn random_texture(rng: &mut Lcg, specular: bool) -> (TexRegs, Latch) { + let mut r = TexRegs::default(); + let target = [tex::MODE_2D, tex::MODE_2D, tex::MODE_2D, tex::MODE_1D, tex::MODE_3D][rng.below(5) as usize]; + r.path = target | if specular { tex::MODE_SEPARATE_SPECULAR } else { 0 }; + let (ls, lt, lr) = (rng.range(1, 7) as u32, rng.range(1, 7) as u32, if target == tex::MODE_3D { rng.range(1, 4) as u32 } else { 0 }); + let detail = rng.chance(5); + let k = rng.range(1, 4) as u32; + r.sampler[0] = (ls << 10) | (lt << 5) | lr; + if detail { + r.sampler[0] |= (k << 25) | ((ls + k).min(31) << 20) | ((lt + k).min(31) << 15) | if rng.chance(2) { 0x8000_0000 } else { 0 }; + } + let min = rng.below(6); + let mag = if detail { 2 + rng.below(3) } else { rng.below(2) }; + let class = 1 + rng.below(6); + let table = rng.chance(6); + r.sampler[4] = (class << 23) | (if table { [20, 24][rng.below(2) as usize] } else { 0 } << 18) | (rng.below(4) << 12) | (rng.below(4) << 10) | (rng.below(4) << 8) | (min << 5) | (mag << 1) | table as u32; + for a in 0..3 { + r.sampler[1 + a] = if rng.chance(3) { rng.below(1 << 7) } else { 0x1ffff }; + } + r.sampler[5] = rng.next() & 0x00ff_ffff; + r.sampler[6] = (rng.next() & 0x00ff_ffff) | if rng.chance(4) { 1 << 24 } else { 0 }; + r.env[0] = 1 + rng.below(5); + r.env[1] = rng.next() & 0x00ff_ffff; + r.env[2] = rng.next() & 0x00ff_ffff; + // LOD: a small bias, the limits around the chain. + r.lod[0] = (rng.range(-512, 512) as u32) & 0x3fff; + r.lod[1] = (rng.range(-512, 512) as u32) & 0x3fff; + r.lod[2] = (rng.range(-15 << 8, 1 << 8) as u32) & 0x3fff; + r.lod[3] = (rng.range(0, 15 << 8) as u32) & 0x3fff; + // Levels: entries 0.. in two memory regions the destination never + // reaches, a storage from the table, the pitch mostly automatic. + let formats = [0u32, 11, 12, 13, 16, 17, 18, 32]; + let format = formats[rng.below(8) as usize]; + for i in 0..super::super::pipe::texture::LEVEL_ENTRIES as u32 { + let base = [0x18000u32, 0x30000][rng.below(2) as usize] + rng.below(0x6000); + let pitch = if rng.chance(4) { rng.range(1, 6) as u32 } else { 0 }; + let e = (i as usize) * 3; + r.levels[e] = base; + r.levels[e + 1] = (format << 20) | pitch | if target == tex::MODE_3D { rng.below(4) << 30 } else { 0 }; + r.levels[e + 2] = if target == tex::MODE_3D { rng.below(3) } else { 0 }; + } + r.level0 = rng.below(4); + if rng.chance(4) { + for c in 0..4 { + r.scale_bias[c * 2] = (rng.range(-8192, 8192) as u32) & 0xffff; + r.scale_bias[c * 2 + 1] = (rng.range(-4096, 4096) as u32) & 0xffff; + } + } + for t in 0..2 { + for i in 0..256 { + r.colour_table[t][i] = rng.next() & 0x00ff_ffff; + } + } + for i in 0..256 { + r.detail[i] = rng.next() & 0x00ff_f000; + } + let l = Latch::of(&r); + (r, l) +} + +#[test] +fn textured_triangles_sample_as_the_cpu_unit() { + let Some(gpu) = gpu() else { return }; + let mut rng = Lcg(20); + let mut p = Pair::new(&gpu, &mut rng, &RGBA_MODE); + for round in 0..40 { + let mut d = DrawState::plain().depth(7); + if round % 3 == 0 { + d = random_state(&mut rng); + } + d.specular = rng.chance(3); + let (regs, latch) = random_texture(&mut rng, d.specular); + d.tex = Some(Box::new((regs, latch))); + let smooth = rng.chance(2); + let st = p.raster_state(&p.raster.clone(), [0.0, 1.0]); + let (ox, oy) = (p.ox, p.oy); + let regs = &d.tex.as_ref().unwrap().0; + let size = [12, 40, 150][rng.below(3) as usize]; + let t: Vec = (0..15).filter_map(|_| tri_textured(&mut rng, &st, smooth, ox, oy, size, Some(regs))).collect(); + draw(&mut p, &d, smooth, &t); + } + p.check("textured triangles"); + let (_, prims) = p.gpu.counts(); + assert!(prims > 300, "{prims} textured records"); +} + + +fn tris(rng: &mut Lcg, p: &mut Pair, smooth: bool, n: usize, size: i32) -> Vec { + let st = p.raster_state(&p.raster.clone(), [0.0, 1.0]); + let (ox, oy) = (p.ox, p.oy); + (0..n).filter_map(|_| tri_of(rng, &st, smooth, ox, oy, size)).collect() +} + +#[test] +fn flat_triangles_store_as_the_cpu_pipeline() { + let Some(gpu) = gpu() else { return }; + let mut rng = Lcg(1); + let mut p = Pair::new(&gpu, &mut rng, &RGBA_MODE); + let d = DrawState::plain().depth(1); + for &size in &[6, 40, 300, 3000] { + let t = tris(&mut rng, &mut p, false, 150, size); + draw(&mut p, &d, false, &t); + } + p.check("flat triangles"); + let (flushes, prims) = p.gpu.counts(); + assert!(prims > 200 && flushes >= 1, "{prims} records in {flushes} batches"); +} + +#[test] +fn smooth_triangles_and_the_alpha_test() { + let Some(gpu) = gpu() else { return }; + let mut rng = Lcg(2); + let mut p = Pair::new(&gpu, &mut rng, &RGBA_MODE); + for round in 0..6 { + let mut d = DrawState::plain().depth(3); + if round % 2 == 1 { + d.shade.alpha = Some((Compare::of(rng.below(8)), rng.below(4096))); + } + if round >= 4 { + d.depth_range = [0.2, 0.8]; + d.steps = -500; + } + let t = tris(&mut rng, &mut p, true, 120, [30, 200][round % 2]); + draw(&mut p, &d, true, &t); + } + p.check("smooth triangles"); +} + +#[test] +fn blends_logic_ops_masks_and_stencil() { + let Some(gpu) = gpu() else { return }; + let mut rng = Lcg(3); + let mut p = Pair::new(&gpu, &mut rng, &RGBA_MODE); + for _ in 0..40 { + let d = random_state(&mut rng); + let smooth = rng.chance(2); + let size = [20, 120][rng.below(2) as usize]; + let t = tris(&mut rng, &mut p, smooth, 25, size); + draw(&mut p, &d, smooth, &t); + } + p.check("random back-end states"); +} + +#[test] +fn polygon_stipples_with_and_without_a_background() { + let Some(gpu) = gpu() else { return }; + let mut rng = Lcg(4); + let mut p = Pair::new(&gpu, &mut rng, &RGBA_MODE); + for round in 0..4 { + let mut d = DrawState::plain().depth(1); + d.shade.stipple = Some(std::array::from_fn(|_| rng.next())); + if round % 2 == 1 { + d.shade.background = Some([rng.unit(), rng.unit(), rng.unit(), rng.unit()]); + } + let smooth = round >= 2; + let t = tris(&mut rng, &mut p, smooth, 60, 80); + draw(&mut p, &d, smooth, &t); + } + p.check("stipples"); +} + +/// A clear on both sides: the CPU as gl_raster_ops() runs it without a +/// ring, the GPU as the ring's rectangles. +fn clear(p: &mut Pair, c: &ClearDraw, x0: i32, y0: i32, x1: i32, y1: i32) { + let slot = Slot::clear(&c.planes, &p.raster).unwrap_or_else(|e| panic!("{e:?}")); + let n = p.raster.clip_kept as usize; + for r in &p.raster.clip_rect[..n] { + let (rx0, ry0, rx1, ry1) = (x0.max(r.x0), y0.max(r.y0), x1.min(r.x1), y1.min(r.y1)); + if rx0 > rx1 || ry0 > ry1 { + continue; + } + for y in ry0..=ry1 { + c.span(y, rx0, rx1); + } + p.gpu.push(Rec::rect(rx0, ry0, rx1, ry1, c), &slot); + } +} + +#[test] +fn clears_and_cpu_shaded_quads_keep_the_order() { + let Some(gpu) = gpu() else { return }; + let mut rng = Lcg(5); + let mut p = Pair::new(&gpu, &mut rng, &RGBA_MODE); + for round in 0..30 { + let planes = p.planes(); + // A clear of some planes, under a colour mask and an id test now + // and then. + let full = rng.chance(2); + let c = ClearDraw { + planes, + targets: [rng.chance(2) || round == 0, rng.chance(2)], + colour: rng.chance(3).then(|| rng.next()), + colour_mask: if full { u32::MAX } else { rng.next() }, + depth: rng.chance(2).then(|| rng.next() & ODSY_DEPTH24_MAX), + stencil: rng.chance(2).then(|| ((rng.next() & 0xff) as u8, (rng.next() & 0xff) as u8)), + id_value: (rng.next() & 0xff) as u8, + id_mask: if rng.chance(3) { (rng.next() & 0xff) as u8 } else { 0 }, + }; + let (x0, y0) = (rng.range(-10, 200), rng.range(-10, 200)); + clear(&mut p, &c, x0, y0, x0 + rng.range(1, 200), y0 + rng.range(1, 200)); + // Triangles over it. + let d = random_state(&mut rng); + let t = tris(&mut rng, &mut p, false, 10, 60); + draw(&mut p, &d, false, &t); + // Quads shaded on the CPU, stored through the same state. + let mut st = BackendState::of(&d.words, true, true); + st.store = st.select(&planes); + let raster_words = RasterWords { mode: mode::FLAT_SHADE, ..p.raster }; + let slot = Slot::new(&d.words, &planes, false, false, &raster_words, None, None).unwrap(); + for _ in 0..40 { + let q = BackQuad { + x: rng.range(0, 256) & !1, + y: rng.range(0, 256) & !1, + cov: (rng.next() & 0xf) as u8, + zq: std::array::from_fn(|_| rng.next() & ODSY_DEPTH24_MAX), + rgba: std::array::from_fn(|_| std::array::from_fn(|_| (rng.next() & 0xfff) as u16)), + ..BackQuad::default() + }; + (st.store)(&st, &planes, &q); + p.gpu.push(Rec::frag(&q), &slot); + } + } + p.check("clears, triangles and shaded quads"); +} + +#[test] +fn index_draws_share_words_between_pixels() { + let Some(gpu) = gpu() else { return }; + let mut rng = Lcg(6); + let mut p = Pair::new(&gpu, &mut rng, &OVERLAY_MODE); + assert_eq!(p.dv.colour[1].kind, super::super::view::Kind::Byte); + for round in 0..12 { + let mut d = DrawState::plain(); + d.index = true; + d.to_aux = round % 2 == 0; + d.words.draw_mask = [4, 5][rng.below(2) as usize]; + d.words.write_mask = (d.words.write_mask & !write::INDEX_MASK) | if rng.chance(2) { write::INDEX_MASK } else { rng.next() & write::INDEX_MASK }; + if rng.chance(2) { + d.words.mode |= mode::INDEX_LOGIC_OP; + d.words.logic_op = rng.below(16); + } + let t = tris(&mut rng, &mut p, false, 40, 50); + draw(&mut p, &d, false, &t); + // A clear of the index plane under an id test. + let planes = p.planes(); + let c = ClearDraw { + planes, + targets: [false, true], + colour: Some(rng.next() & 0xff), + colour_mask: if rng.chance(2) { u32::MAX } else { rng.next() & 0xff }, + depth: None, + stencil: None, + id_value: (rng.next() & 0xff) as u8, + id_mask: if rng.chance(2) { 0x0f } else { 0 }, + }; + let (x0, y0) = (rng.range(0, 200), rng.range(0, 200)); + clear(&mut p, &c, x0, y0, x0 + rng.range(1, 60), y0 + rng.range(1, 60)); + } + p.check("index draws into byte planes"); +} + +/// The depth word under depth ranges other than [0, 1] and with polygon +/// offsets, state by state, so a divergence names its range. +#[test] +fn depth_ranges_and_offsets() { + let Some(gpu) = gpu() else { return }; + let mut rng = Lcg(7); + let mut p = Pair::new(&gpu, &mut rng, &RGBA_MODE); + let mut failed = vec![]; + for round in 0..24 { + let mut d = DrawState::plain().depth(7); + match round % 4 { + 0 => d.depth_range = [0.0, 1.0], + 1 => d.depth_range = [rng.f(-0.1, 0.4), rng.f(0.5, 1.1)], + 2 => d.depth_range = [rng.f(0.0, 0.4), 1.0], + _ => d.depth_range = [0.0, rng.f(0.5, 1.1)], + } + if round >= 12 { + d.steps = [0, 1, -1, 1000, -1000, 1 << 24, -(1 << 24), 1 << 30][rng.below(8) as usize]; + } + let t = tris(&mut rng, &mut p, false, 30, 60); + draw(&mut p, &d, false, &t); + let got = p.gpu.read_words().unwrap(); + let n = got.iter().zip(p.cpu.words_mut().iter()).filter(|(g, w)| g != w).count(); + if n != 0 { + failed.push(format!("round {round}: range {:?} steps {}: {n} words", d.depth_range, d.steps)); + // Resync so the next round stands alone. + p.gpu.write_words(0, p.cpu.words_mut()); + } + } + assert!(failed.is_empty(), "{}", failed.join("\n")); +} + +/// The coded colour fields: 10:10:10:2 and 4:4:4:4 and 5:5:5:1 texels, the +/// 12-bit lanes, two-byte indices, under random states, clears and shaded +/// quads, each pixel mode on its own board. +#[test] +fn coded_colour_fields_convert_as_the_cpu_codecs() { + use super::super::view::Kind; + let Some(gpu) = gpu() else { return }; + // Status words: 4-byte 10:10:10:2 in an 8-byte window of a 16-byte + // pixel with depth; 12-bit lanes (16-byte window); 2-byte 4:4:4:4; + // 2-byte 5:5:5:1; 2-byte index. + let modes: [(u32, &str); 5] = [(0x6960, "packed10"), (0x6cd0, "lanes"), (0x65d0, "4444"), (0x65e0, "5551"), (0x8_65d0, "index16")]; + for (round, (status, name)) in modes.iter().enumerate() { + let mut rng = Lcg(10 + round as u64); + let pixel_mode = [RGBA_MODE[0], RGBA_MODE[1], *status, 0]; + let mut p = Pair::new(&gpu, &mut rng, &pixel_mode); + assert!(matches!(p.dv.colour[0].kind, Kind::Coded(_)), "{name}: {:?}", p.dv.colour[0].kind); + let index = *name == "index16"; + for _ in 0..12 { + let mut d = random_state(&mut rng); + d.index = index; + if index { + d.words.write_mask = (d.words.write_mask & !write::INDEX_MASK) | if rng.chance(2) { write::INDEX_MASK } else { rng.next() & write::INDEX_MASK }; + } + let smooth = rng.chance(2); + let t = tris(&mut rng, &mut p, smooth, 20, 70); + draw(&mut p, &d, smooth, &t); + let planes = p.planes(); + let c = ClearDraw { + planes, + targets: [rng.chance(2), rng.chance(2)], + colour: Some(rng.next()), + colour_mask: if rng.chance(2) { u32::MAX } else { rng.next() }, + depth: rng.chance(2).then(|| rng.next() & ODSY_DEPTH24_MAX), + stencil: None, + id_value: 0, + id_mask: 0, + }; + let (x0, y0) = (rng.range(0, 200), rng.range(0, 200)); + clear(&mut p, &c, x0, y0, x0 + rng.range(1, 60), y0 + rng.range(1, 60)); + } + p.check(name); + } +} + +/// Blit runs: rows of pixels through `blit_pixel`, into RGBA and byte +/// planes under masks, with and without the window's tests, interleaved +/// with triangles, against the CPU's stores in the same order. +#[test] +fn blit_runs_store_as_the_cpu_paint_path() { + use super::super::view::Kind; + let Some(gpu) = gpu() else { return }; + for (round, mode) in [RGBA_MODE, OVERLAY_MODE].iter().enumerate() { + let mut rng = Lcg(30 + round as u64); + let mut p = Pair::new(&gpu, &mut rng, mode); + let byte = p.dv.colour[0].kind == Kind::Byte; + for _ in 0..30 { + // A triangle or two between the runs. + let d = random_state(&mut rng); + let t = tris(&mut rng, &mut p, false, 2, 60); + draw(&mut p, &d, false, &t); + // Rows of pixels: some consecutive, some broken, into one or + // both buffers, a byte plane under a mask. + let planes = p.planes(); + let words = RasterWords { clip_rect: p.raster.clip_rect, clip_kept: p.raster.clip_kept, id_compare: p.raster.id_compare, ..Default::default() }; + let slot = Slot::clear(&planes, &words).unwrap(); + let window = rng.chance(2); + let buffers = 1 + rng.below(3); + let mask = if byte { rng.next() & 0xff } else { u32::MAX }; + // A logical operation now and then, with the index or RGB + // result masks of paint_integer. + let op = rng.chance(3).then(|| rng.below(16) as u8); + let (and, or) = if rng.chance(2) { (0xfff, 0) } else { (u32::MAX, 0xff) }; + let own = rng.chance(4); + let y = rng.range(-5, 260); + let mut x = rng.range(-5, 240); + for _ in 0..rng.range(1, 80) { + if rng.chance(10) { + x += rng.range(2, 9); + } + let word = rng.next(); + let owned = !window || (x >= 0 && y >= 0 && (x as u32) < p.dv.colour[0].width && p.raster.clip_rect[..p.raster.clip_kept as usize].iter().any(|r| x >= r.x0 && x <= r.x1 && y >= r.y0 && y <= r.y1)); + for b in 0..2u32 { + if buffers & (1 << b) == 0 { + continue; + } + let rows = planes.colour[b as usize]; + if owned { + let old = rows.load_word(x, y); + let val = match (op, old) { + (Some(o), Some(old)) => (super::super::logic::logic_op(o, word, old) & and) | or, + (Some(_), None) => u32::MAX, // nothing stored: the GPU returns too + (None, _) => word, + }; + if op.is_some() && old.is_none() { + } else if byte { + if let Some(old) = rows.load1(x, y) { + rows.store1(x, y, (old & !(mask as u8)) | (val as u8 & mask as u8)); + } + } else { + rows.store_word(x, y, val); + } + } + let view = if own { super::super::gpu_unit::BlitView::Own(Slot::own_view(&rows).unwrap()) } else { super::super::gpu_unit::BlitView::Slot(b) }; + let key = super::super::gpu_unit::BlitKey { view, mask, window, op: op.map_or(0, |o| 0x100 | o as u32), and, or }; + p.gpu.blit_pixel(&slot, &key, x, y, word); + } + x += 1; + } + } + p.check(if byte { "blits into byte planes" } else { "blits" }); + } +} + +/// The scan-out on the GPU against `Frame::compose`: random display +/// modes, lookup RAM, overlay modes and cursor over random memory, whole +/// frames compared. +#[test] +fn scanout_composes_as_the_cpu_frame() { + use super::super::consts::{cursor, ODSY_GAMMA_GROUP}; + use super::super::frame::{pbj, Frame}; + use super::super::reader::DisplayRegs; + use super::scanout::VproScanout; + let Some(gpu) = gpu() else { return }; + let mut rng = Lcg(40); + let mut p = Pair::new(&gpu, &mut rng, &RGBA_MODE); + let (buf, words) = p.gpu.vram_buffer(); + let mut scan = VproScanout::new(&gpu, &buf, words).unwrap_or_else(|e| panic!("{e}")); + for round in 0..24 { + let mut regs = DisplayRegs::default(); + // The window-id and frame buffers of the test destination: 128 x 96 + // of 16-byte pixels (or 8 and 4 now and then), stride 16 tiles. + let depth = [3u32, 3, 2, 1][rng.below(4) as usize]; + regs.pbj[pbj::OVERLAY_BASE] = WID_BASE; + regs.pbj[pbj::PIXEL_BASE] = RGBA_MODE[0]; + regs.pbj[pbj::WIDTH] = [128, 97, 256][rng.below(3) as usize]; + regs.pbj[pbj::HEIGHT] = [96, 61, 128][rng.below(3) as usize]; + regs.pbj[pbj::STRIDE] = 16; + regs.pbj[pbj::DEPTH] = depth; + for id in 0..32 { + // A mode word with naturally aligned colour fields, as the X + // server programs them (a field across an 8-byte word is one + // the model does not load); every fourth is RGB through the + // gamma group. + use super::super::frame::mode; + // No 12-bit lanes in a 4-byte pixel: the model's guard lets a + // 6-byte field across the pixel through. + let (format, bytes) = [(mode::RGB4, 2u32), (mode::RGB5, 2), (mode::RGB8, 4), (mode::RGB10, 4), (mode::RGB12, 8), (0, 2), (1, 4)][rng.below(if depth >= 2 { 7 } else { 4 }) as usize]; + let units = if format == 1 { 2 } else if format == 0 { 1 } else { rng.below(8) }; + let at = |rng: &mut Lcg| (rng.below(16 / bytes) * bytes / 2) & 7; + let group = if rng.chance(4) { ODSY_GAMMA_GROUP } else { rng.below(32) }; + let m = (units << mode::FIELD_UNITS_SHIFT) | (at(&mut rng) << mode::A_SHIFT) | (at(&mut rng) << mode::B_SHIFT) | (format << mode::FORMAT_SHIFT) | (group << 4) | (rng.next() & 0xf) | (rng.next() & 0xfc00_0000); + regs.dispmode[id] = m; + } + for o in 0..8 { + regs.overlay_mode[o] = rng.next(); + } + for e in regs.lut.iter_mut() { + *e = rng.next(); + } + for c in regs.cursor.iter_mut() { + *c = rng.next(); + } + regs.cursor[cursor::POSITION as usize] = ((rng.range(-40, 140) as u32 & 0xffff) << 16) | (rng.range(-40, 110) as u32 & 0xffff); + regs.cursor[cursor::CONTROL as usize] = match round % 4 { + 0 => 0, + 1 => cursor::CTL_ENABLE, + 2 => cursor::CTL_ENABLE | cursor::CTL_CROSSHAIR, + _ => cursor::CTL_ENABLE | cursor::CTL_CHANNEL1, + }; + let displayed = rng.next(); + let frame = Frame::snapshot(®s, displayed).expect("a frame"); + let (w, h) = (frame.width as usize, frame.height as usize); + let mut want = vec![0u32; w * h]; + frame.compose(p.cpu.ptr(), ®s, &mut want, w, h); + let mut got = vec![0u32; w * h]; + scan.run(&frame, ®s, w as u32, h as u32, &mut got).unwrap_or_else(|e| panic!("{e}")); + let diffs = want.iter().zip(&got).filter(|(a, b)| a != b).count(); + if diffs != 0 { + let (i, (a, b)) = want.iter().zip(&got).enumerate().find(|(_, (a, b))| a != b).unwrap(); + panic!("round {round}: {diffs} pixels differ, first at ({}, {}): cpu {a:#08x} gpu {b:#08x}", i % w, i / w); + } + } +} diff --git a/src/dev/odyssey/worker.rs b/src/dev/odyssey/worker.rs index 43b6c5c45..b0a44abf0 100644 --- a/src/dev/odyssey/worker.rs +++ b/src/dev/odyssey/worker.rs @@ -169,9 +169,11 @@ impl Board { break 'slice; } } - if fifo.pending() { + let behind = fifo.pending(); + if behind { crate::dlog_dev!(crate::devlog::LogModule::Odyssey, "sgi-odyssey: worker behind: {} stores wait", fifo.stores()); } + self.gpu_upload_tick(!behind); } /// Finish the waiting work now, for an access at addr that must see it, diff --git a/src/disp.rs b/src/disp.rs index 27391c2b2..34a70db15 100644 --- a/src/disp.rs +++ b/src/disp.rs @@ -554,10 +554,17 @@ impl StatusBarTexture { t } - /// Render the status bar and upload to the GL texture; returns the texture handle. - pub fn render_and_upload(&mut self, bar: &mut StatusBar, stats: &BarStats, width: usize, gl: &glow::Context) -> glow::Texture { + /// Render the status bar into `buf` (stride 2048, 0xAABBGGRR) with no + /// GL: what a renderer other than the GL one uploads itself. + pub fn render_pixels(&mut self, bar: &mut StatusBar, stats: &BarStats, width: usize) -> &[u32] { bar.update(stats.hb); bar.render(&mut self.buf, width, 0, stats); + &self.buf + } + + /// Render the status bar and upload to the GL texture; returns the texture handle. + pub fn render_and_upload(&mut self, bar: &mut StatusBar, stats: &BarStats, width: usize, gl: &glow::Context) -> glow::Texture { + self.render_pixels(bar, stats, width); let tex = self.ensure_tex(gl); unsafe { gl.bind_texture(glow::TEXTURE_2D, Some(tex)); diff --git a/src/gfx_display.rs b/src/gfx_display.rs index a7eb73bad..f8416f693 100644 --- a/src/gfx_display.rs +++ b/src/gfx_display.rs @@ -57,6 +57,12 @@ pub trait GfxDisplay: Send + Sync { /// The hardware cursor as the guest has set it, read live from the /// board's registers (not the last composed frame). fn hw_cursor(&self) -> Option { None } + + /// The host GPU the board draws or composes on, when it uses one (the + /// VPro with `raster = "wgpu"`): a wgpu window renderer presents on the + /// same device, so a frame composed there need not come back to the CPU. + #[cfg(feature = "wgpu")] + fn gpu(&self) -> Option> { None } } /// Whether the display thread tears its renderer down when it ends. diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index ff2a9302f..9bf60af8b 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -288,7 +288,7 @@ impl OctaneMachine { let status = crate::disp::StatusSource::new(heartbeat.clone(), fasttick.clone()); let gfx = board.map(|b| ImpactXio::with_board(WIDGET_GFX, b, heart.clone(), mem.clone(), status.clone())); let vpro = if has_vpro { - Some(Odyssey::new(WIDGET_GFX, kind.as_str(), heart.clone(), mem.clone(), status.clone(), (cfg.graphics.threads, cfg.graphics.tile))?) + Some(Odyssey::new(WIDGET_GFX, kind.as_str(), heart.clone(), mem.clone(), status.clone(), (cfg.graphics.threads, cfg.graphics.tile), (cfg.graphics.raster, cfg.graphics.gpu_backend.as_str()))?) } else { None }; diff --git a/src/lib.rs b/src/lib.rs index 954c0962f..8db3414a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ #![allow(dead_code, unused_variables, unused_imports)] -#![cfg_attr(feature = "rex-gpu", recursion_limit = "256")] +#![cfg_attr(any(feature = "rex-gpu", feature = "wgpu"), recursion_limit = "256")] #[cfg(all(feature = "lightning", feature = "developer"))] compile_error!( @@ -204,6 +204,8 @@ pub mod ui; pub mod cpu; pub mod dev; pub mod gfifo; +#[cfg(feature = "wgpu")] +pub mod wgpu; pub mod trace_writer; pub mod gfx_display; pub mod hostinput; diff --git a/src/main.rs b/src/main.rs index cf5b72e10..13a363dd2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,6 +38,7 @@ fn main() { let scroll_pixels_per_line = cfg.mouse_scroll_pixels_per_line; let lock_aspect_ratio = cfg.lock_aspect_ratio; + let ui_renderer = cfg.renderer; let headless = cfg.headless; let gdb_port = cfg.gdb_port; let ci_enabled = cfg.ci; @@ -188,7 +189,7 @@ fn main() { use winit::event_loop::EventLoop; let event_loop = EventLoop::new().unwrap(); let display = machine.get_display().expect("a graphics board must be present in non-headless mode"); - let ui = Ui::new(machine.get_ps2(), display, machine.disc_changer(), machine.get_timer_manager(), &event_loop, scale, scroll_pixels_per_line, lock_aspect_ratio); + let ui = Ui::new(machine.get_ps2(), display, machine.disc_changer(), machine.get_timer_manager(), &event_loop, scale, scroll_pixels_per_line, lock_aspect_ratio, ui_renderer); ui.run(event_loop); } diff --git a/src/ui.rs b/src/ui.rs index c905b797b..16003752b 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -757,6 +757,167 @@ impl Renderer for GlRenderer { } +/// The window drawn through wgpu (`renderer = "wgpu"`): the frame, the +/// debug overlay and the status bar presented by `crate::wgpu::Presenter`, +/// on the device of the board that draws on one, else on its own. Newport's +/// planes are composed on the CPU here (its GL compositor has no wgpu form +/// yet); a board that composes its own frame hands it over finished. +#[cfg(feature = "wgpu")] +struct WgpuRenderer { + presenter: crate::wgpu::Presenter, + window_size: Arc>>, + resize_request: Arc>>, + event_proxy: EventLoopProxy<()>, + scale: u32, + display_res: Arc>, + compositor: SwCompositor, + /// Newport's planes composed on the presenter's device; None until the + /// first such frame, or when it cannot be made (then `compositor`). + ng1: Option>, + /// The frame read back for screenshots and VNC, 0x00RRGGBB. + readback: Vec, + /// The first presentation error, reported once. + failed: bool, + /// The presenter is on the board's own device, so frames the board + /// composes there can be presented as they are. + board_device: bool, +} + +#[cfg(feature = "wgpu")] +impl WgpuRenderer { + fn report(&mut self, r: Result<(), String>) { + if let Err(e) = r { + if !self.failed { + self.failed = true; + eprintln!("iris: wgpu present: {e}"); + } + } + } +} + +#[cfg(feature = "wgpu")] +impl Renderer for WgpuRenderer { + fn present( + &mut self, + screen: &mut Rex3Screen, + overlay: &mut DebugOverlay, + status: &mut StatusBar, + sbtex: &mut StatusBarTexture, + stats: &BarStats, + need_readback: bool, + live_fb_rgb: Option<&[u32]>, + live_fb_aux: Option<&[u32]>, + ) { + let (width, height) = (screen.width, screen.height); + if width == 0 || height == 0 { + return; + } + if let Some((w, h)) = self.window_size.lock().take() { + self.presenter.resize(w, h); + } + *self.display_res.lock() = (width as u32, height as u32); + let prebuilt = screen.prebuilt; + // Newport's planes: composed on the GPU, presented from there, read + // back only for screenshots and VNC (`need_readback`). + if !prebuilt { + if self.ng1.is_none() { + self.ng1 = Some(crate::dev::ng1::wgpu::Ng1Scanout::new(self.presenter.gpu()).map_err(|e| { + eprintln!("iris: Newport's compositor on the GPU is unavailable ({e}); composing on the CPU"); + })); + } + if let Some(Ok(ng1)) = &mut self.ng1 { + let src = screen.compositor_source_from(live_fb_rgb, live_fb_aux); + let composed = if need_readback { + self.readback.resize(width * height, 0); + ng1.run(&src, &mut self.readback).map(|()| Ok(ng1.output())) + } else { + Ok(ng1.compose(&src)) + }; + match composed { + Ok(Ok(buffer)) => { + if need_readback { + for y in 0..height { + let row = y * crate::disp::FB_STRIDE; + for (d, &v) in screen.rgba[row..row + width].iter_mut().zip(&self.readback[y * width..(y + 1) * width]) { + *d = 0xff00_0000 | v.swap_bytes() >> 8; + } + } + } + let ov = if overlay.active() { Some(overlay.render_pixels(&screen.overlay_source())) } else { None }; + let sb = sbtex.render_pixels(status, stats, width); + let frame = crate::wgpu::Frame::Gpu { buffer: &buffer, width: width as u32, height: height as u32 }; + let r = self.presenter.present(frame, ov, sb, crate::disp::FB_STRIDE); + self.report(r); + return; + } + Ok(Err(e)) | Err(e) => { + eprintln!("iris: Newport's compositor on the GPU failed ({e}); composing on the CPU"); + self.ng1 = Some(Err(())); + } + } + } + let src = screen.compositor_source_from(live_fb_rgb, live_fb_aux); + self.compositor.compose_pixels(&src); + } + let ov = if overlay.active() { Some(overlay.render_pixels(&screen.overlay_source())) } else { None }; + let sb = sbtex.render_pixels(status, stats, width); + let rgba: &[u32] = if prebuilt { &screen.rgba } else { self.compositor.pixels() }; + let frame = crate::wgpu::Frame::Cpu { rgba, stride: crate::disp::FB_STRIDE, width: width as u32, height: height as u32 }; + let r = self.presenter.present(frame, ov, sb, crate::disp::FB_STRIDE); + self.report(r); + // Screenshots and VNC read the composed frame from `screen.rgba`; a + // prebuilt frame is there already. + if need_readback && !prebuilt { + let pixels = self.compositor.pixels(); + for y in 0..height { + let row = y * crate::disp::FB_STRIDE; + screen.rgba[row..row + width].copy_from_slice(&pixels[row..row + width]); + } + } + } + + fn resize(&mut self, width: usize, height: usize) { + // As the GL renderer: publish the resolution, ask the event thread + // to snap the window (never touch the window from here). + *self.display_res.lock() = (width as u32, height as u32); + *self.resize_request.lock() = Some(ResizeRequest::Scale(self.scale)); + let _ = self.event_proxy.send_event(()); + } + + fn compositor_status(&self) -> String { + let comp = if matches!(self.ng1, Some(Ok(_))) { "gpu" } else { "sw" }; + format!("renderer=wgpu compositor={comp} adapter={}", self.presenter.gpu().caps.adapter) + } + + fn takes_gpu_frames(&self) -> bool { + self.board_device + } + + fn present_gpu_frame( + &mut self, + frame: &wgpu::Buffer, + width: u32, + height: u32, + screen: &mut Rex3Screen, + overlay: &mut DebugOverlay, + status: &mut StatusBar, + sbtex: &mut StatusBarTexture, + stats: &BarStats, + ) { + if width == 0 || height == 0 { + return; + } + if let Some((w, h)) = self.window_size.lock().take() { + self.presenter.resize(w, h); + } + *self.display_res.lock() = (width, height); + let ov = if overlay.active() { Some(overlay.render_pixels(&screen.overlay_source())) } else { None }; + let sb = sbtex.render_pixels(status, stats, width as usize); + let r = self.presenter.present(crate::wgpu::Frame::Gpu { buffer: frame, width, height }, ov, sb, crate::disp::FB_STRIDE); + self.report(r); + } +} + struct MouseDelta { accum: (f64, f64), wheel: f64, @@ -779,7 +940,7 @@ pub struct Ui { } impl Ui { - pub fn new(ps2: Arc, display: Arc, discs: Option>, timer_manager: Arc, event_loop: &EventLoop<()>, scale: u32, scroll_pixels_per_line: f64, lock_aspect_ratio: bool) -> Self { + pub fn new(ps2: Arc, display: Arc, discs: Option>, timer_manager: Arc, event_loop: &EventLoop<()>, scale: u32, scroll_pixels_per_line: f64, lock_aspect_ratio: bool, renderer: crate::config::UiRenderer) -> Self { // The Indy's default video mode is 1280×1024; open the window at that // size (plus the status bar). The renderer snaps to the real resolution // via resize() once the PROM/IRIX programs its actual mode, keeping `scale`. @@ -790,6 +951,18 @@ impl Ui { .with_resizable(true) .with_inner_size(winit::dpi::PhysicalSize::new(w, h)); + #[cfg(feature = "wgpu")] + if renderer == crate::config::UiRenderer::Wgpu { + match Self::new_wgpu(&display, event_loop, window_builder.clone(), scale) { + Ok((window, window_size, resize_request, display_res)) => { + return Self { ps2, display, discs, window, window_size, resize_request, display_res, timer_manager, initial_scale: scale, scroll_pixels_per_line, lock_aspect_ratio }; + } + Err(e) => eprintln!("iris: the wgpu renderer is unavailable ({e}); drawing with GL"), + } + } + #[cfg(not(feature = "wgpu"))] + let _ = renderer; + let template = ConfigTemplateBuilder::new() .with_alpha_size(8) .with_transparency(false); @@ -919,6 +1092,53 @@ impl Ui { Self { ps2, display, discs, window, window_size, resize_request, display_res, timer_manager, initial_scale: scale, scroll_pixels_per_line, lock_aspect_ratio } } + /// The window and its wgpu renderer, installed in the board's renderer + /// slot. The window is a plain winit one (GL's needs a GL-capable visual + /// from glutin, this one does not); the device is the board's when it + /// draws on one, so a frame it composes there stays on it. + #[cfg(feature = "wgpu")] + #[allow(clippy::type_complexity)] + fn new_wgpu( + display: &Arc, + event_loop: &EventLoop<()>, + attrs: WindowAttributes, + scale: u32, + ) -> Result<(Arc, Arc>>, Arc>>, Arc>), String> { + let gpu = match display.gpu() { + Some(g) => g, + None => crate::wgpu::Gpu::new(&crate::wgpu::GpuOptions::default())?, + }; + #[allow(deprecated)] + let window = Arc::new(event_loop.create_window(attrs).map_err(|e| format!("create the window: {e}"))?); + let size = window.inner_size(); + let presenter = crate::wgpu::Presenter::new( + &gpu, + window.clone(), + (size.width, size.height), + (crate::disp::FB_STRIDE as u32, crate::disp::FB_MAX_H as u32), + (crate::disp::FB_STRIDE as u32, STATUS_BAR_HEIGHT as u32), + )?; + eprintln!("iris: drawing the window with wgpu on {}", gpu.caps.adapter); + let window_size = Arc::new(Mutex::new(None)); + let resize_request = Arc::new(Mutex::new(None)); + let display_res = Arc::new(Mutex::new((1280u32, 1024u32))); + let renderer = WgpuRenderer { + presenter, + window_size: window_size.clone(), + resize_request: resize_request.clone(), + event_proxy: event_loop.create_proxy(), + scale, + display_res: display_res.clone(), + compositor: SwCompositor::new(), + ng1: None, + readback: Vec::new(), + failed: false, + board_device: display.gpu().is_some_and(|g| Arc::ptr_eq(&g, &gpu)), + }; + *display.renderer_slot().lock() = Some(Box::new(renderer)); + Ok((window, window_size, resize_request, display_res)) + } + /// Run the UI event loop (blocks the current thread) pub fn run(self, event_loop: EventLoop<()>) { let Ui { ps2, display, discs, window, window_size, resize_request, display_res, timer_manager, initial_scale, scroll_pixels_per_line, lock_aspect_ratio } = self; diff --git a/src/wgpu/batch.rs b/src/wgpu/batch.rs new file mode 100644 index 000000000..930a9583e --- /dev/null +++ b/src/wgpu/batch.rs @@ -0,0 +1,354 @@ +//! The batch: what the CPU records between two points where memory is +//! observed, and the two dispatches that draw it. +//! +//! A board appends one setup record per primitive (`prim`) and one state +//! slot per distinct state (`slot`); a record names its slot by index. +//! `flush` uploads the records, slots and tile boxes, runs the bin and +//! raster kernels and empties the recorder. Records, slots and the common +//! buffers live in fixed GPU buffers, so a board's bind groups are created +//! once. +//! +//! Buffer bindings: group 0 is the batch (`shaders/common.wgsl`); group 1 +//! is the records at binding 0 and the slots at binding 1; groups from 2 +//! up are the board's (its memory, its tables). `BatchPipelines` builds +//! both kernels around the board's hook fragments. + +use std::marker::PhantomData; +use std::sync::Arc; + +use bytemuck::Pod; + +use super::device::Gpu; +use super::shader::{Fragment, Source}; +use super::tiles::{BatchHeader, BusyHead, DispatchArgs, TileBox, TileGrid, MASK_WORDS, MAX_GROUPS, MAX_PRIMS, TILE_H, TILE_W}; + +/// The batch has no room for another primitive or slot: flush it first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Full; + +const COMMON: Fragment = Fragment::new("common.wgsl", include_str!("shaders/common.wgsl")); +const BIN: Fragment = Fragment::new("bin.wgsl", include_str!("shaders/bin.wgsl")); +const BIN_ACCEPT_ALL: Fragment = Fragment::new("bin_accept_all.wgsl", include_str!("shaders/bin_accept_all.wgsl")); +const RASTER: Fragment = Fragment::new("raster.wgsl", include_str!("shaders/raster.wgsl")); + +/// A board's side of the kernels. +pub struct ArchShaders<'a> { + pub label: &'a str, + /// Fragments defining `arch_tile_begin`, `arch_prim` and + /// `arch_tile_end`, and the group 1 and 2+ bindings they use. + pub raster: &'a [Fragment], + /// A fragment defining `bin_accept`; None accepts every tile of the + /// box. It may bind group 1 and up like the raster fragments. + pub bin_accept: Option, + /// Fragments `#include` may name. + pub library: &'a [Fragment], + /// `const NAME: u32` definitions both kernels see. + pub defines: &'a [(&'a str, u32)], + /// Bind group layouts for groups 2 and up. + pub layouts: &'a [&'a wgpu::BindGroupLayout], +} + +/// The two kernels and the layouts of groups 0 and 1. Group 0 has two +/// layouts: the bin pass also binds the dispatch arguments it raises, +/// which the raster pass consumes as its indirect buffer instead. +pub struct BatchPipelines { + pub bin_layout: wgpu::BindGroupLayout, + pub batch_layout: wgpu::BindGroupLayout, + pub records_layout: wgpu::BindGroupLayout, + pub bin: wgpu::ComputePipeline, + pub raster: wgpu::ComputePipeline, +} + +impl BatchPipelines { + pub fn new(gpu: &Gpu, arch: &ArchShaders<'_>) -> Result { + let common = [ + layout_entry(0, wgpu::BufferBindingType::Uniform), + layout_entry(1, wgpu::BufferBindingType::Storage { read_only: false }), + layout_entry(2, wgpu::BufferBindingType::Storage { read_only: false }), + layout_entry(3, wgpu::BufferBindingType::Storage { read_only: false }), + layout_entry(4, wgpu::BufferBindingType::Storage { read_only: true }), + ]; + let batch_layout = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("batch"), + entries: &common, + }); + let mut with_args = common.to_vec(); + with_args.push(layout_entry(5, wgpu::BufferBindingType::Storage { read_only: false })); + let bin_layout = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("batch bin"), + entries: &with_args, + }); + let records_layout = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("batch records"), + entries: &[ + layout_entry(0, wgpu::BufferBindingType::Storage { read_only: true }), + layout_entry(1, wgpu::BufferBindingType::Storage { read_only: true }), + ], + }); + let mut bin_layouts: Vec<&wgpu::BindGroupLayout> = vec![&bin_layout, &records_layout]; + bin_layouts.extend_from_slice(arch.layouts); + let mut layouts: Vec<&wgpu::BindGroupLayout> = vec![&batch_layout, &records_layout]; + layouts.extend_from_slice(arch.layouts); + + let base = |label: &str| { + let mut s = Source::new(label) + .define("TILE_W", TILE_W) + .define("TILE_H", TILE_H) + .define("MASK_WORDS", MASK_WORDS) + .define("MAX_GROUPS", MAX_GROUPS) + .library(COMMON); + for (k, v) in arch.defines { + s = s.define(k, *v); + } + for f in arch.library { + s = s.library(*f); + } + s + }; + let mut bin_src = base(&format!("{} bin", arch.label)).body(BIN); + bin_src = bin_src.body(arch.bin_accept.unwrap_or(BIN_ACCEPT_ALL)); + let bin = bin_src.compute_pipeline(&gpu.device, "bin", &bin_layouts)?; + + let mut raster_src = base(&format!("{} raster", arch.label)).body(RASTER); + for f in arch.raster { + raster_src = raster_src.body(*f); + } + let raster = raster_src.compute_pipeline(&gpu.device, "raster", &layouts)?; + Ok(BatchPipelines { bin_layout, batch_layout, records_layout, bin, raster }) + } +} + +fn layout_entry(binding: u32, ty: wgpu::BufferBindingType) -> wgpu::BindGroupLayoutEntry { + wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty, has_dynamic_offset: false, min_binding_size: None }, + count: None, + } +} + +/// The recorder and its GPU buffers. `R` is the board's setup record, `S` +/// its state slot; both are plain `repr(C)` data laid out as the board's +/// WGSL declares them. +pub struct Batch { + gpu: Arc, + grid: TileGrid, + tiles_allocated: u32, + max_slots: u32, + records: Vec, + boxes: Vec<[u32; 4]>, + slots: Vec, + header_buf: wgpu::Buffer, + masks_buf: wgpu::Buffer, + busy_buf: wgpu::Buffer, + args_buf: wgpu::Buffer, + flags_buf: wgpu::Buffer, + boxes_buf: wgpu::Buffer, + records_buf: wgpu::Buffer, + slots_buf: wgpu::Buffer, + bin_group: wgpu::BindGroup, + batch_group: wgpu::BindGroup, + records_group: wgpu::BindGroup, + /// Batches flushed so far. + pub flushes: u64, + /// Primitives drawn so far. + pub prims_total: u64, + _p: PhantomData<(R, S)>, +} + +impl Batch { + /// A recorder over `grid` with room for `max_slots` slots a batch. + pub fn new(gpu: &Arc, pipelines: &BatchPipelines, grid: TileGrid, max_slots: u32) -> Batch { + let tiles = grid.tiles().max(1); + let storage = wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST; + let header_buf = gpu.buffer("batch header", 32, wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST); + let masks_buf = gpu.buffer("batch masks", tiles as u64 * MASK_WORDS as u64 * 4, storage); + let busy_buf = gpu.buffer("batch busy", 16 + tiles as u64 * 4, storage); + let args_buf = gpu.buffer("batch dispatch args", 16, storage | wgpu::BufferUsages::INDIRECT); + let flags_buf = gpu.buffer("batch flags", tiles as u64 * 4, storage); + let boxes_buf = gpu.buffer("batch boxes", MAX_PRIMS as u64 * 16, storage); + let records_buf = gpu.buffer("batch records", MAX_PRIMS as u64 * size_of::().max(4) as u64, storage); + let slots_buf = gpu.buffer("batch slots", max_slots.max(1) as u64 * size_of::().max(4) as u64, storage); + let batch_group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("batch"), + layout: &pipelines.batch_layout, + entries: &[ + entry(0, &header_buf), + entry(1, &masks_buf), + entry(2, &busy_buf), + entry(3, &flags_buf), + entry(4, &boxes_buf), + ], + }); + let bin_group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("batch bin"), + layout: &pipelines.bin_layout, + entries: &[ + entry(0, &header_buf), + entry(1, &masks_buf), + entry(2, &busy_buf), + entry(3, &flags_buf), + entry(4, &boxes_buf), + entry(5, &args_buf), + ], + }); + let records_group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("batch records"), + layout: &pipelines.records_layout, + entries: &[entry(0, &records_buf), entry(1, &slots_buf)], + }); + Batch { + gpu: Arc::clone(gpu), + grid, + tiles_allocated: tiles, + max_slots, + records: Vec::with_capacity(MAX_PRIMS as usize), + boxes: Vec::with_capacity(MAX_PRIMS as usize), + slots: Vec::with_capacity(max_slots as usize), + header_buf, + masks_buf, + busy_buf, + args_buf, + flags_buf, + boxes_buf, + records_buf, + slots_buf, + bin_group, + batch_group, + records_group, + flushes: 0, + prims_total: 0, + _p: PhantomData, + } + } + + pub fn grid(&self) -> TileGrid { + self.grid + } + + /// Change the target. Only between batches; a grid with more tiles than + /// were allocated is refused (allocate the largest at construction). + pub fn set_grid(&mut self, grid: TileGrid) -> Result<(), String> { + if !self.is_empty() { + return Err("set_grid on a batch with primitives".into()); + } + if grid.tiles() > self.tiles_allocated { + return Err(format!("grid of {} tiles exceeds the {} allocated", grid.tiles(), self.tiles_allocated)); + } + self.grid = grid; + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } + + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn slots_used(&self) -> usize { + self.slots.len() + } + + /// Append a state slot; its index for records that follow. + pub fn slot(&mut self, s: S) -> Result { + if self.slots.len() as u32 >= self.max_slots { + return Err(Full); + } + self.slots.push(s); + Ok(self.slots.len() as u32 - 1) + } + + /// Append a primitive covering at most the pixel box `x0..=x1`, + /// `y0..=y1`. Ok(None) when the box misses the target. + pub fn prim(&mut self, r: R, x0: i32, y0: i32, x1: i32, y1: i32) -> Result, Full> { + let Some(b) = self.grid.tile_box(x0, y0, x1, y1) else { return Ok(None) }; + self.prim_tiles(r, b).map(Some) + } + + /// Append a primitive over an explicit tile box. + pub fn prim_tiles(&mut self, r: R, b: TileBox) -> Result { + if self.records.len() as u32 >= MAX_PRIMS { + return Err(Full); + } + self.records.push(r); + self.boxes.push(b.packed()); + Ok(self.records.len() as u32 - 1) + } + + /// The group 0 and group 1 bind groups, for a board that encodes its + /// own passes around the batch. + pub fn bind_groups(&self) -> (&wgpu::BindGroup, &wgpu::BindGroup) { + (&self.batch_group, &self.records_group) + } + + /// Upload and draw what was recorded, with `groups` bound from group 2 + /// up. Returns the submission, or None when nothing was recorded. + pub fn flush(&mut self, pipelines: &BatchPipelines, groups: &[&wgpu::BindGroup]) -> Option { + if self.records.is_empty() { + self.slots.clear(); + return None; + } + let q = &self.gpu.queue; + let header = BatchHeader { + prim_count: self.records.len() as u32, + tile_cols: self.grid.cols(), + tile_rows: self.grid.rows(), + mask_words: MASK_WORDS, + origin_x: self.grid.origin_x, + origin_y: self.grid.origin_y, + _pad: [0; 2], + }; + q.write_buffer(&self.header_buf, 0, bytemuck::bytes_of(&header)); + q.write_buffer(&self.busy_buf, 0, bytemuck::bytes_of(&BusyHead::default())); + q.write_buffer(&self.args_buf, 0, bytemuck::bytes_of(&DispatchArgs::EMPTY)); + q.write_buffer(&self.boxes_buf, 0, bytemuck::cast_slice(&self.boxes)); + q.write_buffer(&self.records_buf, 0, bytemuck::cast_slice(&self.records)); + if !self.slots.is_empty() { + q.write_buffer(&self.slots_buf, 0, bytemuck::cast_slice(&self.slots)); + } + + let mut enc = self.gpu.encoder("batch"); + { + let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("bin"), timestamp_writes: None }); + pass.set_pipeline(&pipelines.bin); + pass.set_bind_group(0, &self.bin_group, &[]); + pass.set_bind_group(1, &self.records_group, &[]); + for (i, g) in groups.iter().enumerate() { + pass.set_bind_group(2 + i as u32, *g, &[]); + } + pass.dispatch_workgroups(self.records.len() as u32, 1, 1); + } + { + let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("raster"), timestamp_writes: None }); + pass.set_pipeline(&pipelines.raster); + pass.set_bind_group(0, &self.batch_group, &[]); + pass.set_bind_group(1, &self.records_group, &[]); + for (i, g) in groups.iter().enumerate() { + pass.set_bind_group(2 + i as u32, *g, &[]); + } + pass.dispatch_workgroups_indirect(&self.args_buf, 0); + } + let submission = self.gpu.submit(enc); + self.flushes += 1; + self.prims_total += self.records.len() as u64; + self.records.clear(); + self.boxes.clear(); + self.slots.clear(); + Some(submission) + } + + /// Zero the masks and flags. Only needed after a failed batch; a + /// completed raster pass leaves them clean. + pub fn reset(&self) { + let mut enc = self.gpu.encoder("batch reset"); + enc.clear_buffer(&self.masks_buf, 0, None); + enc.clear_buffer(&self.flags_buf, 0, None); + self.gpu.submit(enc); + } +} + +fn entry(binding: u32, b: &wgpu::Buffer) -> wgpu::BindGroupEntry<'_> { + wgpu::BindGroupEntry { binding, resource: b.as_entire_binding() } +} diff --git a/src/wgpu/device.rs b/src/wgpu/device.rs new file mode 100644 index 000000000..92ee8a030 --- /dev/null +++ b/src/wgpu/device.rs @@ -0,0 +1,247 @@ +//! The adapter, device and queue the boards share, with the limits the +//! raster kernels need and the errors wgpu reports asynchronously. + +use std::fmt; +use std::sync::Arc; +use std::time::Duration; + +use parking_lot::Mutex; + +/// How to open the GPU. All fields have defaults; `backend` is the only +/// one users set. +#[derive(Clone, Debug)] +pub struct GpuOptions { + /// `auto`, `vulkan`, `metal`, `dx12` or `gl`. + pub backend: String, + /// Prefer the discrete adapter. + pub high_performance: bool, + /// Ask for the optional features the kernels can use when present + /// (subgroups, 64-bit integers and atomics, timestamps). + pub optional_features: bool, + /// Prefer a software adapter (lavapipe, SwiftShader) when one exists: + /// what CI runs the goldens on. + pub software: bool, +} + +impl Default for GpuOptions { + fn default() -> Self { + GpuOptions { + backend: "auto".into(), + high_performance: true, + optional_features: true, + software: false, + } + } +} + +/// What the device turned out to have, for logs and for kernels that +/// branch on it. +#[derive(Clone, Debug)] +pub struct Caps { + pub backend: wgpu::Backend, + pub adapter: String, + pub device_type: wgpu::DeviceType, + pub driver: String, + pub max_storage_binding: u64, + pub max_buffer: u64, + pub max_workgroups: u32, + pub subgroups: bool, + pub int64: bool, + pub int64_atomic_min_max: bool, + pub timestamps: bool, +} + +impl fmt::Display for Caps { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:?} {} ({:?}, {}) storage {} MiB buffer {} MiB groups {} subgroups {} int64 {} int64-atomics {} timestamps {}", + self.backend, + self.adapter, + self.device_type, + self.driver, + self.max_storage_binding >> 20, + self.max_buffer >> 20, + self.max_workgroups, + self.subgroups, + self.int64, + self.int64_atomic_min_max, + self.timestamps, + ) + } +} + +/// One GPU for the process. Boards hold an `Arc`; they share the +/// queue, so work submitted by two boards is ordered by submission. +pub struct Gpu { + /// The instance and adapter the device came from: a window's surface + /// for presenting has to come from the same instance, and the adapter + /// says whether it can present to it (`src/wgpu/present.rs`). + pub instance: wgpu::Instance, + pub adapter: wgpu::Adapter, + pub device: wgpu::Device, + pub queue: wgpu::Queue, + pub caps: Caps, + /// The first asynchronous error (validation outside an error scope, or + /// device loss). Checked by `check()`; a kernel bug shows up here. + error: Arc>>, +} + +impl Gpu { + /// Open the GPU `opts` asks for. Fails with the reasons each candidate + /// backend gave when none works. + pub fn new(opts: &GpuOptions) -> Result, String> { + let mut failures = Vec::new(); + for (name, backends) in candidates(&opts.backend)? { + match Self::open(backends, opts) { + Ok(gpu) => { + log::info!("wgpu: {}", gpu.caps); + return Ok(gpu); + } + Err(e) => failures.push(format!("{name}: {e}")), + } + } + Err(format!("no usable GPU for backend={}: {}", opts.backend, failures.join("; "))) + } + + fn open(backends: wgpu::Backends, opts: &GpuOptions) -> Result, String> { + let mut desc = wgpu::InstanceDescriptor::new_without_display_handle(); + desc.backends = backends; + let instance = wgpu::Instance::new(desc); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions { + power_preference: if opts.high_performance { + wgpu::PowerPreference::HighPerformance + } else { + wgpu::PowerPreference::None + }, + force_fallback_adapter: opts.software, + compatible_surface: None, + ..Default::default() + })) + .map_err(|e| format!("request adapter: {e}"))?; + let info = adapter.get_info(); + let have = adapter.features(); + let limits = adapter.limits(); + + // Features: only what is there. Kernels branch on `caps`. + let mut features = wgpu::Features::empty(); + if opts.optional_features { + for f in [ + wgpu::Features::SUBGROUP, + wgpu::Features::SHADER_INT64, + wgpu::Features::SHADER_INT64_ATOMIC_MIN_MAX, + wgpu::Features::TIMESTAMP_QUERY, + ] { + if have.contains(f) { + features |= f; + } + } + } + // Limits: the adapter's own for buffer sizes, so a 128 MiB board + // memory is one binding; the defaults elsewhere. + let mut required = wgpu::Limits::default(); + required.max_storage_buffer_binding_size = limits.max_storage_buffer_binding_size; + required.max_buffer_size = limits.max_buffer_size; + required.max_storage_buffers_per_shader_stage = + limits.max_storage_buffers_per_shader_stage.min(16).max(required.max_storage_buffers_per_shader_stage); + required.max_compute_workgroups_per_dimension = limits.max_compute_workgroups_per_dimension; + required.max_compute_invocations_per_workgroup = limits.max_compute_invocations_per_workgroup; + required.max_compute_workgroup_storage_size = limits.max_compute_workgroup_storage_size; + + let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { + label: Some("IRIS wgpu raster"), + required_features: features, + required_limits: required, + ..Default::default() + })) + .map_err(|e| format!("request device on {}: {e}", info.name))?; + + let error = Arc::new(Mutex::new(None)); + let sink = Arc::clone(&error); + device.on_uncaptured_error(Arc::new(move |e| { + let msg = format!("wgpu error: {e}"); + log::error!("{msg}"); + sink.lock().get_or_insert(msg); + })); + let sink = Arc::clone(&error); + device.set_device_lost_callback(move |reason, msg| { + let msg = format!("wgpu device lost ({reason:?}): {msg}"); + log::error!("{msg}"); + sink.lock().get_or_insert(msg); + }); + + let caps = Caps { + backend: info.backend, + adapter: info.name.clone(), + device_type: info.device_type, + driver: format!("{} {}", info.driver, info.driver_info), + max_storage_binding: limits.max_storage_buffer_binding_size, + max_buffer: limits.max_buffer_size, + max_workgroups: limits.max_compute_workgroups_per_dimension, + subgroups: features.contains(wgpu::Features::SUBGROUP), + int64: features.contains(wgpu::Features::SHADER_INT64), + int64_atomic_min_max: features.contains(wgpu::Features::SHADER_INT64_ATOMIC_MIN_MAX), + timestamps: features.contains(wgpu::Features::TIMESTAMP_QUERY), + }; + Ok(Arc::new(Gpu { instance, adapter, device, queue, caps, error })) + } + + /// The first asynchronous error, if any. Kernels that misbehave are + /// reported here, not at the call that queued them. + pub fn check(&self) -> Result<(), String> { + match self.error.lock().clone() { + Some(e) => Err(e), + None => Ok(()), + } + } + + /// Submit one command buffer. + pub fn submit(&self, encoder: wgpu::CommandEncoder) -> wgpu::SubmissionIndex { + self.queue.submit([encoder.finish()]) + } + + /// Block until `submission` has finished (or everything, with None). + pub fn wait(&self, submission: Option) -> Result<(), String> { + self.device + .poll(wgpu::PollType::Wait { submission_index: submission, timeout: Some(Duration::from_secs(30)) }) + .map(|_| ()) + .map_err(|e| format!("wait for GPU: {e}"))?; + self.check() + } + + /// Let callbacks run without blocking. + pub fn poll(&self) { + let _ = self.device.poll(wgpu::PollType::Poll); + } + + /// Run `f` after every submission so far has finished. The callback + /// runs on a wgpu thread or inside a later `poll`; keep it short. + pub fn on_done(&self, f: impl FnOnce() + Send + 'static) { + self.queue.on_submitted_work_done(f); + } + + /// A command encoder with a label. + pub fn encoder(&self, label: &str) -> wgpu::CommandEncoder { + self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some(label) }) + } + + /// A buffer of `size` bytes with `usage`, zeroed. + pub fn buffer(&self, label: &str, size: u64, usage: wgpu::BufferUsages) -> wgpu::Buffer { + self.device.create_buffer(&wgpu::BufferDescriptor { label: Some(label), size, usage, mapped_at_creation: false }) + } +} + +fn candidates(backend: &str) -> Result, String> { + Ok(match backend { + "vulkan" => vec![("vulkan", wgpu::Backends::VULKAN)], + "metal" => vec![("metal", wgpu::Backends::METAL)], + "dx12" => vec![("dx12", wgpu::Backends::DX12)], + "gl" => vec![("gl", wgpu::Backends::GL)], + "auto" if cfg!(target_os = "macos") => vec![("metal", wgpu::Backends::METAL)], + "auto" if cfg!(target_os = "windows") => { + vec![("vulkan", wgpu::Backends::VULKAN), ("dx12", wgpu::Backends::DX12)] + } + "auto" => vec![("vulkan", wgpu::Backends::VULKAN), ("gl", wgpu::Backends::GL)], + other => return Err(format!("backend={other}: want auto, vulkan, metal, dx12 or gl")), + }) +} diff --git a/src/wgpu/memory.rs b/src/wgpu/memory.rs new file mode 100644 index 000000000..c446cb87f --- /dev/null +++ b/src/wgpu/memory.rs @@ -0,0 +1,125 @@ +//! A board's memory on the GPU: one storage buffer nothing on the CPU +//! maps. Writes go up through the queue; reads come back through a staging +//! buffer and a wait. Both are ordered against the kernels by submission +//! order, so a readback issued after a batch sees that batch. + +use std::sync::Arc; + +use super::device::Gpu; + +pub struct GpuMemory { + gpu: Arc, + pub buffer: wgpu::Buffer, + size: u64, + staging: Staging, +} + +/// A readback buffer, grown on demand and reused: copies ranges of a GPU +/// buffer into itself, waits, and maps. +#[derive(Default)] +pub struct Staging { + buffer: Option, +} + +impl Staging { + /// Read `ranges` (offset, length; each 4-aligned) of `src` in one copy + /// and one wait, concatenated in order. Waits for every submission so + /// far. + pub fn read(&mut self, gpu: &Gpu, src: &wgpu::Buffer, ranges: &[(u64, u64)]) -> Result, String> { + let span: u64 = ranges.iter().map(|r| r.1).sum(); + if span == 0 { + return Ok(Vec::new()); + } + let staging = match &self.buffer { + Some(s) if s.size() >= span => s, + _ => { + let s = gpu.buffer("readback", span.max(1 << 16), wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST); + self.buffer.insert(s) + } + }; + let mut enc = gpu.encoder("readback"); + let mut at = 0; + for &(off, len) in ranges { + debug_assert!(off % 4 == 0 && len % 4 == 0 && off + len <= src.size()); + enc.copy_buffer_to_buffer(src, off, staging, at, len); + at += len; + } + let submission = gpu.submit(enc); + let (tx, rx) = std::sync::mpsc::sync_channel(1); + staging.slice(..span).map_async(wgpu::MapMode::Read, move |r| { + let _ = tx.send(r); + }); + gpu.wait(Some(submission))?; + rx.recv().map_err(|e| format!("readback callback: {e}"))?.map_err(|e| format!("map readback: {e}"))?; + let out = { + let view = staging.slice(..span).get_mapped_range().map_err(|e| format!("readback range: {e}"))?; + view.to_vec() + }; + staging.unmap(); + Ok(out) + } +} + +impl GpuMemory { + /// `size` bytes, zeroed, usable as read-write storage and as a copy + /// source and destination. + pub fn new(gpu: &Arc, label: &str, size: u64) -> Result { + if size > gpu.caps.max_storage_binding { + return Err(format!( + "{label}: {size} bytes do not fit one storage binding of {} bytes", + gpu.caps.max_storage_binding + )); + } + let buffer = gpu.buffer( + label, + size, + wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST, + ); + Ok(GpuMemory { gpu: Arc::clone(gpu), buffer, size, staging: Staging::default() }) + } + + pub fn size(&self) -> u64 { + self.size + } + + /// Queue a write of `bytes` at `offset`. Lands before anything + /// submitted after it. + pub fn write(&self, offset: u64, bytes: &[u8]) { + debug_assert!(offset + bytes.len() as u64 <= self.size); + self.gpu.queue.write_buffer(&self.buffer, offset, bytes); + } + + /// Zero `len` bytes at `offset`, ordered like a write. + pub fn clear(&self, offset: u64, len: u64) { + let mut enc = self.gpu.encoder("memory clear"); + enc.clear_buffer(&self.buffer, offset, Some(len)); + self.gpu.submit(enc); + } + + /// Read `len` bytes at `offset`, waiting for every submission so far. + /// `offset` and `len` are rounded out to 4 bytes. + pub fn read(&mut self, offset: u64, len: u64) -> Result, String> { + let start = offset & !3; + let end = (offset + len + 3) & !3; + let mut out = self.read_ranges(&[(start, end - start)])?; + let skip = (offset - start) as usize; + out.drain(..skip); + out.truncate(len as usize); + Ok(out) + } + + /// Read several ranges (offset, length; each 4-aligned) in one copy + /// and one wait, concatenated in order. + pub fn read_ranges(&mut self, ranges: &[(u64, u64)]) -> Result, String> { + self.staging.read(&self.gpu, &self.buffer, ranges) + } + + /// The whole memory. + pub fn read_all(&mut self) -> Result, String> { + self.read(0, self.size) + } + + pub fn gpu(&self) -> &Arc { + &self.gpu + } +} diff --git a/src/wgpu/mod.rs b/src/wgpu/mod.rs new file mode 100644 index 000000000..eb1cff4b0 --- /dev/null +++ b/src/wgpu/mod.rs @@ -0,0 +1,63 @@ +//! Host-GPU rasterisation on wgpu. +//! +//! The boards' raster units (IMPACT's RSS engines, the GR2's RE3s, the +//! VPro's raster unit) are replaced by compute kernels, and the board's +//! memory lives in a GPU buffer that nothing on the CPU maps: every read of +//! it is a readback. This module is the part every board shares; the +//! board-specific kernels and the glue that turns its register words into +//! batch records live in `src/dev//wgpu`. The design is in +//! `clanker/plans/wgpu-raster.md`. +//! +//! The shape of the work: +//! +//! - A **batch** is a run of primitives in submission order with the state +//! each one was issued under. The CPU side (`batch`) appends a fixed-size +//! *setup record* per primitive and a *state slot* per distinct state; +//! a record names its slot. A batch closes when something has to observe +//! memory (a readback, a frame end), when it is full, or on a time cap. +//! - The **bin** kernel (`shaders/bin.wgsl`) gives every 8x8 tile a bitmask +//! over the batch's primitives and appends each touched tile to a busy +//! list. Setting bits commutes, and the bit position is the primitive +//! index, so the tile's list is in submission order without a sort. +//! - The **raster** kernel (`shaders/raster.wgsl` around the board's +//! `arch_*` hooks) runs one 64-lane workgroup per busy tile, dispatched +//! indirectly from the busy count. The workgroup is the tile's only +//! writer for the batch, walks its mask in order and stores pixels +//! itself, so primitive order per pixel needs no atomics and no +//! cross-workgroup synchronisation. +//! +//! `device` opens the adapter and holds the queue; `shader` assembles WGSL +//! from the files on disk (`include_str!`, with `#include`); `tiles` is the +//! tile geometry and the buffer layouts both sides agree on; `memory` is a +//! board's memory on the GPU with uploads and readbacks; `batch` is the +//! recorder and the two dispatches. + +pub mod batch; +pub mod device; +pub mod memory; +pub mod present; +pub mod scanout; +pub mod shader; +pub mod tiles; + +#[cfg(test)] +mod tests; + +/// Library fragments a board's shaders may `#include`. +pub mod library { + use super::shader::Fragment; + + /// 64-bit integers on two u32 halves, with a correctly rounded + /// conversion to f32 (`shaders/i64.wgsl`). + pub const I64: Fragment = Fragment::new("i64.wgsl", include_str!("shaders/i64.wgsl")); + /// A correctly rounded f32 division (`shaders/float.wgsl`). + pub const FLOAT: Fragment = Fragment::new("float.wgsl", include_str!("shaders/float.wgsl")); +} + +pub use batch::{ArchShaders, Batch, BatchPipelines, Full}; +pub use device::{Gpu, GpuOptions}; +pub use memory::{GpuMemory, Staging}; +pub use present::{Frame, Placement, Presenter}; +pub use scanout::{Scanout, ScanoutPipeline, ScanoutShaders}; +pub use shader::{Fragment, Source}; +pub use tiles::{TileBox, TileGrid, TILE_H, TILE_W}; diff --git a/src/wgpu/present.rs b/src/wgpu/present.rs new file mode 100644 index 000000000..da925e1b2 --- /dev/null +++ b/src/wgpu/present.rs @@ -0,0 +1,447 @@ +//! Presenting frames into a window through wgpu. +//! +//! A [`Presenter`] owns the window's surface on a [`Gpu`] and draws three +//! layers into it each frame: the frame, letterboxed into the window above +//! the status bar; the debug overlay, alpha-blended over the frame; and the +//! status bar. The frame comes either from the CPU ([`Frame::Cpu`], uploaded) +//! or from a buffer already on the device ([`Frame::Gpu`], a board's +//! scan-out output, unpacked by a small kernel with no readback). Sampling +//! is nearest at an integer scale and linear otherwise, as the GL renderer +//! does. +//! +//! The surface must come from the instance the device was opened on; a +//! board that draws on a [`Gpu`] shares it with the window this way +//! (`GfxDisplay::gpu`). Threading follows the GL renderer's rules: make the +//! presenter on the thread that owns the window, present from the board's +//! display thread. + +use std::sync::Arc; + +use bytemuck::{Pod, Zeroable}; + +use super::shader::{Fragment, Source}; +use super::Gpu; + +const PRESENT: Fragment = Fragment::new("present.wgsl", include_str!("shaders/present.wgsl")); +const UNPACK: Fragment = Fragment::new("unpack.wgsl", include_str!("shaders/unpack.wgsl")); + +/// Where a quad goes (normalised device coordinates: x0, top, x1, bottom) +/// and the texture coordinates it shows (u0, v0, u1, v1). +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable, Default, PartialEq)] +struct Quad { + rect: [f32; 4], + uv: [f32; 4], +} + +/// A frame to present. +pub enum Frame<'a> { + /// Words with R in the low byte and A in the high (0xAABBGGRR), rows of + /// `stride` words: what the boards' `Rex3Screen.rgba` holds. + Cpu { rgba: &'a [u32], stride: usize, width: u32, height: u32 }, + /// 0x00RRGGBB words on this presenter's device, rows of `width`. + Gpu { buffer: &'a wgpu::Buffer, width: u32, height: u32 }, +} + +/// One drawn layer: its texture, the quad's uniform and bind groups for +/// both samplers. +struct Layer { + texture: wgpu::Texture, + uniform: wgpu::Buffer, + nearest: wgpu::BindGroup, + linear: wgpu::BindGroup, + width: u32, + height: u32, + last: Quad, +} + +/// The window's surface and what draws into it. +pub struct Presenter { + gpu: Arc, + surface: wgpu::Surface<'static>, + config: wgpu::SurfaceConfiguration, + opaque: wgpu::RenderPipeline, + blended: wgpu::RenderPipeline, + frame: Layer, + overlay: Layer, + status: Layer, + unpack: wgpu::ComputePipeline, + unpack_layout: wgpu::BindGroupLayout, + unpack_size: wgpu::Buffer, + /// The bind group for the last device frame unpacked, and its buffer. + unpack_group: Option<(wgpu::Buffer, wgpu::BindGroup)>, + status_rows: u32, + pub frames: u64, +} + +/// What a frame's placement in the window comes to. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Placement { + /// The frame's quad in window pixels: x0, y0 (top), x1, y1. + pub frame: [f32; 4], + /// The status bar's height in window pixels, under the frame. + pub status: f32, + /// The scale the frame is drawn at. + pub scale: f32, +} + +impl Placement { + /// The GL renderer's letterbox: the frame and the status bar under it + /// scaled together to fit the window, centred, the left and top edges + /// on whole pixels. + pub fn letterbox(width: f32, height: f32, status_rows: f32, win_w: f32, win_h: f32) -> Placement { + let total_h = height + status_rows; + let scale = (win_w / width).min(win_h / total_h); + let (w, h, sb) = (width * scale, height * scale, status_rows * scale); + let x0 = ((win_w - w) * 0.5).floor(); + let y0 = ((win_h - h - sb) * 0.5).floor(); + Placement { frame: [x0, y0, x0 + w, y0 + h], status: sb, scale } + } +} + +impl Presenter { + /// A presenter for `window` on `gpu`, its surface `width` x `height` + /// pixels. Frames are at most `frame_w` x `frame_h`; the status bar is + /// `status_w` x `status_rows`. + pub fn new( + gpu: &Arc, + window: impl Into>, + (width, height): (u32, u32), + (frame_w, frame_h): (u32, u32), + (status_w, status_rows): (u32, u32), + ) -> Result { + let surface = gpu.instance.create_surface(window).map_err(|e| format!("create surface: {e}"))?; + if !gpu.adapter.is_surface_supported(&surface) { + return Err(format!("{} cannot present to this window", gpu.caps.adapter)); + } + let caps = surface.get_capabilities(&gpu.adapter); + // A plain (non-sRGB) format: the frame's bytes are display values + // already, as on the GL path; an sRGB target would encode them again. + let format = caps + .formats + .iter() + .copied() + .find(|f| !f.is_srgb()) + .or_else(|| caps.formats.first().copied()) + .ok_or("the surface offers no format")?; + let present_mode = if caps.present_modes.contains(&wgpu::PresentMode::Fifo) { wgpu::PresentMode::Fifo } else { caps.present_modes[0] }; + let config = wgpu::SurfaceConfiguration { + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + format, + width: width.max(1), + height: height.max(1), + present_mode, + desired_maximum_frame_latency: 2, + alpha_mode: caps.alpha_modes[0], + view_formats: vec![], + color_space: Default::default(), + }; + surface.configure(&gpu.device, &config); + + let device = &gpu.device; + let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("present layer"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::VERTEX, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }, + ], + }); + let module = Source::new("present").body(PRESENT).compile(device)?; + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("present"), + bind_group_layouts: &[Some(&layout)], + immediate_size: 0, + }); + let pipeline = |blend: Option| { + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("present"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { module: &module, entry_point: Some("quad_vs"), compilation_options: Default::default(), buffers: &[] }, + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("quad_fs"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { format, blend, write_mask: wgpu::ColorWrites::ALL })], + }), + multiview_mask: None, + cache: None, + }) + }; + let opaque = pipeline(None); + let blended = pipeline(Some(wgpu::BlendState::ALPHA_BLENDING)); + + let sampler = |filter: wgpu::FilterMode| { + device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("present"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + mag_filter: filter, + min_filter: filter, + ..Default::default() + }) + }; + let (nearest, linear) = (sampler(wgpu::FilterMode::Nearest), sampler(wgpu::FilterMode::Linear)); + let make_layer = |label: &str, w: u32, h: u32, storage: bool| { + let mut usage = wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST; + if storage { + usage |= wgpu::TextureUsages::STORAGE_BINDING; + } + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage, + view_formats: &[], + }); + let view = texture.create_view(&Default::default()); + let uniform = gpu.buffer(label, size_of::() as u64, wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST); + let group = |s: &wgpu::Sampler| { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(label), + layout: &layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(&view) }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(s) }, + wgpu::BindGroupEntry { binding: 2, resource: uniform.as_entire_binding() }, + ], + }) + }; + let (n, l) = (group(&nearest), group(&linear)); + Layer { texture, uniform, nearest: n, linear: l, width: w, height: h, last: Quad::default() } + }; + let frame = make_layer("present frame", frame_w, frame_h, true); + let overlay = make_layer("present overlay", frame_w, frame_h, false); + let status = make_layer("present status bar", status_w, status_rows.max(1), false); + + let unpack_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("present unpack"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Storage { read_only: true }, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::StorageTexture { + access: wgpu::StorageTextureAccess::WriteOnly, + format: wgpu::TextureFormat::Rgba8Unorm, + view_dimension: wgpu::TextureViewDimension::D2, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }, + ], + }); + let unpack = Source::new("present unpack").body(UNPACK).compute_pipeline(device, "unpack", &[&unpack_layout])?; + let unpack_size = gpu.buffer("present unpack size", 16, wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST); + gpu.check()?; + Ok(Presenter { + gpu: Arc::clone(gpu), + surface, + config, + opaque, + blended, + frame, + overlay, + status, + unpack, + unpack_layout, + unpack_size, + unpack_group: None, + status_rows, + frames: 0, + }) + } + + /// The device this presenter draws with. + pub fn gpu(&self) -> &Arc { + &self.gpu + } + + /// The window's new size in pixels. + pub fn resize(&mut self, width: u32, height: u32) { + let (w, h) = (width.max(1), height.max(1)); + if (w, h) != (self.config.width, self.config.height) { + self.config.width = w; + self.config.height = h; + self.surface.configure(&self.gpu.device, &self.config); + } + } + + /// Words of 0xAABBGGRR, rows of `stride`, into `layer` at the top left. + fn upload(&self, layer: &Layer, rgba: &[u32], stride: usize, width: u32, height: u32) { + let (w, h) = (width.min(layer.width), height.min(layer.height)); + if w == 0 || h == 0 || rgba.len() < (h as usize - 1) * stride + w as usize { + return; + } + self.gpu.queue.write_texture( + wgpu::TexelCopyTextureInfo { texture: &layer.texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + bytemuck::cast_slice(&rgba[..(h as usize - 1) * stride + w as usize]), + wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(4 * stride as u32), rows_per_image: Some(h) }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + } + + /// Draw `frame` letterboxed, `overlay` (0xAABBGGRR, stride `stride`) + /// over it when given, and `status` (stride `stride`, `status_rows` + /// rows) under it, and present. A frame the surface cannot take now + /// (minimised, being resized) is skipped. + pub fn present(&mut self, frame: Frame<'_>, overlay: Option<&[u32]>, status: &[u32], stride: usize) -> Result<(), String> { + let (fw, fh) = match frame { + Frame::Cpu { width, height, .. } | Frame::Gpu { width, height, .. } => (width.min(self.frame.width), height.min(self.frame.height)), + }; + if fw == 0 || fh == 0 { + return Ok(()); + } + let target = match self.surface.get_current_texture() { + wgpu::CurrentSurfaceTexture::Success(t) => t, + wgpu::CurrentSurfaceTexture::Suboptimal(t) => { + // Draw this one; configure again for the next. + self.surface.configure(&self.gpu.device, &self.config); + t + } + wgpu::CurrentSurfaceTexture::Outdated | wgpu::CurrentSurfaceTexture::Lost => { + self.surface.configure(&self.gpu.device, &self.config); + return Ok(()); + } + wgpu::CurrentSurfaceTexture::Timeout | wgpu::CurrentSurfaceTexture::Occluded => return Ok(()), + wgpu::CurrentSurfaceTexture::Validation => return self.gpu.check(), + }; + + let mut enc = self.gpu.encoder("present"); + match frame { + Frame::Cpu { rgba, stride, .. } => self.upload(&self.frame, rgba, stride, fw, fh), + Frame::Gpu { buffer, .. } => { + if self.unpack_group.as_ref().is_none_or(|(b, _)| b != buffer) { + let view = self.frame.texture.create_view(&Default::default()); + let group = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("present unpack"), + layout: &self.unpack_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: buffer.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&view) }, + wgpu::BindGroupEntry { binding: 2, resource: self.unpack_size.as_entire_binding() }, + ], + }); + self.unpack_group = Some((buffer.clone(), group)); + } + self.gpu.queue.write_buffer(&self.unpack_size, 0, bytemuck::cast_slice(&[fw, fh, 0u32, 0u32])); + let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("present unpack"), timestamp_writes: None }); + pass.set_pipeline(&self.unpack); + pass.set_bind_group(0, &self.unpack_group.as_ref().expect("the unpack group").1, &[]); + pass.dispatch_workgroups(fw.div_ceil(8), fh.div_ceil(8), 1); + } + } + if let Some(o) = overlay { + self.upload(&self.overlay, o, stride, fw, fh); + } + self.upload(&self.status, status, stride, fw, self.status_rows); + + let p = Placement::letterbox(fw as f32, fh as f32, self.status_rows as f32, self.config.width as f32, self.config.height as f32); + let integer = p.scale == p.scale.round() && p.scale >= 1.0; + let r = p.frame; + let win = (self.config.width as f32, self.config.height as f32); + let q = &self.gpu.queue; + place(q, win, &mut self.frame, r, fw, fh); + place(q, win, &mut self.overlay, r, fw, fh); + place(q, win, &mut self.status, [r[0], r[3], r[2], r[3] + p.status], fw, self.status_rows); + + let view = target.texture.create_view(&Default::default()); + { + let mut pass = enc.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("present"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &view, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), store: wgpu::StoreOp::Store }, + })], + ..Default::default() + }); + fn group(l: &Layer, integer: bool) -> &wgpu::BindGroup { + if integer { &l.nearest } else { &l.linear } + } + pass.set_pipeline(&self.opaque); + pass.set_bind_group(0, group(&self.frame, integer), &[]); + pass.draw(0..6, 0..1); + if overlay.is_some() { + pass.set_pipeline(&self.blended); + pass.set_bind_group(0, group(&self.overlay, integer), &[]); + pass.draw(0..6, 0..1); + pass.set_pipeline(&self.opaque); + } + pass.set_bind_group(0, group(&self.status, integer), &[]); + pass.draw(0..6, 0..1); + } + self.gpu.submit(enc); + self.gpu.queue.present(target); + self.frames += 1; + self.gpu.check() + } +} + +/// Point `layer`'s quad at window pixels `r` (x0, y0, x1, y1) of a window +/// `win` pixels in size, showing its top-left `w` x `h` texels; the uniform +/// is written only when that changes. +fn place(queue: &wgpu::Queue, (ww, wh): (f32, f32), layer: &mut Layer, r: [f32; 4], w: u32, h: u32) { + let ndc_x = |x: f32| x / ww * 2.0 - 1.0; + let ndc_y = |y: f32| 1.0 - y / wh * 2.0; + let q = Quad { + rect: [ndc_x(r[0]), ndc_y(r[1]), ndc_x(r[2]), ndc_y(r[3])], + uv: [0.0, 0.0, w as f32 / layer.width as f32, h as f32 / layer.height as f32], + }; + if q != layer.last { + queue.write_buffer(&layer.uniform, 0, bytemuck::bytes_of(&q)); + layer.last = q; + } +} + +#[cfg(test)] +mod tests { + use super::Placement; + + /// The GL renderer's letterbox: a 1280 x 1024 frame with a 16-row bar in + /// a window of exactly that size is drawn at 1x from the top left, and in + /// a wider window it is centred at the height's scale. + #[test] + fn the_letterbox_matches_the_gl_renderer() { + let p = Placement::letterbox(1280.0, 1024.0, 16.0, 1280.0, 1040.0); + assert_eq!((p.frame, p.status, p.scale), ([0.0, 0.0, 1280.0, 1024.0], 16.0, 1.0)); + let p = Placement::letterbox(1280.0, 1024.0, 16.0, 3000.0, 2080.0); + assert_eq!((p.frame, p.status, p.scale), ([220.0, 0.0, 2780.0, 2048.0], 32.0, 2.0)); + } +} diff --git a/src/wgpu/scanout.rs b/src/wgpu/scanout.rs new file mode 100644 index 000000000..b48c80205 --- /dev/null +++ b/src/wgpu/scanout.rs @@ -0,0 +1,182 @@ +//! The display's scanout on the GPU: a board's compose, one lane per screen +//! pixel, from its memory into a frame buffer on the GPU, read back for +//! whoever shows it until the window shares the device. +//! +//! A board provides a fragment defining `scanout_pixel(x, y) -> u32` +//! (0x00RRGGBB), the display slot its kernel reads at group 1 binding 0 +//! (the scan-out registers and tables of the frame, as the board packs +//! them), and the bind group layouts of its memory from group 2 up, bound +//! as its raster kernels bind them. `shaders/scanout.wgsl` is the driver. + +use std::sync::Arc; + +use super::device::Gpu; +use super::memory::Staging; +use super::shader::{Fragment, Source}; + +const DRIVER: Fragment = Fragment::new("scanout.wgsl", include_str!("shaders/scanout.wgsl")); + +/// A board's side of the scanout kernel. +pub struct ScanoutShaders<'a> { + pub label: &'a str, + /// Fragments defining `scanout_pixel` and declaring the display slot + /// at group 1 binding 0. + pub body: &'a [Fragment], + pub library: &'a [Fragment], + pub defines: &'a [(&'a str, u32)], + /// Bind group layouts for groups 2 and up. + pub layouts: &'a [&'a wgpu::BindGroupLayout], +} + +/// The kernel and the layouts of groups 0 and 1. +pub struct ScanoutPipeline { + pub frame_layout: wgpu::BindGroupLayout, + pub slot_layout: wgpu::BindGroupLayout, + pub pipeline: wgpu::ComputePipeline, +} + +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct Header { + width: u32, + height: u32, + stride: u32, + pad: u32, +} + +fn entry(binding: u32, ty: wgpu::BufferBindingType) -> wgpu::BindGroupLayoutEntry { + wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty, has_dynamic_offset: false, min_binding_size: None }, + count: None, + } +} + +impl ScanoutPipeline { + pub fn new(gpu: &Gpu, arch: &ScanoutShaders<'_>) -> Result { + let frame_layout = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("scanout frame"), + entries: &[entry(0, wgpu::BufferBindingType::Uniform), entry(1, wgpu::BufferBindingType::Storage { read_only: false })], + }); + let slot_layout = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("scanout slot"), + entries: &[entry(0, wgpu::BufferBindingType::Storage { read_only: true })], + }); + let mut layouts: Vec<&wgpu::BindGroupLayout> = vec![&frame_layout, &slot_layout]; + layouts.extend_from_slice(arch.layouts); + let mut src = Source::new(&format!("{} scanout", arch.label)); + for (k, v) in arch.defines { + src = src.define(k, *v); + } + for f in arch.library { + src = src.library(*f); + } + src = src.body(DRIVER); + for f in arch.body { + src = src.body(*f); + } + let pipeline = src.compute_pipeline(&gpu.device, "scanout", &layouts)?; + Ok(ScanoutPipeline { frame_layout, slot_layout, pipeline }) + } +} + +/// A frame on the GPU and the slot the kernel reads, with a readback. +pub struct Scanout { + gpu: Arc, + header_buf: wgpu::Buffer, + out_buf: wgpu::Buffer, + slot_buf: wgpu::Buffer, + frame_group: wgpu::BindGroup, + slot_group: wgpu::BindGroup, + staging: Staging, + max_pixels: u32, + /// Frames composed and bytes read back so far. + pub frames: u64, + pub read_back_bytes: u64, +} + +impl Scanout { + /// Room for `max_pixels` and a slot of `slot_bytes`. + pub fn new(gpu: &Arc, pipeline: &ScanoutPipeline, max_pixels: u32, slot_bytes: u64) -> Scanout { + let header_buf = gpu.buffer("scanout header", 16, wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST); + let out_buf = gpu.buffer("scanout frame", max_pixels.max(1) as u64 * 4, wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC); + let slot_buf = gpu.buffer("scanout slot", slot_bytes.max(16), wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST); + let frame_group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("scanout frame"), + layout: &pipeline.frame_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: header_buf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: out_buf.as_entire_binding() }, + ], + }); + let slot_group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("scanout slot"), + layout: &pipeline.slot_layout, + entries: &[wgpu::BindGroupEntry { binding: 0, resource: slot_buf.as_entire_binding() }], + }); + Scanout { gpu: Arc::clone(gpu), header_buf, out_buf, slot_buf, frame_group, slot_group, staging: Staging::default(), max_pixels, frames: 0, read_back_bytes: 0 } + } + + /// The device the scan-out runs on. + pub fn gpu(&self) -> &Arc { + &self.gpu + } + + /// Compose a `width` x `height` frame under `slot` with `groups` bound + /// from group 2 up, and read it back into `out` (rows of `width`). + /// Ordered after every submission so far, so it shows whole batches. + pub fn run(&mut self, pipeline: &ScanoutPipeline, width: u32, height: u32, slot: &[u8], groups: &[&wgpu::BindGroup], out: &mut [u32]) -> Result<(), String> { + let pixels = width as u64 * height as u64; + if pixels == 0 { + return Ok(()); + } + if out.len() < pixels as usize { + return Err(format!("scanout: a {width} x {height} frame does not fit")); + } + self.compose(pipeline, width, height, slot, groups)?; + let bytes = self.staging.read(&self.gpu, &self.out_buf, &[(0, pixels * 4)])?; + out[..pixels as usize].copy_from_slice(bytemuck::cast_slice(&bytes)); + self.read_back_bytes += bytes.len() as u64; + self.gpu.check() + } + + /// The buffer frames are composed into: 0x00RRGGBB words, rows of the + /// last frame's width. + pub fn output(&self) -> &wgpu::Buffer { + &self.out_buf + } + + /// Compose a frame as `run` does into [`output`](Self::output) and + /// submit it, with no readback: a renderer on the same device presents + /// it from there. + pub fn compose(&mut self, pipeline: &ScanoutPipeline, width: u32, height: u32, slot: &[u8], groups: &[&wgpu::BindGroup]) -> Result<(), String> { + let pixels = width as u64 * height as u64; + if pixels == 0 { + return Ok(()); + } + if pixels > self.max_pixels as u64 { + return Err(format!("scanout: a {width} x {height} frame does not fit")); + } + if slot.len() as u64 > self.slot_buf.size() { + return Err(format!("scanout: a slot of {} bytes does not fit {}", slot.len(), self.slot_buf.size())); + } + let q = &self.gpu.queue; + q.write_buffer(&self.header_buf, 0, bytemuck::bytes_of(&Header { width, height, stride: width, pad: 0 })); + q.write_buffer(&self.slot_buf, 0, slot); + let mut enc = self.gpu.encoder("scanout"); + { + let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("scanout"), timestamp_writes: None }); + pass.set_pipeline(&pipeline.pipeline); + pass.set_bind_group(0, &self.frame_group, &[]); + pass.set_bind_group(1, &self.slot_group, &[]); + for (i, g) in groups.iter().enumerate() { + pass.set_bind_group(2 + i as u32, *g, &[]); + } + pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1); + } + self.gpu.submit(enc); + self.frames += 1; + self.gpu.check() + } +} diff --git a/src/wgpu/shader.rs b/src/wgpu/shader.rs new file mode 100644 index 000000000..a002aed3d --- /dev/null +++ b/src/wgpu/shader.rs @@ -0,0 +1,198 @@ +//! WGSL assembled from source files on disk. +//! +//! The kernels are kept as `.wgsl` files next to the code that uses them +//! and compiled into the binary with `include_str!`. WGSL has no include, +//! so a `Source` is a list of named fragments; a line `#include "name"` +//! splices the fragment of that name in place, once. Constants both sides +//! must agree on (tile size, mask words) come in through `define`, which +//! prepends `const NAME: u32 = value;`. Every fragment is preceded by a +//! comment naming it, so the line numbers in naga's errors can be read +//! back to a file. + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +/// A named piece of WGSL, usually `Fragment::new("x.wgsl", include_str!("x.wgsl"))`. +#[derive(Clone, Copy, Debug)] +pub struct Fragment { + pub name: &'static str, + pub text: &'static str, +} + +impl Fragment { + pub const fn new(name: &'static str, text: &'static str) -> Fragment { + Fragment { name, text } + } +} + +/// A shader under assembly. +#[derive(Clone, Debug, Default)] +pub struct Source { + label: String, + defines: Vec<(String, String)>, + /// Fragments by name, for `#include`. + library: BTreeMap<&'static str, &'static str>, + /// The fragments that form the module body, in order. + body: Vec, +} + +impl Source { + pub fn new(label: &str) -> Source { + Source { label: label.to_owned(), ..Default::default() } + } + + /// A fragment `#include` can name. Including is not emitting. + pub fn library(mut self, f: Fragment) -> Source { + self.library.insert(f.name, f.text); + self + } + + /// A fragment of the module body. + pub fn body(mut self, f: Fragment) -> Source { + self.library.insert(f.name, f.text); + self.body.push(f); + self + } + + /// `const NAME: u32 = value;` at the top. + pub fn define(mut self, name: &str, value: u32) -> Source { + self.defines.push((name.to_owned(), format!("{value}u"))); + self + } + + /// `const NAME: i32 = value;` at the top. + pub fn define_i32(mut self, name: &str, value: i32) -> Source { + self.defines.push((name.to_owned(), format!("{value}i"))); + self + } + + /// The assembled WGSL. + pub fn text(&self) -> Result { + let mut out = String::new(); + for (name, value) in &self.defines { + let ty = if value.ends_with('i') { "i32" } else { "u32" }; + let _ = writeln!(out, "const {name}: {ty} = {value};"); + } + let mut included = Vec::new(); + for f in &self.body { + self.splice(f.name, f.text, &mut out, &mut included, 0)?; + } + Ok(out) + } + + fn splice( + &self, + name: &str, + text: &str, + out: &mut String, + included: &mut Vec, + depth: usize, + ) -> Result<(), String> { + if depth > 16 { + return Err(format!("{}: include nesting too deep at {name}", self.label)); + } + if included.iter().any(|n| n == name) { + return Ok(()); + } + included.push(name.to_owned()); + let _ = writeln!(out, "// ---- {name}"); + for line in text.lines() { + let trimmed = line.trim_start(); + if let Some(rest) = trimmed.strip_prefix("#include") { + let inc = rest.trim().trim_matches('"'); + let Some(body) = self.library.get(inc) else { + return Err(format!("{}: {name} includes unknown fragment {inc:?}", self.label)); + }; + self.splice(inc, body, out, included, depth + 1)?; + let _ = writeln!(out, "// ---- {name} (continued)"); + } else { + out.push_str(line); + out.push('\n'); + } + } + Ok(()) + } + + /// Compile. A validation error comes back as text with the composed + /// source's line numbers, not as a panic. + pub fn compile(&self, device: &wgpu::Device) -> Result { + let text = self.text()?; + // IRIS_WGPU_DUMP=: the composed WGSL of every kernel, for + // offline tools. + if let Ok(dir) = std::env::var("IRIS_WGPU_DUMP") { + let name = self.label.replace(['/', ' '], "_"); + let _ = std::fs::create_dir_all(&dir); + let _ = std::fs::write(format!("{dir}/{name}.wgsl"), &text); + } + let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some(&self.label), + source: wgpu::ShaderSource::Wgsl(text.as_str().into()), + }); + if let Some(e) = pollster::block_on(scope.pop()) { + return Err(format!("{}: {e}\n{}", self.label, numbered(&text))); + } + Ok(module) + } + + /// A compute pipeline for `entry` in this source, with the given bind + /// group layouts. + pub fn compute_pipeline( + &self, + device: &wgpu::Device, + entry: &str, + layouts: &[&wgpu::BindGroupLayout], + ) -> Result { + let module = self.compile(device)?; + let groups: Vec> = layouts.iter().map(|l| Some(*l)).collect(); + let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some(&self.label), + bind_group_layouts: &groups, + immediate_size: 0, + }); + let scope = device.push_error_scope(wgpu::ErrorFilter::Validation); + let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some(&format!("{}::{entry}", self.label)), + layout: Some(&layout), + module: &module, + entry_point: Some(entry), + compilation_options: Default::default(), + cache: None, + }); + if let Some(e) = pollster::block_on(scope.pop()) { + return Err(format!("{}::{entry}: {e}", self.label)); + } + Ok(pipeline) + } +} + +fn numbered(text: &str) -> String { + let mut s = String::new(); + for (i, l) in text.lines().enumerate() { + let _ = writeln!(s, "{:5} {l}", i + 1); + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn includes_splice_once_and_defines_lead() { + let s = Source::new("t") + .define("TILE_W", 8) + .library(Fragment::new("a.wgsl", "fn a() -> u32 { return 1u; }")) + .body(Fragment::new("m.wgsl", "#include \"a.wgsl\"\n#include \"a.wgsl\"\nfn m() -> u32 { return a() + TILE_W; }")); + let t = s.text().unwrap(); + assert!(t.starts_with("const TILE_W: u32 = 8u;\n")); + assert_eq!(t.matches("fn a()").count(), 1); + assert!(t.contains("// ---- a.wgsl")); + } + + #[test] + fn unknown_include_is_an_error() { + let s = Source::new("t").body(Fragment::new("m.wgsl", "#include \"nope.wgsl\"")); + assert!(s.text().unwrap_err().contains("nope.wgsl")); + } +} diff --git a/src/wgpu/shaders/bin.wgsl b/src/wgpu/shaders/bin.wgsl new file mode 100644 index 000000000..191049fb1 --- /dev/null +++ b/src/wgpu/shaders/bin.wgsl @@ -0,0 +1,47 @@ +// The bin pass: one workgroup per primitive, lanes over the rows of its +// tile box. A lane sets the primitive's bit in each tile's mask and puts +// a tile it is the first to touch on the busy list. Nothing here depends +// on order: the bit position is the primitive index. +// +// `bin_accept(p, tx, ty)` is the board's hook: true when primitive p may +// cover tile (tx, ty). The default accepts the whole box; a board can +// test its edge functions at the tile's corners. + +#include "common.wgsl" + +// The raster pass's indirect dispatch arguments, bound only here: a +// buffer cannot be read-write storage and the indirect source in one +// dispatch. x and y are folded so the busy count fits under MAX_GROUPS. +struct DispatchArgs { + x: atomic, + y: atomic, + z: u32, + pad: u32, +} +@group(0) @binding(5) var args: DispatchArgs; + +@compute @workgroup_size(64) +fn bin(@builtin(workgroup_id) wg: vec3, @builtin(local_invocation_index) lane: u32) { + let p = wg.x; + if p >= batch.prim_count { + return; + } + let b = prim_boxes[p]; + let word = p / 32u; + let bit = 1u << (p % 32u); + for (var ty = b.y + lane; ty <= b.w; ty += 64u) { + for (var tx = b.x; tx <= b.z; tx++) { + if !bin_accept(p, tx, ty) { + continue; + } + let t = tile_index(tx, ty); + atomicOr(&masks[t * MASK_WORDS + word], bit); + if atomicExchange(&busy_flags[t], 1u) == 0u { + let i = atomicAdd(&busy.count, 1u); + busy.tiles[i] = t; + atomicMax(&args.x, min(i + 1u, MAX_GROUPS)); + atomicMax(&args.y, i / MAX_GROUPS + 1u); + } + } + } +} diff --git a/src/wgpu/shaders/bin_accept_all.wgsl b/src/wgpu/shaders/bin_accept_all.wgsl new file mode 100644 index 000000000..e0b34367f --- /dev/null +++ b/src/wgpu/shaders/bin_accept_all.wgsl @@ -0,0 +1,4 @@ +// The default bin hook: every tile of the box. +fn bin_accept(p: u32, tx: u32, ty: u32) -> bool { + return true; +} diff --git a/src/wgpu/shaders/common.wgsl b/src/wgpu/shaders/common.wgsl new file mode 100644 index 000000000..1592a9191 --- /dev/null +++ b/src/wgpu/shaders/common.wgsl @@ -0,0 +1,36 @@ +// Batch layout shared by the bin and raster kernels. src/wgpu/tiles.rs +// defines the same structs; TILE_W, TILE_H, MASK_WORDS and MAX_GROUPS +// arrive as defines. + +struct BatchHeader { + prim_count: u32, + tile_cols: u32, + tile_rows: u32, + mask_words: u32, + origin_x: i32, + origin_y: i32, + pad0: u32, + pad1: u32, +} + +// The busy count and the busy tiles. +struct BusyList { + count: atomic, + pad0: u32, + pad1: u32, + pad2: u32, + tiles: array, +} + +@group(0) @binding(0) var batch: BatchHeader; +// MASK_WORDS words per tile, one bit per primitive of the batch. +@group(0) @binding(1) var masks: array>; +@group(0) @binding(2) var busy: BusyList; +// One word per tile: set when the tile is on the busy list. +@group(0) @binding(3) var busy_flags: array>; +// Per primitive, its inclusive tile box: x0, y0, x1, y1. +@group(0) @binding(4) var prim_boxes: array>; + +fn tile_index(tx: u32, ty: u32) -> u32 { + return ty * batch.tile_cols + tx; +} diff --git a/src/wgpu/shaders/float.wgsl b/src/wgpu/shaders/float.wgsl new file mode 100644 index 000000000..a45c18468 --- /dev/null +++ b/src/wgpu/shaders/float.wgsl @@ -0,0 +1,55 @@ +// Floating point the boards' models can be held to. Vulkan, Metal and +// D3D guarantee correctly rounded f32 add, subtract and multiply, but a +// quotient may be off by a few units in the last place (RADV computes +// it as a reciprocal and a multiply), and nothing stops a compiler fusing +// a multiply into an add. The kernels keep a product apart from a sum by +// multiplying it by a one the compiler cannot see (each board's `p()`), +// and divide here: the quotient of the two mantissas as integers, rounded +// to nearest even, which is what the host's division gives. + +// a / b, correctly rounded, for normal operands; the hardware's quotient +// for zero, denormal, infinite and NaN ones (the model's range tests +// drop those fragments either way). +fn div_exact(a: f32, b: f32) -> f32 { + let ia = bitcast(a); + let ib = bitcast(b); + let ea = (ia >> 23u) & 0xffu; + let eb = (ib >> 23u) & 0xffu; + if ea == 0u || eb == 0u || ea == 255u || eb == 255u { + return a / b; + } + let ma = (ia & 0x7fffffu) | 0x800000u; + let mb = (ib & 0x7fffffu) | 0x800000u; + // (ma << 26) / mb by restoring division: 26 or 27 quotient bits, the + // remainder below mb. + var rem: u32; + var q: u32; + if ma >= mb { + rem = ma - mb; + q = 1u; + } else { + rem = ma; + q = 0u; + } + for (var i = 0u; i < 26u; i++) { + rem = rem << 1u; + q = q << 1u; + if rem >= mb { + rem -= mb; + q |= 1u; + } + } + let n = select(26u, 27u, q >= (1u << 26u)); + let drop = n - 24u; + var m = q >> drop; + let round_bit = (q >> (drop - 1u)) & 1u; + let sticky = (q & ((1u << (drop - 1u)) - 1u)) != 0u || rem != 0u; + if round_bit == 1u && (sticky || (m & 1u) == 1u) { + m += 1u; + } + // a / b = m * 2^(n - 50 + ea - eb); m is at most 2^24, exact in f32, + // and the scale is a power of two. + let e = i32(ea) - i32(eb) + i32(n) - 50; + let sign = (ia ^ ib) & 0x80000000u; + return bitcast(bitcast(ldexp(f32(m), e)) | sign); +} diff --git a/src/wgpu/shaders/i64.wgsl b/src/wgpu/shaders/i64.wgsl new file mode 100644 index 000000000..1eb63da98 --- /dev/null +++ b/src/wgpu/shaders/i64.wgsl @@ -0,0 +1,93 @@ +// 64-bit integers on two 32-bit halves, for the boards' exact edge +// functions: WGSL has no i64 everywhere, and a driver's i64 to f32 +// conversion is not guaranteed to round as the host's does. Only what +// the kernels need: add, multiply by a small unsigned, compare with 0 +// and 1, and a correctly rounded (nearest even) conversion to f32, which +// is what Rust's `as f32` does. + +struct I64 { + lo: u32, + hi: u32, +} + +fn i64_add(a: I64, b: I64) -> I64 { + let lo = a.lo + b.lo; + let carry = select(0u, 1u, lo < a.lo); + return I64(lo, a.hi + b.hi + carry); +} + +// a * k for k below 2^16, wrapping as a two's complement product does. +fn i64_mul_u16(a: I64, k: u32) -> I64 { + let p0 = (a.lo & 0xffffu) * k; + let p1 = (a.lo >> 16u) * k; + let lo = p0 + (p1 << 16u); + let carry = select(0u, 1u, lo < p0); + return I64(lo, a.hi * k + (p1 >> 16u) + carry); +} + +fn i64_is_neg(a: I64) -> bool { + return (a.hi & 0x80000000u) != 0u; +} + +// a >= need, for need 0 or 1. +fn i64_ge_need(a: I64, need: u32) -> bool { + if i64_is_neg(a) { + return false; + } + return need == 0u || (a.lo | a.hi) != 0u; +} + +fn i64_neg(a: I64) -> I64 { + let lo = ~a.lo + 1u; + let carry = select(0u, 1u, lo == 0u); + return I64(lo, ~a.hi + carry); +} + +// f32(a), rounded to nearest even. +fn i64_to_f32(a: I64) -> f32 { + var m = a; + let neg = i64_is_neg(a); + if neg { + m = i64_neg(a); + } + if (m.lo | m.hi) == 0u { + return 0.0; + } + // The magnitude's bit length. + var n: u32; + if m.hi != 0u { + n = 64u - countLeadingZeros(m.hi); + } else { + n = 32u - countLeadingZeros(m.lo); + } + var r: f32; + if n <= 24u { + // Exact. + r = f32(m.lo); + } else { + // The top 25 bits (24 to keep and the rounding bit), and whether + // anything below them is set. + let s = n - 25u; + var top: u32; + var sticky: bool; + if s >= 32u { + let t = s - 32u; + top = m.hi >> t; + sticky = m.lo != 0u || (t != 0u && (m.hi & ((1u << t) - 1u)) != 0u); + } else if s == 0u { + top = m.lo; + sticky = false; + } else { + top = (m.lo >> s) | (m.hi << (32u - s)); + sticky = (m.lo & ((1u << s) - 1u)) != 0u; + } + var keep = top >> 1u; + let round_bit = (top & 1u) != 0u; + if round_bit && (sticky || (keep & 1u) != 0u) { + keep += 1u; + } + // keep is at most 2^24: exact in f32, and the scale is a power of two. + r = ldexp(f32(keep), i32(s + 1u)); + } + return select(r, -r, neg); +} diff --git a/src/wgpu/shaders/present.wgsl b/src/wgpu/shaders/present.wgsl new file mode 100644 index 000000000..5dbdcb4b5 --- /dev/null +++ b/src/wgpu/shaders/present.wgsl @@ -0,0 +1,37 @@ +// Presenting into a window: textured quads (the frame letterboxed, the +// debug overlay over it, the status bar under it). + +// Where a quad goes, in normalised device coordinates (x0, top, x1, +// bottom), and the texture coordinates it shows (u0, v0 at the top left, +// u1, v1 at the bottom right). +struct Quad { + rect: vec4, + uv: vec4, +} + +@group(0) @binding(0) var tex: texture_2d; +@group(0) @binding(1) var samp: sampler; +@group(0) @binding(2) var quad: Quad; + +struct VOut { + @builtin(position) pos: vec4, + @location(0) uv: vec2, +} + +@vertex +fn quad_vs(@builtin(vertex_index) i: u32) -> VOut { + var corner = array, 6>( + vec2(0.0, 0.0), vec2(1.0, 0.0), vec2(0.0, 1.0), + vec2(0.0, 1.0), vec2(1.0, 0.0), vec2(1.0, 1.0), + ); + let k = corner[i]; + var o: VOut; + o.pos = vec4(mix(quad.rect.x, quad.rect.z, k.x), mix(quad.rect.y, quad.rect.w, k.y), 0.0, 1.0); + o.uv = vec2(mix(quad.uv.x, quad.uv.z, k.x), mix(quad.uv.y, quad.uv.w, k.y)); + return o; +} + +@fragment +fn quad_fs(v: VOut) -> @location(0) vec4 { + return textureSample(tex, samp, v.uv); +} diff --git a/src/wgpu/shaders/raster.wgsl b/src/wgpu/shaders/raster.wgsl new file mode 100644 index 000000000..80472bbd6 --- /dev/null +++ b/src/wgpu/shaders/raster.wgsl @@ -0,0 +1,57 @@ +// The raster pass driver: one 64-lane workgroup per busy tile, one pixel +// a lane, dispatched indirectly from the busy list. The workgroup is the +// only writer of its tile for the batch: it walks the tile's mask in +// primitive order and calls the board's hooks, which store pixels +// themselves. On the way it clears the mask words and the busy flag, so +// the next batch starts clean without a buffer clear. +// +// The board provides: +// fn arch_tile_begin(tile: u32, px: i32, py: i32, lane: u32) +// fn arch_prim(p: u32, px: i32, py: i32, lane: u32) +// fn arch_tile_end(tile: u32, px: i32, py: i32, lane: u32) +// (px, py) is the lane's pixel in target coordinates. Hooks run in +// workgroup-uniform control flow (the tile and the mask words are read +// with workgroupUniformLoad), so they may use barriers and subgroup +// operations as long as they keep every lane in the same branches and +// gate only their stores on coverage. + +#include "common.wgsl" + +var wg_tile: u32; +var wg_bits: u32; + +const NO_TILE: u32 = 0xffffffffu; + +@compute @workgroup_size(64) +fn raster(@builtin(workgroup_id) wg: vec3, @builtin(local_invocation_index) lane: u32) { + if lane == 0u { + let idx = wg.x + wg.y * MAX_GROUPS; + wg_tile = select(NO_TILE, busy.tiles[idx], idx < atomicLoad(&busy.count)); + } + let tile = workgroupUniformLoad(&wg_tile); + if tile == NO_TILE { + return; + } + let tx = tile % batch.tile_cols; + let ty = tile / batch.tile_cols; + let px = batch.origin_x + i32(tx * TILE_W + lane % TILE_W); + let py = batch.origin_y + i32(ty * TILE_H + lane / TILE_W); + + arch_tile_begin(tile, px, py, lane); + let base = tile * MASK_WORDS; + for (var w = 0u; w < MASK_WORDS; w++) { + if lane == 0u { + wg_bits = atomicExchange(&masks[base + w], 0u); + } + var bits = workgroupUniformLoad(&wg_bits); + while bits != 0u { + let p = w * 32u + firstTrailingBit(bits); + bits &= bits - 1u; + arch_prim(p, px, py, lane); + } + } + arch_tile_end(tile, px, py, lane); + if lane == 0u { + atomicStore(&busy_flags[tile], 0u); + } +} diff --git a/src/wgpu/shaders/scanout.wgsl b/src/wgpu/shaders/scanout.wgsl new file mode 100644 index 000000000..8b8a01c30 --- /dev/null +++ b/src/wgpu/shaders/scanout.wgsl @@ -0,0 +1,23 @@ +// The scanout pass driver: one lane per screen pixel, the board's +// `scanout_pixel(x, y) -> u32` (0x00RRGGBB) into a frame of `stride` +// pixels a row. Group 0 is the frame; group 1 binding 0 is the board's +// display slot, declared by the board's fragment; groups 2 and up are the +// board's memory, bound as its raster kernels bind it. + +struct ScanoutHeader { + width: u32, + height: u32, + stride: u32, + pad: u32, +} + +@group(0) @binding(0) var scan: ScanoutHeader; +@group(0) @binding(1) var scan_out: array; + +@compute @workgroup_size(8, 8) +fn scanout(@builtin(global_invocation_id) g: vec3) { + if g.x >= scan.width || g.y >= scan.height { + return; + } + scan_out[g.y * scan.stride + g.x] = scanout_pixel(g.x, g.y); +} diff --git a/src/wgpu/shaders/unpack.wgsl b/src/wgpu/shaders/unpack.wgsl new file mode 100644 index 000000000..450cbde1a --- /dev/null +++ b/src/wgpu/shaders/unpack.wgsl @@ -0,0 +1,16 @@ +// A frame of 0x00RRGGBB words already on the device (a board's scan-out), +// rows of `size.x`, into the presenter's frame texture: no readback. + +@group(0) @binding(0) var frame_words: array; +@group(0) @binding(1) var frame_out: texture_storage_2d; +@group(0) @binding(2) var size: vec4; + +@compute @workgroup_size(8, 8) +fn unpack(@builtin(global_invocation_id) g: vec3) { + if g.x >= size.x || g.y >= size.y { + return; + } + let w = frame_words[g.y * size.x + g.x]; + let c = vec4(f32((w >> 16u) & 0xffu), f32((w >> 8u) & 0xffu), f32(w & 0xffu), 255.0) / 255.0; + textureStore(frame_out, vec2(g.xy), c); +} diff --git a/src/wgpu/tests/flat.wgsl b/src/wgpu/tests/flat.wgsl new file mode 100644 index 000000000..517af1deb --- /dev/null +++ b/src/wgpu/tests/flat.wgsl @@ -0,0 +1,65 @@ +// The test board: flat-coloured triangles with a write mask into a +// plane of u32 pixels, STRIDE words a row, WIDTH x HEIGHT visible. +// Coordinates are 1/16 pixel; the sample is the pixel centre; the fill +// rule is top-left. src/wgpu/tests/mod.rs has the same arithmetic in Rust. + +struct Rec { + x0: i32, y0: i32, + x1: i32, y1: i32, + x2: i32, y2: i32, + colour: u32, + slot: u32, +} + +struct Slot { + mask: u32, + pad0: u32, + pad1: u32, + pad2: u32, +} + +@group(1) @binding(0) var recs: array; +@group(1) @binding(1) var slots: array; +@group(2) @binding(0) var vram: array; + +var addr: u32; +var pixel: u32; +var in_target: bool; + +fn edge(ax: i32, ay: i32, bx: i32, by: i32, px: i32, py: i32) -> i32 { + return (bx - ax) * (py - ay) - (by - ay) * (px - ax); +} + +// Top-left rule for a positively oriented triangle: a top edge runs +// right with no rise, a left edge runs up. +fn inside(e: i32, dx: i32, dy: i32) -> bool { + return e > 0 || (e == 0 && ((dy == 0 && dx > 0) || dy < 0)); +} + +fn arch_tile_begin(tile: u32, px: i32, py: i32, lane: u32) { + in_target = px >= 0 && py >= 0 && u32(px) < WIDTH && u32(py) < HEIGHT; + addr = u32(py) * STRIDE + u32(px); + pixel = select(0u, vram[addr], in_target); +} + +fn arch_prim(p: u32, px: i32, py: i32, lane: u32) { + let r = recs[p]; + let sx = px * 16 + 8; + let sy = py * 16 + 8; + let e0 = edge(r.x1, r.y1, r.x2, r.y2, sx, sy); + let e1 = edge(r.x2, r.y2, r.x0, r.y0, sx, sy); + let e2 = edge(r.x0, r.y0, r.x1, r.y1, sx, sy); + let covered = inside(e0, r.x2 - r.x1, r.y2 - r.y1) + && inside(e1, r.x0 - r.x2, r.y0 - r.y2) + && inside(e2, r.x1 - r.x0, r.y1 - r.y0); + if covered { + let m = slots[r.slot].mask; + pixel = (pixel & ~m) | (r.colour & m); + } +} + +fn arch_tile_end(tile: u32, px: i32, py: i32, lane: u32) { + if in_target { + vram[addr] = pixel; + } +} diff --git a/src/wgpu/tests/mod.rs b/src/wgpu/tests/mod.rs new file mode 100644 index 000000000..db14d5d7b --- /dev/null +++ b/src/wgpu/tests/mod.rs @@ -0,0 +1,312 @@ +//! End-to-end tests of the common layer on a test board: flat triangles +//! into a u32 plane, checked word for word against a CPU rasteriser with +//! the same arithmetic. They need a GPU (or lavapipe) and skip without +//! one. + +use std::sync::Arc; + +use bytemuck::{Pod, Zeroable}; + +use super::batch::{ArchShaders, Batch, BatchPipelines, Full}; +use super::device::{Gpu, GpuOptions}; +use super::memory::GpuMemory; +use super::shader::Fragment; +use super::tiles::{TileGrid, MAX_PRIMS}; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +struct Rec { + x: [i32; 3], + y: [i32; 3], + colour: u32, + slot: u32, +} + +// WGSL lays the record out as x0,y0,x1,y1,x2,y2: keep the Rust side in +// the same order. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +struct RecWgsl { + v: [i32; 6], + colour: u32, + slot: u32, +} + +impl Rec { + fn wgsl(&self) -> RecWgsl { + RecWgsl { + v: [self.x[0], self.y[0], self.x[1], self.y[1], self.x[2], self.y[2]], + colour: self.colour, + slot: self.slot, + } + } +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +struct Slot { + mask: u32, + pad: [u32; 3], +} + +const FLAT: Fragment = Fragment::new("flat.wgsl", include_str!("flat.wgsl")); + +struct Board { + gpu: Arc, + pipelines: BatchPipelines, + batch: Batch, + vram: GpuMemory, + vram_group: wgpu::BindGroup, + width: u32, + height: u32, + stride: u32, +} + +fn gpu() -> Option> { + match Gpu::new(&GpuOptions::default()) { + Ok(g) => Some(g), + Err(e) => { + eprintln!("skipped: {e}"); + None + } + } +} + +impl Board { + fn new(gpu: Arc, width: u32, height: u32) -> Board { + let stride = (width + 15) & !15; + let layout = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("flat vram"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: false }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }], + }); + let pipelines = BatchPipelines::new( + &gpu, + &ArchShaders { + label: "flat", + raster: &[FLAT], + bin_accept: None, + library: &[], + defines: &[("WIDTH", width), ("HEIGHT", height), ("STRIDE", stride)], + layouts: &[&layout], + }, + ) + .unwrap_or_else(|e| panic!("{e}")); + let vram = GpuMemory::new(&gpu, "flat vram", stride as u64 * height as u64 * 4).unwrap(); + let vram_group = gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("flat vram"), + layout: &layout, + entries: &[wgpu::BindGroupEntry { binding: 0, resource: vram.buffer.as_entire_binding() }], + }); + let batch = Batch::new(&gpu, &pipelines, TileGrid::new(width, height), 64); + Board { gpu, pipelines, batch, vram, vram_group, width, height, stride } + } + + fn flush(&mut self) { + self.batch.flush(&self.pipelines, &[&self.vram_group]); + } + + fn plane(&mut self) -> Vec { + let bytes = self.vram.read_all().unwrap(); + bytemuck::cast_slice(&bytes).to_vec() + } +} + +/// The CPU side of flat.wgsl. +struct Reference { + plane: Vec, + width: u32, + height: u32, + stride: u32, +} + +fn edge(ax: i32, ay: i32, bx: i32, by: i32, px: i32, py: i32) -> i32 { + (bx - ax) * (py - ay) - (by - ay) * (px - ax) +} + +fn inside(e: i32, dx: i32, dy: i32) -> bool { + e > 0 || (e == 0 && ((dy == 0 && dx > 0) || dy < 0)) +} + +impl Reference { + fn draw(&mut self, r: &Rec, mask: u32) { + let (x, y) = (r.x, r.y); + let x0 = (x.iter().copied().min().unwrap() >> 4).max(0); + let x1 = ((x.iter().copied().max().unwrap() + 15) >> 4).min(self.width as i32 - 1); + let y0 = (y.iter().copied().min().unwrap() >> 4).max(0); + let y1 = ((y.iter().copied().max().unwrap() + 15) >> 4).min(self.height as i32 - 1); + for py in y0..=y1 { + for px in x0..=x1 { + let (sx, sy) = (px * 16 + 8, py * 16 + 8); + let e0 = edge(x[1], y[1], x[2], y[2], sx, sy); + let e1 = edge(x[2], y[2], x[0], y[0], sx, sy); + let e2 = edge(x[0], y[0], x[1], y[1], sx, sy); + if inside(e0, x[2] - x[1], y[2] - y[1]) + && inside(e1, x[0] - x[2], y[0] - y[2]) + && inside(e2, x[1] - x[0], y[1] - y[0]) + { + let i = (py as u32 * self.stride + px as u32) as usize; + self.plane[i] = (self.plane[i] & !mask) | (r.colour & mask); + } + } + } + } +} + +/// A deterministic generator. +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 33) as u32 + } + + fn range(&mut self, lo: i32, hi: i32) -> i32 { + lo + (self.next() % (hi - lo) as u32) as i32 + } +} + +/// A triangle with positive orientation, vertices in 1/16 pixel around +/// the target (some outside it), of a size drawn from `max`. +fn triangle(rng: &mut Lcg, width: u32, height: u32, max: i32) -> Rec { + let cx = rng.range(-32, width as i32 + 32) * 16; + let cy = rng.range(-32, height as i32 + 32) * 16; + let mut r = Rec::default(); + for i in 0..3 { + r.x[i] = cx + rng.range(-max, max); + r.y[i] = cy + rng.range(-max, max); + } + if edge(r.x[0], r.y[0], r.x[1], r.y[1], r.x[2], r.y[2]) < 0 { + r.x.swap(1, 2); + r.y.swap(1, 2); + } + r.colour = rng.next(); + r +} + +fn bbox(r: &Rec) -> (i32, i32, i32, i32) { + ( + r.x.iter().copied().min().unwrap() >> 4, + r.y.iter().copied().min().unwrap() >> 4, + (r.x.iter().copied().max().unwrap() + 15) >> 4, + (r.y.iter().copied().max().unwrap() + 15) >> 4, + ) +} + +fn compare(gpu_plane: &[u32], reference: &Reference, what: &str) { + let mut bad = 0; + for y in 0..reference.height { + for x in 0..reference.width { + let i = (y * reference.stride + x) as usize; + if gpu_plane[i] != reference.plane[i] { + if bad < 8 { + eprintln!("{what}: ({x},{y}) gpu {:08x} cpu {:08x}", gpu_plane[i], reference.plane[i]); + } + bad += 1; + } + } + } + assert_eq!(bad, 0, "{what}: {bad} pixels differ"); +} + +fn run(width: u32, height: u32, batches: usize, per_batch: usize, max: i32, seed: u64) { + let Some(gpu) = gpu() else { return }; + let mut board = Board::new(gpu, width, height); + let mut reference = Reference { plane: vec![0; (board.stride * height) as usize], width, height, stride: board.stride }; + let mut rng = Lcg(seed); + let masks = [0xffff_ffff, 0x00ff_ffff, 0xff00_ff00, 0x0000_00ff]; + for b in 0..batches { + let mut slots = Vec::new(); + for m in masks { + slots.push(board.batch.slot(Slot { mask: m, pad: [0; 3] }).unwrap()); + } + for _ in 0..per_batch { + let mut r = triangle(&mut rng, width, height, max); + let s = (rng.next() % 4) as usize; + r.slot = slots[s]; + let (x0, y0, x1, y1) = bbox(&r); + match board.batch.prim(r.wgsl(), x0, y0, x1, y1) { + Ok(_) => reference.draw(&r, masks[s]), + Err(Full) => panic!("batch full at {per_batch}"), + } + } + board.flush(); + if b % 4 == 3 { + let plane = board.plane(); + compare(&plane, &reference, &format!("after batch {b}")); + } + } + let plane = board.plane(); + compare(&plane, &reference, "final"); + board.gpu.check().unwrap(); +} + +#[test] +fn small_triangles_match_the_reference() { + run(320, 200, 8, 500, 16 * 12, 1); +} + +#[test] +fn large_overlapping_triangles_keep_order() { + run(640, 480, 6, 200, 16 * 400, 2); +} + +#[test] +fn odd_sized_target_and_offscreen_boxes() { + run(333, 211, 4, 300, 16 * 60, 3); +} + +#[test] +fn full_batches_flush_and_continue() { + let Some(gpu) = gpu() else { return }; + let (width, height) = (256u32, 256u32); + let mut board = Board::new(gpu, width, height); + let mut reference = Reference { plane: vec![0; (board.stride * height) as usize], width, height, stride: board.stride }; + let mut rng = Lcg(7); + let mut pushed = 0u32; + let mut flushes = 0; + while pushed < MAX_PRIMS * 2 + 17 { + let slot = match board.batch.slot(Slot { mask: !0, pad: [0; 3] }) { + Ok(s) => s, + Err(Full) => { + board.flush(); + flushes += 1; + continue; + } + }; + let mut r = triangle(&mut rng, width, height, 16 * 20); + r.slot = slot; + let (x0, y0, x1, y1) = bbox(&r); + match board.batch.prim(r.wgsl(), x0, y0, x1, y1) { + Ok(_) => { + reference.draw(&r, !0); + pushed += 1; + } + Err(Full) => { + board.flush(); + flushes += 1; + } + } + } + board.flush(); + assert!(flushes >= 2, "expected the recorder to fill up, flushed {flushes} times"); + let plane = board.plane(); + compare(&plane, &reference, "after refills"); +} + +#[test] +fn empty_flush_is_a_no_op() { + let Some(gpu) = gpu() else { return }; + let mut board = Board::new(gpu, 64, 64); + assert!(board.batch.flush(&board.pipelines, &[&board.vram_group]).is_none()); + assert!(board.plane().iter().all(|&w| w == 0)); +} diff --git a/src/wgpu/tiles.rs b/src/wgpu/tiles.rs new file mode 100644 index 000000000..d6f4fedbd --- /dev/null +++ b/src/wgpu/tiles.rs @@ -0,0 +1,139 @@ +//! Tile geometry and the batch buffer layouts the Rust side and the WGSL +//! side agree on. `shaders/common.wgsl` declares the same structs; the +//! constants reach it through `Source::define`. + +use bytemuck::{Pod, Zeroable}; + +/// A tile is 8x8 pixels: one 64-lane workgroup, one pixel a lane. +pub const TILE_W: u32 = 8; +pub const TILE_H: u32 = 8; +pub const LANES: u32 = TILE_W * TILE_H; + +/// Primitives a batch can hold, and the mask words per tile that implies. +pub const MAX_PRIMS: u32 = 4096; +pub const MASK_WORDS: u32 = MAX_PRIMS / 32; + +/// wgpu's cap on workgroups along one dispatch axis. The raster dispatch +/// folds the busy count over x and y when it is larger. +pub const MAX_GROUPS: u32 = 65535; + +/// The tiles over a target of `width` x `height` pixels at `origin`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TileGrid { + pub origin_x: i32, + pub origin_y: i32, + pub width: u32, + pub height: u32, +} + +impl TileGrid { + pub fn new(width: u32, height: u32) -> TileGrid { + TileGrid { origin_x: 0, origin_y: 0, width, height } + } + + pub fn cols(&self) -> u32 { + self.width.div_ceil(TILE_W) + } + + pub fn rows(&self) -> u32 { + self.height.div_ceil(TILE_H) + } + + pub fn tiles(&self) -> u32 { + self.cols() * self.rows() + } + + /// The tiles a pixel box `x0..=x1`, `y0..=y1` (inclusive, in target + /// pixels) touches, or None when it misses the grid. + pub fn tile_box(&self, x0: i32, y0: i32, x1: i32, y1: i32) -> Option { + let (w, h) = (self.width as i64, self.height as i64); + let x0 = (x0 as i64 - self.origin_x as i64).max(0); + let y0 = (y0 as i64 - self.origin_y as i64).max(0); + let x1 = (x1 as i64 - self.origin_x as i64).min(w - 1); + let y1 = (y1 as i64 - self.origin_y as i64).min(h - 1); + if x0 > x1 || y0 > y1 { + return None; + } + Some(TileBox { + x0: (x0 / TILE_W as i64) as u16, + y0: (y0 / TILE_H as i64) as u16, + x1: (x1 / TILE_W as i64) as u16, + y1: (y1 / TILE_H as i64) as u16, + }) + } +} + +/// An inclusive box of tiles. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TileBox { + pub x0: u16, + pub y0: u16, + pub x1: u16, + pub y1: u16, +} + +impl TileBox { + pub fn count(&self) -> u32 { + (self.x1 - self.x0 + 1) as u32 * (self.y1 - self.y0 + 1) as u32 + } + + /// As the bin kernel reads it: x0, y0, x1, y1. + pub fn packed(&self) -> [u32; 4] { + [self.x0 as u32, self.y0 as u32, self.x1 as u32, self.y1 as u32] + } +} + +/// `BatchHeader` in common.wgsl: the batch's uniform. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +pub struct BatchHeader { + pub prim_count: u32, + pub tile_cols: u32, + pub tile_rows: u32, + pub mask_words: u32, + pub origin_x: i32, + pub origin_y: i32, + pub _pad: [u32; 2], +} + +/// The head of `BusyList` in common.wgsl: the busy count, then the tile +/// indices. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +pub struct BusyHead { + pub count: u32, + pub _pad: [u32; 3], +} + +/// `DispatchArgs` in bin.wgsl: the raster pass's indirect arguments, in +/// a buffer of their own. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +pub struct DispatchArgs { + pub x: u32, + pub y: u32, + pub z: u32, + pub _pad: u32, +} + +impl DispatchArgs { + /// A dispatch of no groups, for the bin pass to raise. + pub const EMPTY: DispatchArgs = DispatchArgs { x: 0, y: 1, z: 1, _pad: 0 }; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tile_box_clips_to_the_grid() { + let g = TileGrid::new(1280, 1024); + assert_eq!(g.cols(), 160); + assert_eq!(g.rows(), 128); + assert_eq!(g.tile_box(-5, -5, 7, 7), Some(TileBox { x0: 0, y0: 0, x1: 0, y1: 0 })); + assert_eq!(g.tile_box(8, 8, 15, 15), Some(TileBox { x0: 1, y0: 1, x1: 1, y1: 1 })); + assert_eq!(g.tile_box(1279, 1023, 5000, 5000), Some(TileBox { x0: 159, y0: 127, x1: 159, y1: 127 })); + assert_eq!(g.tile_box(1280, 0, 1300, 10), None); + assert_eq!(g.tile_box(0, 0, -1, 5), None); + } +} diff --git a/test/gltest/glsuite-compare.sh b/test/gltest/glsuite-compare.sh new file mode 100755 index 000000000..a41f82ebb --- /dev/null +++ b/test/gltest/glsuite-compare.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Compare two result sets from glsuite.sh / xsweep.sh (host side): +# test/gltest/glsuite-compare.sh shared/results/cpu shared/results/gpu +A=${1:?reference}; B=${2:?candidate} +sum() { tail -1 "$1" 2>/dev/null; } +echo "== state and pixel suites (summary lines; failing cases only in one side listed)" +for f in glstate blendtest seltest cicopy pixtest pixtest-rgba8888 pixtest-ci8 pixtest-ci12; do + a=$(sum $A/$f.txt); b=$(sum $B/$f.txt) + [ "$a" = "$b" ] && echo " same $f: $a" || echo " DIFFER $f: [$a] vs [$b]" + diff <(grep -E "^FAIL" $A/$f.txt | cut -d: -f1) <(grep -E "^FAIL" $B/$f.txt | cut -d: -f1) | grep "^[<>]" | sed 's/^/ only in candidate:/' +done +echo "== visual sweep" +a=$(sum $A/vissweep.txt); b=$(sum $B/vissweep.txt) +[ "$a" = "$b" ] && echo " same: $a" || echo " DIFFER: [$a] vs [$b]" +diff <(grep -E "^[0-9,]+ +[sd]b" $A/vissweep.txt) <(grep -E "^[0-9,]+ +[sd]b" $B/vissweep.txt) | grep "^[<>]" | sed 's/^/ /' +echo "== texture sweep" +diff $A/texsweep.txt $B/texsweep.txt > /dev/null && echo " same" || diff $A/texsweep.txt $B/texsweep.txt | grep "^[<>]" | sed 's/^/ /' +echo "== lines and points (GL readbacks)" +n=0; d=0; for f in $A/lines/*.ppm; do n=$((n+1)); cmp -s $f $B/lines/$(basename $f) || { d=$((d+1)); echo " differs: $(basename $f)"; }; done; echo " $d of $n differ" +echo "== scenes (GL readbacks, and the screen through scrsave cropped to the window)" +for f in $A/scenes/*.ppm; do n=$(basename $f); cmp -s $f $B/scenes/$n || echo " readback differs: $n $(compare -metric AE $f $B/scenes/$n null: 2>&1)"; done +for f in $A/scenes/*.rgb; do + n=$(basename $f .rgb); o=$(grep -o "origin [0-9]*,[0-9]*" $A/scenes/$n.log | head -1 | tr -dc '0-9,') + [ -n "$o" ] || continue; x=${o%,*}; y=${o#*,} + c=$(compare -metric AE <(magick $f -crop 256x256+$x+$y +repage ppm:-) <(magick $B/scenes/$n.rgb -crop 256x256+$x+$y +repage ppm:-) null: 2>&1) + [ "${c%% *}" = "0" ] || echo " screen differs: $n $c" +done +echo "== X server modes" +for d in $A/x-*; do m=$(basename $d) + q="$(grep -c FAIL $d/quadrants.txt 2>/dev/null)/$(grep -c FAIL $B/$m/quadrants.txt 2>/dev/null) quadrant failures" + t=$(cmp -s $d/tri_front.ppm $B/$m/tri_front.ppm && echo "tri readback same" || echo "TRI READBACK DIFFERS") + s=$(compare -metric AE $d/desktop.rgb $B/$m/desktop.rgb null: 2>&1) + echo " $m: $q, $t, desktop pixels differing $s" +done +echo "== piglit" +if [ -f $A/piglit/results ] && [ -f $B/piglit/results ]; then + echo " reference: $(cut -f1 $A/piglit/results | sort | uniq -c | tr -s ' ' | tr '\n' ' ')" + echo " candidate: $(cut -f1 $B/piglit/results | sort | uniq -c | tr -s ' ' | tr '\n' ' ')" + join -t $'\t' -1 2 -2 2 <(sort -t $'\t' -k2 $A/piglit/results) <(sort -t $'\t' -k2 $B/piglit/results) | awk -F'\t' '$2 != $3 {print " " $1 ": " $2 " -> " $3}' +fi diff --git a/test/gltest/glsuite.sh b/test/gltest/glsuite.sh new file mode 100644 index 000000000..a6dcc8493 --- /dev/null +++ b/test/gltest/glsuite.sh @@ -0,0 +1,68 @@ +#!/bin/sh +# Every GL check suite in the guest, for comparing two runs (two raster +# units, two builds). Expects test/gltest copied to /shared/gltest and the +# piglit irix tree to /shared/piglit (TESTING.md). +# sh glsuite.sh TAG [suites...] suites: state pix vis tex lines scenes piglit +# Results in /usr/tmp/res-TAG, copied to /shared/results/TAG as each +# suite finishes; progress in its "progress" file. +TAG=${1:?tag}; shift +SUITES=${*:-"state pix vis tex lines scenes piglit"} +R=/usr/tmp/res-$TAG +S=/shared/results/$TAG +DISPLAY=:0; export DISPLAY +mkdir -p $R $S +rm -rf /usr/tmp/gltest; cp -r /shared/gltest /usr/tmp/gltest +cd /usr/tmp/gltest +xset s off 2>/dev/null; xset s noblank 2>/dev/null +out() { cp -r $R/* $S/ 2>/dev/null; } +stamp() { echo "`date '+%H:%M:%S'` $*" >> $R/progress; out; } +has() { case " $SUITES " in *" $1 "*) return 0 ;; esac; return 1; } + +stamp start $SUITES +if has state; then + ./glstate > $R/glstate.txt 2>&1 + ./blendtest > $R/blendtest.txt 2>&1 + ./seltest > $R/seltest.txt 2>&1 + ./cicopy > $R/cicopy.txt 2>&1 + stamp state +fi +if has pix; then + ./pixtest > $R/pixtest.txt 2>&1 + ./pixtest --rgba8888 > $R/pixtest-rgba8888.txt 2>&1 + ./pixtest --ci 8 > $R/pixtest-ci8.txt 2>&1 + ./pixtest --ci 12 > $R/pixtest-ci12.txt 2>&1 + stamp pix +fi +if has vis; then + sh vissweep.sh > $R/vissweep.txt 2>&1 + stamp vis +fi +if has tex; then + HOLD=300 sh texsweep.sh > $R/texsweep.txt 2>&1 + stamp tex +fi +if has lines; then + sh linesweep.sh $R/lines > $R/linesweep.txt 2>&1 + stamp lines +fi +if has scenes; then + # Each scene read back through GL (--readout) and, while held, the + # whole screen through X (scrsave): two readback paths of one image. + mkdir -p $R/scenes + for p in points lines lloop lstrip triangles tstrip tfan quads qstrip polygon; do + ./glprim -p $p --smooth --size 128x128 --read front --readout $R/scenes/prim_$p --hold 0 > $R/scenes/prim_$p.log 2>&1 + done + for sc in depth stencil alphatest blend lit litlocal litrgb twoside fog litcmat litnorm dlist clipplane fogexp fogexp2 blendsmooth quadrants copycolor copydepth tex texmod; do + ./glprim -p none --scene $sc --size 256x256 --read front --readout $R/scenes/$sc --hold 4000 > $R/scenes/$sc.log 2>&1 & + sleep 3 + /usr/sbin/scrsave $R/scenes/$sc.rgb + wait + done + stamp scenes +fi +if has piglit; then + rm -rf /usr/tmp/piglit; cp -r /shared/piglit /usr/tmp/piglit + cd /usr/tmp/piglit/irix && ./piglit-run.sh -t 300 -o $R/piglit lists/vpro.list > $R/piglit.log 2>&1 + stamp piglit +fi +stamp done diff --git a/test/gltest/xsweep.sh b/test/gltest/xsweep.sh new file mode 100644 index 000000000..94a454165 --- /dev/null +++ b/test/gltest/xsweep.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Restart the X server in each root visual class and depth, log in again +# (autologin), and capture: the desktop through scrsave, xdpyinfo, and a GL +# quadrants check and a smooth triangle read back through GL and scrsave. +# sh xsweep.sh TAG [modes...] modes: tc24 pc8 pc12 tc12 tc15 tc30 dc24 +# Results in /usr/tmp/res-TAG/x-MODE, copied to /shared/results/TAG. +TAG=${1:?tag}; shift +MODES=${*:-"pc8 pc12 tc12 tc15 tc30 dc24 tc24"} +R=/usr/tmp/res-$TAG; S=/shared/results/$TAG +X=/var/X11/xdm/Xservers +DISPLAY=:0; export DISPLAY +mkdir -p $R $S +[ -f $X.orig ] || cp $X $X.orig +G=/usr/tmp/gltest/glprim +[ -x $G ] || { rm -rf /usr/tmp/gltest; cp -r /shared/gltest /usr/tmp/gltest; } +stamp() { echo "`date '+%H:%M:%S'` $*" >> $R/progress; cp -r $R/* $S/ 2>/dev/null; } +base=":0 secure /usr/bin/X11/X -bs -nobitscale -c -solidroot sgilightblue -cursorFG red -cursorBG white" +for m in $MODES; do + case $m in + tc24) o="-depth 24 -class TrueColor" ;; + pc8) o="-depth 8 -class PseudoColor" ;; + pc12) o="-depth 12 -class PseudoColor" ;; + tc12) o="-depth 12 -class TrueColor" ;; + tc15) o="-depth 15 -class TrueColor" ;; + tc30) o="-depth 30 -class TrueColor" ;; + dc24) o="-depth 24 -class DirectColor" ;; + *) echo "unknown mode $m"; continue ;; + esac + D=$R/x-$m; mkdir -p $D + grep -v "^:0" $X.orig > $X; echo "$base $o" >> $X + touch /etc/autologin.on + killall Xsgi; sleep 10 + n=0 + while [ $n -lt 60 ] && ! ps -ef | grep -v grep | grep -q toolchest; do sleep 5; n=`expr $n + 1`; done + if [ $n -ge 60 ]; then stamp "$m: no desktop"; continue; fi + sleep 25 + xset s off; xset s noblank + /usr/bin/X11/xdpyinfo > $D/xdpyinfo.txt 2>&1 + /usr/sbin/scrsave $D/desktop.rgb + $G -p none --scene quadrants --check --read front --readout $D/quadrants --hold 0 > $D/quadrants.txt 2>&1 + $G -p triangles --smooth --size 256x256 --read front --readout $D/tri --hold 4000 > $D/tri.txt 2>&1 & + sleep 3; /usr/sbin/scrsave $D/tri.rgb; wait + stamp "$m: `grep -c FAIL $D/quadrants.txt` quadrant failures, root `grep 'depth of root' $D/xdpyinfo.txt`" +done +# Back to the server as it was. +cp $X.orig $X; touch /etc/autologin.on; killall Xsgi +stamp done