Persist bounded realtime invalidation replay across restarts and servers

.github/workflows/ci.yml

@@ -302,14 +302,18 @@ jobs:
           key: zig-global-${{ runner.os }}-zig0.16.0-${{ hashFiles('vendor/sqlite/sqlite3.c', 'build.zig', 'build.zig.zon') }}
           restore-keys: zig-global-${{ runner.os }}-zig0.16.0-
       - name: Unit + PostgreSQL driver tests (-Dpostgres=true -Dvector=true)
-        run: mise exec zig@0.16.0 -- zig build test --summary all -Dpostgres=true -Dvector=true -Dquery-workbench=true
+        run: mise exec zig@0.16.0 -- zig build test --summary all -Dpostgres=true -Dvector=true -Dquery-workbench=true -Ddurable-realtime=true
       - name: PostgreSQL workbench paired gating and live HTTP
         run: |
           mise exec zig@0.16.0 -- zig build query-workbench-fixture -Dpostgres=true --prefix zig-out/workbench-off
           mise exec zig@0.16.0 -- zig build query-workbench-fixture -Dpostgres=true -Dquery-workbench=true --prefix zig-out/workbench-on
           bash scripts/check-query-workbench-gating.sh zig-out/workbench-off/bin/query-workbench-fixture zig-out/workbench-on/bin/query-workbench-fixture
           mise exec python@3.13 -- python -m pip install --quiet pytest
           ZIGBASE_TEST_QUERY_WORKBENCH_OFF_BINARY="$GITHUB_WORKSPACE/zig-out/workbench-off/bin/query-workbench-fixture" ZIGBASE_TEST_QUERY_WORKBENCH_BINARY="$GITHUB_WORKSPACE/zig-out/workbench-on/bin/query-workbench-fixture" mise exec python@3.13 -- python -m pytest tests/admin/test_query_workbench_postgres.py -q
+      - name: Durable realtime PostgreSQL cross-instance recovery
+        run: |
+          mise exec zig@0.16.0 -- zig build -Dpostgres=true -Ddurable-realtime=true --prefix zig-out/replay-pg
+          ZIGBASE_TEST_DURABLE_REALTIME_BINARY="$GITHUB_WORKSPACE/zig-out/replay-pg/bin/zigbase" mise exec python@3.13 -- python -m pytest tests/admin/test_durable_realtime.py::test_postgres_cross_instance_replay -q
       - name: Rails PostgreSQL row-source integration
         run: |
           mise exec python@3.13 -- python -m pip install --quiet pytest -r tools/rails/requirements-postgres.txt
@@ -935,6 +939,11 @@ jobs:
           mise exec zig@0.16.0 -- zig build test durable-uploads-fixture check-resumable-contracts -Dresumable-uploads=true -Ddurable-resumable-uploads=true --prefix zig-out/durable-uploads --summary all
           bash scripts/check-gating.sh --durable-uploads zig-out/resumable/bin/resumable-uploads-fixture zig-out/durable-uploads/bin/durable-uploads-fixture
           ZIGBASE_TEST_DURABLE_UPLOADS_BINARY="$GITHUB_WORKSPACE/zig-out/durable-uploads/bin/durable-uploads-fixture" mise exec python@3.13 -- python -m pytest tests/admin/test_durable_uploads.py -q -n 2 --dist load --durations=20
+      - name: Durable realtime replay regressions
+        if: matrix.lane == 'coordination'
+        run: |
+          mise exec zig@0.16.0 -- zig build -Ddurable-realtime=true -Drealtime-backfill=true --prefix zig-out/durable-realtime
+          ZIGBASE_TEST_DURABLE_REALTIME_BINARY="$GITHUB_WORKSPACE/zig-out/durable-realtime/bin/zigbase" mise exec python@3.13 -- python -m pytest tests/admin/test_durable_realtime.py -q
       - name: Opt-in realtime backfill regressions
         if: matrix.lane == 'coordination'
         run: |

BACKLOG.md

@@ -185,6 +185,11 @@ product priority and acceptance evidence; it does not create duplicate implement
 - [x] Exercise benchmark correctness under ReleaseSafe in CI.
 - [x] Bounded single-process SQLite backfill with current authorization and explicit gap semantics (#412).
 - [ ] Durable cross-instance replay with explicit resource and retention budgets.
+  Implemented: opt-in transactional SQLite/PostgreSQL built-in REST invalidations,
+  shared 4096-entry / 4 MiB / 24-hour retention, commit-ordered checkpoints,
+  restart/cross-instance recovery, and current per-item authorization. Data/raw-SQL,
+  hook side-writes and custom-channel capture remain outside this REST scope;
+  end-to-end replay coverage and configurable per-workload retention remain open.
 
 ## Analytics batching (#401)
 

build.zig

@@ -12,6 +12,11 @@ const BuildOptionValues = struct {
     s3: bool,
     file_inventory: bool,
     realtime_backfill: bool,
+    durable_realtime: bool,
+    replay_max_entries: u32,
+    replay_max_bytes: u32,
+    replay_max_frame_bytes: u32,
+    replay_retention_seconds: u32,
     resumable_uploads: bool,
     image_thumbnails: bool,
     durable_resumable_uploads: bool,
@@ -116,7 +121,15 @@ pub fn build(b: *std.Build) void {
     // a bare UnknownCommand.
     const dev_tools = b.option(bool, "dev-tools", "Compile development CLI verbs: init, agents-md, typegen, capabilities, routes, migrate preview, tune, diagnostics (default: on)") orelse true;
     const file_inventory = b.option(bool, "file-inventory", "Compile read-only local/S3 file inventory reporting (default: off)") orelse false;
-    const realtime_backfill = b.option(bool, "realtime-backfill", "Compile bounded process-local record invalidation backfill (default: off)") orelse false;
+    const replay_max_entries = b.option(u32, "replay-max-entries", "Durable journal entry budget (1..65536)") orelse 4096;
+    const replay_max_bytes = b.option(u32, "replay-max-bytes", "Durable journal frame byte budget (1..1073741824)") orelse 4194304;
+    const replay_max_frame_bytes = b.option(u32, "replay-max-frame-bytes", "Durable journal per-frame byte budget (1..1048576)") orelse 65536;
+    const replay_retention_seconds = b.option(u32, "replay-retention-seconds", "Durable journal retention (1..31536000 seconds)") orelse 86400;
+    if (replay_max_entries == 0 or replay_max_entries > 65536 or replay_max_bytes == 0 or replay_max_bytes > 1073741824 or replay_max_frame_bytes == 0 or replay_max_frame_bytes > 1048576 or replay_max_frame_bytes > replay_max_bytes or replay_retention_seconds == 0 or replay_retention_seconds > 31536000)
+        @panic("invalid durable replay budgets; frame bytes must not exceed total bytes");
+    const durable_realtime = b.option(bool, "durable-realtime", "Compile transactional durable REST record replay (default: off)") orelse false;
+    const local_backfill = b.option(bool, "realtime-backfill", "Compile bounded process-local record invalidation backfill (default: off)") orelse false;
+    const realtime_backfill = durable_realtime or local_backfill;
     const resumable_uploads = b.option(bool, "resumable-uploads", "Compile bounded process-local resumable file uploads (default: off)") orelse false;
     const image_thumbnails = b.option(bool, "image-thumbnails", "Compile ImageMagick thumbnail integration (default: off)") orelse false;
     const durable_resumable_uploads = b.option(bool, "durable-resumable-uploads", "Compile opt-in SQLite upload persistence (requires resumable-uploads)") orelse false;
@@ -172,6 +185,11 @@ pub fn build(b: *std.Build) void {
         .dev_tools = dev_tools,
         .file_inventory = file_inventory,
         .realtime_backfill = realtime_backfill,
+        .durable_realtime = durable_realtime,
+        .replay_max_entries = replay_max_entries,
+        .replay_max_bytes = replay_max_bytes,
+        .replay_max_frame_bytes = replay_max_frame_bytes,
+        .replay_retention_seconds = replay_retention_seconds,
         .resumable_uploads = resumable_uploads,
         .image_thumbnails = image_thumbnails,
         .durable_resumable_uploads = durable_resumable_uploads,

changelog.d/durable-realtime.md

@@ -0,0 +1,3 @@
+### Features
+
+- Add opt-in durable REST record invalidation replay on SQLite and PostgreSQL, with transactionally ordered checkpoints surviving restarts and cross-instance reads, bounded shared retention, and current authorization. Journal failures roll back record writes. Enable with `-Ddurable-realtime=true`; raw SQL, Data facade writes and hook side-writes remain outside capture coverage.

docs/api.md

@@ -1449,6 +1449,79 @@ rewrites, consulted before the marker. See
 
 ## Realtime (WebSocket + SSE)
 
+### Durable record invalidation replay (SQLite and PostgreSQL)
+
+Build with `-Ddurable-realtime=true` (and `-Dpostgres=true` for PostgreSQL).
+This also enables `GET /api/realtime/backfill` with the checkpoint protocol and
+current authorization described below, replacing the process-local store with a
+transactional database journal. Checkpoints survive application restarts and work
+on other instances connected to the same database and PostgreSQL schema.
+Cursor positions are encrypted and authenticated with a persistent journal secret
+and fresh random nonces; they bind collection identity and rename generation
+without exposing other collections' aggregate write positions.
+
+Built-in REST record create, update and delete operations append invalidations
+**inside the record transaction**, including resumable record updates. A journal
+failure rolls back the mutation; an oversized delete snapshot also rejects the
+mutation. Response serialization, after-hooks and live transport run after commit,
+so failure there does not remove the committed invalidation. Create/update entries
+retain only record IDs. Deletes retain private at-rest authorization snapshots;
+replay never returns these snapshots or historical field values.
+
+A single metadata row serializes journal positions through commit on both
+backends. PostgreSQL sequence allocation cannot run ahead of an earlier uncommitted
+writer. This introduces a shared lock across journaled mutations: capacity should
+be measured for the application, especially transactions that also perform slow
+work. Replay reads a consistent database snapshot, releases that reader, then
+rechecks current authorization independently for each item.
+
+By default the journal has a global ceiling of **4096 entries and 4 MiB of encoded frames**,
+with **64 KiB per frame** and **24-hour retention**. Build-time controls are
+`-Dreplay-max-entries` (1..65536), `-Dreplay-max-bytes` (1..1073741824),
+`-Dreplay-max-frame-bytes` (1..1048576, no greater than total bytes), and
+`-Dreplay-retention-seconds` (1..31536000). Successful replay responses report
+these effective budgets as `retention: {maxEntries,maxBytes,maxFrameBytes,seconds}`.
+All writer replicas must use the same budgets. Expiry is stamped when captured;
+changing retention does not recompute existing event expiry. Lowering size budgets
+prunes on the next write; lowering frame budgets can reject older larger frames,
+requiring a new checkpoint and snapshot. `/api/meta` exposes `realtimeBackfill`
+and `durableRealtime` capability booleans, distinguishing the two modes.
+These budgets bound
+retained logical data, not total database/WAL size or process RSS. Indexes, database
+pages and transient serialization/authorization memory add overhead. Each write
+prunes expired entries and evicts the oldest prefix until both size budgets fit;
+idle expired rows are reclaimed on the next write. Cursors represent the last consumed
+sequence: a cursor strictly below the discarded/expired prefix requires reset, while
+a cursor equal to its last sequence is still caught up. This also keeps fresh head
+checkpoints usable on an idle journal after every retained entry expires. Reads reject stale checkpoints
+even before physical cleanup. Clock differences can shorten useful retention;
+keep replica clocks synchronized. A backwards clock never revives a pruned cursor.
+Pages copy at most 128 frames (8 MiB encoded with default frame budget), plus temporary database and JSON
+storage. Admission/concurrency budgets remain application responsibilities.
+
+Unlike the local store, retention is shared across collections: a busy collection
+can cause another collection's checkpoint to require reset. Invalid, expired,
+evicted, collection-renamed or wrong-collection checkpoints return the same `409 resetRequired`
+envelope below. Obtain a new checkpoint before reloading a snapshot. A cursorless
+request does not reserve retention. Snapshot reload and idempotent application of
+invalidations are still required; WS/SSE messages do not carry journal cursors.
+
+Enable this option on **every writer** in the deployment. Writes from older or
+non-enabled binaries, raw SQL, migrations, the `Data` facade, hook side-writes,
+authentication helpers and custom channels are outside this journal's REST scope.
+Changing capture coverage requires clients to discard checkpoints and reload;
+the journal cannot detect writes performed by code that bypasses it. This is durable
+REST invalidation recovery, not exactly-once delivery or an audit/event-sourcing log.
+Database backups must include the journal tables. Restoring or rewinding a database
+requires clients to discard checkpoints and reload snapshots; restored history can
+reuse earlier journal positions. Before serving a restored database, with all
+instances stopped, replace `_replay_state.secret` with a newly generated
+cryptographically random 32-byte secret encoded as 64 hexadecimal characters.
+This forces old cursors to return `409`, including clients unaware of the restore. Normal database durability settings
+apply. Startup creates `_replay_state` and `_replay_events` in SQLite `main` or the
+PostgreSQL current schema; internal SQL qualifies that schema so temporary tables
+and search-path fallbacks cannot redirect journal operations.
+
 ### Optional record invalidation backfill (SQLite)
 
 Build with `-Drealtime-backfill=true` to register
@@ -1791,6 +1864,8 @@ full, only the work-count rejection is counted because it is checked first.
     "oauth2": true,
     "postgres": false,
     "queryWorkbench": false,
+    "realtimeBackfill": false,
+    "durableRealtime": false,
     "s3": false,
     "senders": true,
     "tenancy": true,
@@ -1801,6 +1876,7 @@ full, only the work-count rejection is counted because it is checked first.
     "health": "/api/health",
     "state": "/api/state",
     "realtimeSse": "/api/realtime/sse",
+    "realtimeBackfill": null,
     "queryWorkbench": null
   },
   "limits": {
@@ -1837,6 +1913,8 @@ binary carries:
 | `mailWebhook` | The mail bounce/complaint webhook route is mounted. |
 | `oauth2` | The OAuth2+PKCE auth method route group is mounted. |
 | `postgres` | The binary was compiled with `-Dpostgres` (the pure-Zig PostgreSQL backend is linked in). |
+| `realtimeBackfill` | The backfill route is compiled in (process-local SQLite unless durableRealtime is true). |
+| `durableRealtime` | Transactional REST replay is enabled, with database-backed cross-instance checkpoints. |
 | `s3` | The binary was compiled with `-Ds3` (the S3-compatible storage backend is linked in). |
 | `senders` | The verified-senders email route group is mounted. |
 | `tenancy` | Multi-tenancy is configured (`App(.{ .tenancy = ... })`). |

docs/framework.md

@@ -6153,6 +6153,11 @@ code to comptime-dead when off, so a build that doesn't need a feature doesn't p
 | `-Dvector` | off | Opt-in nearest-neighbor `?vector=` KNN search — sqlite-vec on SQLite, pgvector on Postgres. → [docs/search.md](./search.md#vector-search-opt-in) |
 | `-Dpostgres` | off | Opt-in pure-Zig PostgreSQL wire-protocol backend, alongside the default SQLite one. → [docs/postgres.md](./postgres.md) |
 | `-Dfile-inventory` | off | Read-only `files inventory` plus offline local/SQLite `files reconcile` (dry-run unless `--apply`). Bounded pages; no HTTP surface. Inventory supports S3 with `-Ds3=true`; reconciliation refuses S3/PostgreSQL/custom storage. The boot-lifetime local-storage maintenance lease remains in default builds. |
+| `-Ddurable-realtime` | off | Transactional built-in REST record replay on SQLite/PostgreSQL; implies realtime-backfill. Shared 4096-entry / 4 MiB journal, 64 KiB frame ceiling, 24-hour retention; restart/cross-instance checkpoints and current authorization. Raw SQL/Data/hook side-writes are outside capture scope. See API realtime section. |
+| `-Dreplay-max-entries` | 4096 | Durable journal global entry budget, 1..65536. All replicas must agree. |
+| `-Dreplay-max-bytes` | 4194304 | Durable journal encoded-frame byte budget, 1..1073741824. |
+| `-Dreplay-max-frame-bytes` | 65536 | Maximum individual durable frame bytes, 1..1048576, at most total byte budget. |
+| `-Dreplay-retention-seconds` | 86400 | Durable event lifetime stamped on capture, 1..31536000 seconds; physical expiry cleanup occurs on writes. |
 | `-Drealtime-backfill` | off | Single-process SQLite record invalidation backfill: 16 lazy collection slots, each 256 entries / 64 KiB (1 MiB encoded total), current authorization, explicit reset on gaps or slot replacement. No historical payloads or durable/cross-instance guarantee. See the API realtime section. |
 | `-Dimage-thumbnails` | off | Named PNG/JPEG/WebP derivatives on built-in local storage via a trusted external ImageMagick executable; configure `.files.thumbnails`. Subprocess support, routes and admission state are excluded when off; no image codec is linked. See [thumbnails](thumbnails.md). |
 | `-Dresumable-uploads` | off | Principal-bound network-resume for file fields on existing records. Process-local by default, fully buffered, configurable session/byte/chunk/expiry budgets. SQLite restart persistence requires the additional `-Ddurable-resumable-uploads` flag and `.files.resumable.durable = true`; neither mode provides cross-instance durability. See [resumable uploads](resumable-uploads.md). |

src/api/meta.zig

@@ -47,6 +47,8 @@ pub fn handle(ctx: *http.RequestCtx) !http.Response {
             .oauth2 = g.oauth2,
             .postgres = build_options.postgres,
             .queryWorkbench = build_options.query_workbench,
+            .realtimeBackfill = build_options.realtime_backfill,
+            .durableRealtime = build_options.durable_realtime,
             .s3 = build_options.s3,
             .senders = g.senders,
             .tenancy = g.tenancy,
@@ -57,6 +59,7 @@ pub fn handle(ctx: *http.RequestCtx) !http.Response {
             .health = "/api/health",
             .state = app.features_public_route,
             .realtimeSse = "/api/realtime/sse",
+            .realtimeBackfill = if (build_options.realtime_backfill) "/api/realtime/backfill" else @as(?[]const u8, null),
             .queryWorkbench = if (build_options.query_workbench) "/api/query-workbench/stats" else @as(?[]const u8, null),
         },
         .limits = .{ .maxUploadSize = app.max_upload_size },