diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d72fd520 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.pdf binary diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ecb2281e..60a66f05 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -13,10 +13,25 @@ permissions: contents: read jobs: + # `scripts/ci-local.sh` runs `cargo fmt --all --check` as its first step. CI + # did not, which made the script stricter than CI instead of equal to it, and + # formatting drift reached master unnoticed. Same command, same arguments. + fmt: + name: Formatting + runs-on: ubicloud-standard-2 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - name: Formatting (cargo fmt --all --check) + run: cargo fmt --all --check + build: name: Build and test (${{ matrix.name }}) runs-on: ubicloud-standard-2 - timeout-minutes: 15 + # A cold all-target workspace build plus compile-fail's nested Cargo checks + # exceeded 30 minutes when GitHub's cache service was unavailable. Keep + # enough room for a real from-scratch release gate. + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -62,6 +77,42 @@ jobs: - name: Clippy (deny warnings) run: cargo clippy --workspace --all-targets ${{ matrix.args }} -- -D warnings + cell_lock_models: + name: Archived-row lock concurrency models + runs-on: ubicloud-standard-2 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo test --release --lib cell_lock_models + env: + RUSTFLAGS: --cfg wt_loom + CARGO_TARGET_DIR: target/cell-lock-loom + + no_default_features: + name: Library without default features + runs-on: ubicloud-standard-2 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: sh scripts/check-no-std.sh -p worktable --lib --no-default-features + - run: sh scripts/check-no-std.sh --manifest-path tests/nostd-consumer/Cargo.toml + - run: cargo test --manifest-path tests/nostd-consumer/Cargo.toml + - run: rustup target add x86_64-pc-windows-gnu + - run: sh scripts/check-no-std.sh -p worktable --lib --no-default-features + env: + NO_STD_TARGET: x86_64-pc-windows-gnu + CARGO_TARGET_DIR: target/no-std-cross + - name: Portable search feature combinations + run: | + for search in wti-predictable-search wti-hybrid-search wti-std-search; do + sh scripts/check-no-std.sh -p worktable --lib --no-default-features --features "$search,logical-index-persistence,versioned-row-publication,runtime-backends" + done + - run: cargo clippy -p worktable --lib --no-default-features -- -D warnings + duplicate_index_crates: name: One version of each shared index crate runs-on: ubicloud-standard-2 @@ -91,14 +142,14 @@ jobs: echo "$duplicates" echo echo "WorkTablesIndex, data_bucket and worktable move as one train." - echo "Publish them in lockstep, or pin them to agree." + echo "Publish compatible releases in dependency order." exit 1 fi echo "one version of each: ok" publish: if: github.event_name == 'push' && github.ref == 'refs/heads/master' - needs: [build, clippy_check, duplicate_index_crates] + needs: [fmt, build, clippy_check, cell_lock_models, no_default_features, duplicate_index_crates] runs-on: ubicloud-standard-2 timeout-minutes: 45 steps: @@ -109,26 +160,49 @@ jobs: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} run: | set -euo pipefail - if [ -z "${CARGO_REGISTRY_TOKEN:-}" ]; then echo "CARGO_REGISTRY_TOKEN not set; skipping publish"; exit 0; fi + if [ -z "${CARGO_REGISTRY_TOKEN:-}" ]; then + echo "CARGO_REGISTRY_TOKEN is not set on this repository" >&2 + exit 1 + fi # Publish in dependency order. worktable_codegen depends on # worktable_dsl and worktable depends on worktable_codegen, both by - # path with an exact version, so each must be on the registry before + # path with a caret requirement, so each must be on the registry before # the next is packaged. Omitting worktable_dsl here is what made # `cargo publish -p worktable_codegen` fail with "no matching package # named `worktable_dsl` found" the moment the DSL extraction landed. + manifest_version() { + cargo read-manifest --manifest-path "$1" | jq -er '.version' + } + + is_published() { + cargo info --registry crates-io "$1@$2" >/dev/null 2>&1 + } + + wait_until_published() { + package=$1 + version=$2 + for _ in $(seq 1 60); do + if is_published "$package" "$version"; then + return 0 + fi + sleep 5 + done + echo "$package $version did not appear on crates.io in five minutes" >&2 + return 1 + } + publish_if_new() { - crate="$1" + package="$1" manifest="$2" - version=$(sed -n 's/^version = "\(.*\)"$/\1/p' "$manifest" | head -1) - # crates.io index paths: four or more characters is {first two}/{next two}/{name}. - if curl -fsSL "https://index.crates.io/wo/rk/$crate" \ - | sed -n 's/.*"vers":"\([^"]*\)".*/\1/p' | grep -qx "$version"; then - echo "$crate $version is already on crates.io; skipping" + version=$(manifest_version "$manifest") + if is_published "$package" "$version"; then + echo "$package $version is already on crates.io; skipping" return 0 fi - echo "publishing $crate $version" - cargo publish -p "$crate" + echo "publishing $package $version" + cargo publish -p "$package" + wait_until_published "$package" "$version" } publish_if_new worktable_dsl dsl/Cargo.toml diff --git a/.gitignore b/.gitignore index 021f6977..08959348 100644 --- a/.gitignore +++ b/.gitignore @@ -9,10 +9,17 @@ Cargo.lock *.DS_Store tests/data/* !tests/data/expected/ -!tests/data/persist_index_table_of_contents.wt.idx +tests/non-existent/ /.claude/settings.local.json # cargo-mutants run output /mutants.out/ /mutants.out.old/ + +# Generated documents. `docs/wt-user-guide.typ` and the columnar guide are the +# sources; the PDFs are build output and were tracked, so every edit to a guide +# put a new binary blob in the history. Rebuild with: +# typst compile docs/wt-user-guide.typ docs/wt-user-guide.pdf +*.pdf +.DS_Store diff --git a/AGENTS.md b/AGENTS.md index b7f0dc9b..810c0f24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ natively, and Claude Code loads it through the `@AGENTS.md` import in - **Keep `cargo fmt` and `cargo clippy --workspace --all-targets -- -D warnings` clean.** Lint failures are part of the build here, not advisory. Note `--workspace`: without it `worktable_codegen` is never linted, and note `-D warnings`, because CI denies what your terminal merely prints. - **Shell scripts are POSIX `sh`.** `#!/bin/sh`, and none of `[[ ]]`, arrays, `echo -e`, process substitution or `pipefail`. Check with `sh -n` before committing. Bash is not guaranteed to be the system shell, and a script that only runs on one machine is not a check. - **Publishing to crates.io is irreversible.** A version number can never be reused, and yanking does not delete. Run `cargo publish --dry-run` first, publish from the merged default branch, and tag the release. -- **A pre-release version (`-alpha`, `-beta`) needs an exact dependency pin.** A plain `"2.0"` requirement will not match `2.0.0-alpha.1`, so consumers must be bumped deliberately. +- **Use caret dependency requirements, including our prerelease packages.** Name the prerelease explicitly, for example `^1.0.0-beta.19`; `^1.0` alone does not opt into prereleases. Do not introduce exact pins. Record resolved versions in release and benchmark evidence. - **Docs describe what is true now.** If you change behaviour, update the README and any affected doc in the same change. ## Build & test diff --git a/CHANGELOG.md b/CHANGELOG.md index b37f3eed..ec079e55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,349 @@ Change Log ========== +## [1.9.0-alpha1] + + +### Changed + +- **`s3-support` no longer drags in an async HTTP stack, and no longer needs a + tokio reactor.** It used `reqwest`, which meant hyper, h2, tower and tokio — + 91 crates — to make four calls: a PUT and three GETs against presigned URLs, + with no streaming, no multipart and no auth headers, because `rusty-s3` puts + the signature in the URL. + + Worse than the size, it was silently incompatible with the default runtime. An + S3 write from the persistence worker panicked with `there is no reactor + running, must be called from the context of a Tokio 1.x runtime`, because the + worker runs on nagoya and `reqwest` looks for tokio's thread-local handle. + `cargo test --all-features` had been red on that for as long as the feature + existed. + + It now uses a blocking client. The `async fn` signatures are unchanged, so no + call site moved; the blocking happens inside them, which is what this crate's + filesystem layer already does deliberately — neither `tokio::fs` nor + `async-fs` performs asynchronous file I/O either, and `fsx` measured 12,316 + rows/sec through `tokio::fs` against 74,728 blocking. + + | | before | after | + |---|---:|---:| + | crates in the s3 build | 198 | **170** | + | what the feature costs | 91 | **63** | + | tokio present | yes | **no** | + + A persistence worker makes one request at a time from its own thread, which is + the shape a blocking call fits. Async HTTP exists to multiplex many + connections onto few threads, which this is not. + +### Added + + +- **`queries:` on a `vec: true` table.** It was refused wholesale; it now + generates `update_`, `delete_` and `update__in_place` under + the same names the paged table uses, so a declaration reads the same either + way. + + These are named wrappers rather than a new execution path: a declared update + is `update(&pk, |row| ..)` with the columns filled in from a generated struct, + and `update` already repairs every index the edit moved a row under. That + makes delegating to it both the shortest implementation and the only one that + cannot get index repair wrong in a second place. + + `by` may name the primary key, a unique secondary, or a non-unique secondary. + All three are **equality** lookups, which is the only shape a declared query + has, so every backend answers them — **including `fxhash`**. Nothing here + needs an ordered index, which is why these are emitted whatever the `using` + clause says while `range` and `range_by_` are not. The restriction on a hash + index is ordering, not queries. + + A non-unique key names many rows, so those methods return how many they + touched rather than whether they touched one. A `by` column with no index is + refused, naming the index to add: scanning instead would turn a keyed + operation into a linear one silently. + + The signatures differ from the paged table's on purpose — synchronous, and + `&mut self` — so a call cannot move silently between the two shapes. +- `using fxhash`, a hash-shaped index backend. Accepted on `vec: true` and + **refused on a paged table**, which is the whole story: `UniqueIndex` requires + `range_values` and `range_links` and a hash map cannot answer either, a paged + table generates `select_by__range` for every index, and a persisted + index's on-disk form *is* sorted pages — `from_persisted` rebuilds each one + with `attach_nodes`. The `vec: true` generator is the one that asks its index + only for point operations and a single order-independent walk, so it is the + one place a hash map fits. + + Worth 4.9x on build and 4.0x on lookup at a million rows against the default + arctic backend (`perf-benchmarks/benchmarks/fx-index.rs`), which is a larger + factor than anything else in the backend list. + + A table using it has **no `range` and no `range_by_`**. Not a method that + panics and not one that returns insertion order while claiming key order: the + methods are not generated, so asking for one is a compile error at the call + site. A paged declaration that asks for it is refused with an error naming + `vec: true`. + + `with_capacity` now reserves an `fxhash` index alongside the row vector, and + that is most of the build win — without it the same table measured 2.4x rather + than 4.9x. It reserves nothing for the tree backends, deliberately: making + allocation completely free measures at **0.92x** for Arctic, below one, + because it changes where nodes land and sequential order is worse for a tree + walked in key order. +- Ranges on a `vec: true` table: `range(bounds)` by primary key and + `range_by_(bounds)` for each unique secondary index, both + `DoubleEndedIterator` so `.rev()` works. This cost nothing to add and was + simply never exposed. Every backend the `using` clause can name is an ordered + tree, `UniqueIndex` has always required `range_links`, and the index was + answering ranges the whole time. + + It is not a sorted vector: the keys arrive in order and the rows they name are + wherever insertion put them, so a long range is a walk of random accesses. + `range_by_` is emitted for unique indexes only, because a non-unique one holds + a posting list per key and has no single row to yield. + +- Ghosted deletes on a `vec: true` table, with `compact` to reclaim. `delete` is + now O(1): the row leaves its slot and its index entries, and no other position + changes. It used to close the hole with `Vec::remove`, which meant a memmove of + every row above it plus a rewrite of every index entry above it — **21 + milliseconds per delete at a million rows**, so two hundred deletes took four + seconds. + + The cost is that slots accumulate until `compact()` is called, which is the + paged table's ghost-and-vacuum model applied to a vector. `ghost_count()` and + `slots()` report the state so a caller can decide when compaction is worth its + cost; `compact()` keeps the row vector's capacity for reuse and + `shrink_to_fit()` gives it back. + + Two consequences worth reading before upgrading. `select_all()` returns + `impl Iterator` instead of `&[Row]`, because with a hole in it the + live rows are no longer a contiguous slice — call `.iter()` on the result no + longer, and `.count()` where you had `.len()`. And a slot now costs + `size_of::>()`, which for a row with no spare bit pattern is the + row plus its alignment; `used_bytes()` counts slots for that reason. + +- `vec: true` composes with `partition_by`. It was refused, on the grounds + that a `Vec` table "is one contiguous `Vec` and has nothing to partition", + which reads the relationship backwards: partitioning is what makes the `Vec` + shape correct, because a `Vec` table is single-writer and grows linearly and + cutting the data into many small independent ones is how you stop both from + mattering. + + The router needs `Default`, `used_bytes` and `row_count` from whatever it + holds. A `vec: true` table already had the first, and now has the other two. + Its `insert` still takes `&mut self`, so a partition is populated and then + handed over with `partition_or_insert_with` rather than mutated through the + `Arc` the router returns. + +- `AtomicKeyTable`, ported from `worktable-vec`, which is now deprecated: this + was the last thing in that crate living nowhere else. A fixed-capacity, + open-addressed table whose rows are claimed with one `compare_exchange` and + then updated through `&V`, so many writers share a key without a lock and + without a reallocation. The case it exists for is a counter table: sixteen + workers recording timings against a handful of named sites. + + **There is no row snapshot, and there will not be one.** A reader of two + fields reads two atomics, so a count of 10 beside a total of 900 is + observable even though no writer left the row that way. A sequence lock or a + lock per row would put back the contended cache line the type exists to + avoid. A caller who needs two values to agree packs them into one atomic: two + `u32` counters in an `AtomicU64`, one `fetch_add`, one load. That is the + supported answer and there is a worked example in the tests. + +- A lint on a narrow primary key. `u8` or `bool` as the primary key of an + **unpartitioned** table means a table that can never hold more than 256 or 2 + rows, which is usually a key that was meant to be wider. Beside + `partition_by` the same key is correct and the lint is silent: a narrow key + is what makes a dense partition possible, and warning about it there would be + telling people to undo the optimisation. + + A lint and not a ban. It arrives as a deprecation warning, because a proc + macro cannot emit one directly; the note names the column and the row count, + and `#[allow(deprecated)]` on the module turns it off for a table where 256 + rows is what was meant. Everything it emits lives inside an anonymous `const` + and is not nameable. + +- A **dense partition**, generated when `partition_max_size` is `bool`, `u8` + or `u16`. `DenseTable` addresses rows by position: the primary key *is* + the row's index, so there is no primary index, no pages, no links, no free + list, no epoch domain, no lock map and no CDC. A lookup is a bounds check and + a load. + + Measured on one declaration at two widths, 200 partitions of 23 rows, + counting what the allocator was asked for: + + | | bytes per partition | + |---|---:| + | full table, empty | 28,404 | + | **dense, empty** | **108** | + | full table, 23 rows of an 88-byte row | 32,900 | + | **dense, same** | **3,180** | + + The empty figure is the one to read. The saving is fixed apparatus allocated + at partition creation, so it is about 28 KB per partition whatever the row + width is; the ratio falls as rows get wider only because the rows themselves + grow. At 2,000 symbols that is roughly 56 MB. + + `insert`, `upsert`, `update`, `delete` and `select` all take `&self`, because + `partition_or_create` hands out an `Arc`. A generated `update_` edits + one field in place without cloning the row. Writes serialise per partition + rather than per cell, which is stated in the module documentation rather than + implied: nothing here is async, so no write spans a suspension point, and a + partition is a much smaller thing to lock than a table. + + The width is a **bound, not a reservation**: the row vector grows to the + highest key used, so a `u16` partition holding three rows holds three slots + and an empty one allocates nothing at all. + + Refused, by name and with the way out: a primary key that is not a single + unsigned column, a width the key cannot count to (`u16` beside a `u8` key + declares 65,536 rows into a partition that holds 256), and `persist: true`, + which a dense partition has no engine to honour. + + It carries `queries:`. An `update` or `delete` query keyed by the primary key + generates the same method name against the same `Query` struct the + paged table generates, so a call reads identically; the signature does not, + deliberately, because there is no `.await` and no `WorkTableError`, and a + call that moved between the shapes should fail to compile rather than + quietly change what it guarantees. A query keyed by any other column is + refused: a dense partition has no secondary index, and scanning it instead + would be a keyed operation silently becoming a linear one. `in_place` is + refused as a synonym, because every update here is already in place. + + Note that `memory_by_key` and `memory_total` **cannot see this saving**. They + report `used_bytes`, which is rows plus indexes and excludes the fixed floor + by definition, so the two shapes measure the same through them. That is + pinned by a test so the conclusion is not drawn twice. + +- `partition_max_size`, required beside `partition_by`. It says how many rows a + single partition holds, written as an index width rather than a count: + `bool` is 2 rows, `u8` is 256, `u16` is 65,536, and `u32` or `u64` mean + unbounded in practice. It is positional, directly after `partition_by`. + + A **type** and not a literal, for the same reason `columnar_slot_id: + ColumnSlotId16` already is one: it is an index width, which is what the + generator needs, and a count is not a power of two and duplicates a constant + that lives in the caller's code and will drift. + + Required rather than defaulted, because a default would pick one of the two + shapes for the author and generate the other one silently. Nothing in a + partitioned declaration said which shape it was getting: `exchange_id: u8 + primary_key` reads as a big table with a suspiciously tiny key, when the + truth is many little tables that each need only a byte, and two declarations + differing by 28 KB a partition looked identical. + + **Breaking for any existing partitioned declaration.** Adding + `partition_max_size: u64,` after `partition_by` restores exactly the previous + behaviour. + +- `no_std` support. A consumer with `default-features = false` can invoke + `worktable!` and use `insert`, `select` and `select_all`. Verified by + `tests/nostd-consumer`, a crate outside the workspace that invokes the macro: + this crate builds without `std` whether or not the macro is sound, because the + expansion only happens where the macro is called. +- Columnar fields and columnar indexes: `columnar(chunk_rows(n), + compression(name))` on a column, a `columnar_indexes` block with `cluster_by`, + and `columnar_slot_id` / `columnar_chunk_rows` in `config`. +- A schema-selected runtime: `runtime: nagoya()` or `runtime: tokio`. + Six flavors, all sharing one pool implementation, so the choice costs no extra + code and no rebuild. +- `page_size` on a persisted table, at any size with a 512-byte floor. It was + refused outright while the on-disk seeks used a hardcoded constant. +- `vec: true`, a `worktable!` whose rows live in one contiguous `Vec` with + an index of positions into it, and which pays for none of the paging, + archived rows, lock map, CDC or async surface a paged table carries. + + It is a key rather than a second macro. `worktable_vec!` existed briefly and + emitted `VecRow` and `VecTable`, which is a parallel vocabulary + to learn and a redefinition error when one table was declared both ways. One + macro means one `Row` and one `WorkTable` whatever the storage + is. `vec` is positional: name, version, vec, persist, partition_by, + partition_max_size, then the blocks. + + It is a flag rather than a `storage:` key, which is what it was called for + half a day. The grammar keeps the shape `persist:` already has and gains no + new noun; the model still resolves it to one enum, because the schema is + serialized, round-tripped and handed to a TypeScript emitter, and serde + enforces no cross-field invariant. + + The two are **not** interchangeable, deliberately. The signatures differ four + ways, so moving a declaration between them fails to compile at every call + site rather than silently weakening its guarantees. A paged `insert` is + `async fn(&self, Row) -> Result`; a vec one is + `fn(&mut self, Row) -> Result<(), Row>`. Select clones a row out of the first + and lends one from the second. + + `persist`, `queries`, `columnar_indexes`, `runtime`, `partition_by` and + `config` are refused with an error naming what to use instead, rather than + accepted as no-ops. `persist` in particular: this table has no engine, no + task and no flush, so it pays no synchronisation for durability it was not + asked for. Rows go to bytes and back when you call for it. + + It honours `using` as a paged table does and defaults to the same backend: + arctic, with `worktables_index`, `congee` and `indexset` (a plain `BTreeMap`) + available. A non-unique index needs a multimap, which only arctic and + indexset have, so the other two are refused for one by name. + + Measured at 200,000 rows, nine interleaved rounds, p50: 6.4 ms against the + 13.6 ms a hand-written `Vec` plus `BTreeMap` takes, and level with + `worktable-vec`'s own `ArcticTable` at 6.4 ms. 10.0 ms once it also + maintains a secondary index nothing it is compared against has. + + `using indexset` is the reason to pick `BTreeMap` deliberately: `delete` + moves every position above the hole, which a `BTreeMap` does in place and an + ART does by reinserting each affected entry. +- `vec: true` generates `unload` and `load`: rows out + as 16 KiB pages and back, each page standing alone so damage is local and an + append does not rewrite the file. Every page carries a CRC-32 of its body and + a row directory, and a row-type fingerprint refuses another table's file + rather than reading it as debris. + + The indexes are not written. They are positions into the row vector, so they + are rebuilt on load, which is cheaper than writing, validating and keeping + them consistent with the rows on disk. + + The codec is ported from `worktable-vec`'s `hydrate`, where the format was + designed. **The files are not interchangeable**: that crate stores + `Vec<(K, V)>` because its value type has no key in it, and a `worktable!` row + already carries its primary key as a column, so this stores `Vec` and + does not write the key twice. Different archives, different fingerprints, and + the fingerprint is what turns that from silent misreading into a refusal. + + rkyv's derives are unconditional, since `persist` is refused and there is + nothing left to gate them with. Measured at 20 tables of five columns: 305 ms + without them, 470 ms with, so about 8 ms a table. Real, and not worth a key. +- The default index backend is `arctic`, not `worktables_index`. A composite + primary key keeps `worktables_index`, because arctic cannot represent a tuple + key. **Arctic cannot key an optional or variable-width column**, so an index + over `String optional` must now say `using worktables_index` where it + previously needed nothing. +- Only `congee` requires `persist` to be stated explicitly. Arctic no longer + does, having become the default. +- The filesystem goes through `worktable::prelude::fsx` and names no async + runtime. Measured: `tokio::fs` ran scattered updates at 12,316 rows per second + against 74,728 on `std::fs`. + +### Fixed + +- A page gap in the persisted batch save. Two writers allocating at once could + hand the queue the higher page first, leaving the skipped ids as holes of + zeros that the file spanned; the next batch touching one parsed the hole and + looked up a key it never held. Reduced from a 40 minute reproduction to a + 0.01s unit test. +- Batch collection was quadratic on scattered writes. It grouped by page while + validity is decided by event order, so a workload writing to scattered pages + re-collected almost everything each round: 4,000 operations cost 202,000 + collections and 198,000 requeues. Selection is event-ordered now, 16.1s to + 0.14s. +- A 500 ms sleep on every collection retry that needed no wait. +- Eight tests named for concurrency ran on a current-thread runtime, where + spawned tasks never overlap. +- The macro emitted names a `no_std` consumer could not resolve, and + `futures::future::join_all` where the prelude should have been. +- `worktable-schemas` counted the `tests/ui` refusal corpus as rejections, so it + reported nine failures on a healthy tree. +- `Schema` parsed `columnar_indexes` and dropped it, so `to_dsl` emitted a + columnar table without its clustering and every consumer downstream, including + the TypeScript emitter, was blind to it. + ## [1.0.0-beta.19] ### Changed @@ -57,6 +400,697 @@ Change Log - Persisted primary/secondary index reconstruction and validation failures that could otherwise expose missing, duplicate, or mismatched rows. +## [1.0.0-beta.17] + +### Added + +- `worktable_dsl`, a standalone crate holding the schema language. A schema can + now be read as data and written back, two schemas can be compared and the + cost of the difference reported, and declarations can be found across a + source tree. +- Every generated table embeds its own declaration, so the schema is + recoverable from the code the macro produced. + +### Changed + +- Dependency requirements on the index and reclamation crates are carets rather + than exact pins, and `ps-reclaim` moved to 0.1.1 taken from the registry. +- Retirement runs through the reclamation domain rather than through the guard. + +## [1.0.0-beta.16] + +### Changed + +- A batch pins its reclamation domain once instead of once per row, and + reclamation goes through `ps-reclaim`. +- Requires data_bucket 0.5.5. + +## [1.0.0-beta.15] + +### BC Breaks + +- Non-unique index entries are identified and ordered by their `(key, value)` + pair, and the discriminator is gone. This is a persisted format change. An + index file written by beta.14 or earlier orders entries within a key by + discriminator, so it must be reindexed rather than loaded. + +### Fixed + +- Inserting into a non-unique index no longer scans every entry sharing the + key. On a table that puts a whole generation under one key, a one-file update + measured 698 ms on beta.13 and 15.1 s on beta.14; the per-row cost is back + from 330 us to roughly 9 us. +- Index pages reconstruct in order of their minimum rather than their node id, + so a page that merely ends late no longer sorts ahead of one that starts + earlier. + +## [1.0.0-beta.14] + +### Added + +- `insert_many` with all-or-nothing semantics and CDC batch operations, and + `reserve_pks` for atomic primary key range reservation, both generated on + in-memory and persisted tables. +- Per-table epoch pin domains. The global reader counter is replaced by + epoch-based retirement reclamation, and removed partitions are reclaimed + through the shared router under the same grace period. +- Non-unique Arctic indexes for fixed-width integer keys, generated for + in-memory and persisted tables, with a pair-list checkpoint and WAL. + +### Changed + +- The persistence queue takes batches on a single wakeup, deduplicates page + queries when collecting a multi-row batch, and caps rows per group id so the + analyzer drain stays linear. +- The table-global page barrier is narrowed to one barrier per page. +- Requires WorkTablesIndex 0.0.8 and data_bucket 0.5.4. + +### Fixed + +- Mutation stripes are acquired as a batch without deadlocking. +- A unique-collision unwind on a persisted table survives reload. + +## [1.0.0-beta.13] + +The audit-fix release. Most of it is durability and concurrency correctness +rather than new surface. + +### BC Breaks + +- Persisted tables reject any `page_size` other than 16384 instead of writing a + file that cannot be read back. +- A torn table-of-contents page 1 fails loudly instead of silently starting + from an empty table. +- In-place update is rejected on indexed columns rather than leaving the index + stale. +- An exhausted autoincrement generator panics instead of wrapping around and + handing out keys that are already in use. + +### Fixed + +- A failed data write no longer leaves published index keys behind. Insert, + update and delete each roll their index changes back. +- Index pages are written before the table of contents that references them, + and table-of-contents key updates are guarded against segment overflow. +- Data-file accounting: the u32 page-offset wrap when writing the last page's + data length, files whose length is an exact page multiple failing to reopen, + and non-extending writes being counted into the last page's length. +- Vacuum no longer panics on a failed row move, no longer counts its scratch + pages in `pages_freed`, and never reports a source page fully moved when a + row was skipped. +- A cancelled lock wait releases its registered op-lock, and a row that + vanishes mid-update returns `NotFound` instead of panicking. +- The persistence worker refuses new operations once `Drop` has aborted it, + propagates `insert_cdc` serialization failure instead of panicking, and keeps + surviving data-only writes when event removal empties a batch. +- ART checkpoints are atomic and clean up stale temporaries. +- Arctic returns an empty range for `Excluded` bounds with no neighbour. +- Row counts include inserts and deletes that reused a slot, and the + `PageIsFull` page switch is serialized against racing inserters. +- A misplaced `persist` or `partition_by` in a declaration now names the + position it belongs in. + +## [1.0.0-beta.12] + +### Added + +- `partition_by`: one declared table type, many routed instances, with + `partition_ref` for borrowing a partition rather than cloning it. + +### Changed + +- `system_info` no longer copies every data page, and partition metrics scan + and allocate once instead of three times. + +### Fixed + +- A use-after-free in partition removal. +- `close` reported success having persisted nothing. +- One panic inside the router no longer disables the router. + +## [1.0.0-beta.11] + +### Changed + +- Requires the reviewed ART backend releases. + +## [1.0.0-beta.10] + +### Fixed + +- Table-of-contents inserts carry across persisted segments, reload insertion + stays on the fast path, and the insert API keeps its previous shape. + +## [1.0.0-beta.9] + +### Fixed + +- Persistence health is preserved across page splits. + +## [1.0.0-beta.8] + +### Fixed + +- Multi-row persistence order is preserved, and overlapping durable row writes + are ordered against each other. + +## [1.0.0-beta.7] + +### Fixed + +- The sized indexed update path is preserved. + +## [1.0.0-beta.6] + +### Changed + +- Fixed-width updates stay in place. + +### Fixed + +- Vacuum revalidates links after row locking. + +## [1.0.0-beta.5] + +### Added + +- Checked offline recovery load. + +### Changed + +- WorkTablesIndex structural persistence moved off the mutation path. +- Logical WTI mutation stripes hash with FxHash, reusable data ranges are + subtracted in one pass, and full-row updates generate distinct paths. + +### Fixed + +- Same-size unsized updates apply in place instead of going through reinsert. +- Full-table scans re-resolve stale links. +- Vacuumed pages are reusable after reload. +- Cancelled lock acquirers are cleaned up. +- A panic while loading a persisted table is contained instead of unwinding + into the caller. + +## [1.0.0-beta.4] + +### Fixed + +- Release hardening and torn-store refusal are consolidated, so a torn store + refuses cleanly. + +## [1.0.0-beta.3] + +### Changed + +- Depends on the published index dependency chain rather than git revisions. +- Row publication is concurrency-safe by default. +- Persistence failures are terminal instead of leaving the table in a state + that looks usable. + +### Fixed + +- Synchronous insert is serialized against row mutations. +- Page and link reclamation no longer overlap, and vacuum page reuse is + deferred through the read grace period. +- Upsert retry backoff is bounded and its shift is capped, so same-key churn + cannot livelock. +- Fragmented unsized index pages are compacted. +- A stale multimap removal lookup is avoided. + +## [1.0.0-beta.2] + +### Added + +- Native ART index backends persist. + +### Changed + +- The temporary rusty-s3 fork is retired in favour of the published crate. +- Stable index reads use the specialized path by default. + +### Fixed + +- Same-key upserts linearize. +- Bounded retry for transient index misses is gated rather than always on. +- Reused persistence slots coalesce. + +## [1.0.0-beta.1] + +### Added + +- Per-index backend selection in the `worktable!` declaration, with unique-index + adapters for Arctic, Congee and a parallel upstream indexset. Persistence is + preserved across indexset providers. + +## [0.9.4] + +### Changed + +- Requires data_bucket 0.4.1, and the temporary git patch is retired. + +### Fixed + +- A torn store refuses cleanly instead of terminating the process by signal. + +## [0.9.3] + +### Fixed + +- `worktable_version!` stays read-only when the primary key is unsized. + +## [0.9.2] + +### Fixed + +- Duplicate-key secondary indexes reconstruct correctly on reload. +- Nodes sharing a maximum key order correctly, and pages are no longer re-sorted + on reload. +- Space files flush before an operation reports done. + +## [0.9.1] + +### Changed + +- The proc-macro crate is published as `worktable_codegen` again, after a brief + release under the name `worktable_macros`. + +## [0.9.0] + +### Changed + +- Moves to WorkTablesIndex 0.0.1 and data_bucket 0.4.0. +- The unsound lock-free persistence queue is replaced with a mutexed + `VecDeque`. + +### Fixed + +- Row lock acquisition and vacuum no longer race between check and act. +- `wait_for_ops` no longer returns while a popped operation is still in flight. +- Upsert retries an existence flip instead of surfacing it to the caller. +- A multi-row update locks one validated snapshot, predicate included, and + delete by non-unique index snapshots validated primary keys. +- Gapped event streams are never force-applied to the on-disk index, and the + whole batch is scanned for event-id gaps rather than the last thirty events. +- A failed batch sub-operation is reported without cancelling the rest of the + work. +- `save_batch_data` tracks the real maximum created page id. +- Vacuum persists row moves through CDC, so persisted tables survive + defragmentation. + +## [0.9.0-beta0.2.3] + +### Fixed + +- Primary key generator state is preserved across migration reinserts. + +## [0.9.0-beta0.2.2] + +### Changed + +- Range ordering query logic reworked. + +## [0.9.0-beta0.2.1] + +### Changed + +- Update locks spin before returning a `Pending` state. + +## [0.9.0-beta0.2.0] + +### Added + +- Migrations. + +## [0.9.0-beta0.1.4] + +### Fixed + +- Page-not-found bug in the table of contents. + +## [0.9.0-beta0.1.1] + +### Fixed + +- Persistence bug affecting operations that fail. + +## [0.9.0-alpha8] + +### Changed + +- S3 integration moves to a different client crate. + +## [0.9.0-alpha7] + +### Fixed + +- S3 integration bug. + +## [0.9.0-alpha6] + +### Changed + +- Moves to rustls. + +## [0.9.0-alpha5] + +### Added + +- nanoid support for primary keys. + +## [0.9.0-alpha4] + +### Fixed + +- Vacuum logic. + +## [0.9.0-alpha3] + +### Fixed + +- The S3 macro. + +## [0.9.0-alpha2] + +### Added + +- S3 sync feature. + +## [0.9.0-alpha1] + +### Changed + +- Persistence is moved behind separate traits. + +## [0.8.23] + +### Changed + +- `Lock`s are reworked around RAII guards. + +## [0.8.22] + +### Added + +- `MemStat` derive on the generated primary key type. + +### Changed + +- `DataPages` select is generic over the input link type. + +## [0.8.21] + +### Changed + +- `delete` is generic, matching `insert` and `update`. + +## [0.8.20] + +### Added + +- Vacuum. + +## [0.8.19] + +### Fixed + +- Optional fields in persisted tables. + +## [0.8.18] + +### Fixed + +- Persisted table code failed to compile when the declaration used `optional` + fields. + +## [0.8.17] + +### Changed + +- Updated `indexset`. + +## [0.8.16] + +### Changed + +- Dependencies are pinned to exact versions. + +## [0.8.15] + +### Fixed + +- Empty link registry. + +## [0.8.13] + +### Changed + +- Bumped `indexset`. + +## [0.8.12] + +### Changed + +- Bumped `data_bucket` to 0.3.5 and `wt-indexset` to 0.12.11, and the crate now + declares its repository. + +## [0.8.11] + +### Changed + +- Bumped `indexset`. + +## [0.8.10] + +### Changed + +- Bumped `data_bucket` to 0.3.3 and `wt-indexset` to 0.12.9. + +## [0.8.9] + +### Fixed + +- Empty node bug. + +## [0.8.8] + +### Added + +- Every `AtomicU*` and `AtomicI*` type is usable as a primary key. + +## [0.8.7] + +### Changed + +- Dependency bumps. + +## [0.8.6] + +### Fixed + +- An `update`-related bug. + +## [0.8.5] + +### Fixed + +- Another `update`-related bug. + +## [0.8.4] + +### Changed + +- Codegen version bump. + +## [0.8.3] + +### Fixed + +- `delete` queries on a table whose primary key is not named `id`. +- An update bug, by way of an `indexset` update. + +## [0.8.1] + +### Added + +- The macro reports an error when an index names a column that does not exist, + and declaration errors are raised as `syn::Error`s with usable messages. + +### Fixed + +- `UnsizedNode` split. + +## [0.8.0] + +### Fixed + +- Unsized node bug. + +## [0.7.2] + +### Fixed + +- A further `update` bug. + +## [0.7.1] + +### Fixed + +- An update violation. + +## [0.7.0] + +### Fixed + +- Reinsert bug. + +## [0.6.14] + +### Added + +- Ghost inserts. A row is staged invisible and becomes visible only once its + index entries are in place, so a concurrent reader never observes a + half-inserted row. + +## [0.6.13] + +### Fixed + +- Concurrency bugs in `select`. + +## [0.6.12] + +### Fixed + +- A further locking bug. + +## [0.6.11] + +### Fixed + +- Locking bugs for unsized types, and an `UnsizedNode` bug on `update`. + +### Changed + +- Dependency bumps. + +## [0.6.10] + +### Changed + +- Republished against `worktable_codegen` 0.6.9. No library change. + +## [0.6.9] + +### Fixed + +- Concurrent persistence issues. + +## [0.6.8] + +### Fixed + +- `wait_for_ops` logic. + +## [0.6.7] + +### Fixed + +- `delete` on persisted tables. + +## [0.6.5] + +### Added + +- Custom derives can be attached to the generated row type. + +## [0.6.4] + +### Fixed + +- `uuid` usage. + +## [0.6.3] + +### Fixed + +- A debug `println!` on the persistence batch path no longer writes to stdout. + +## [0.6.2] + +### Fixed + +- Table-of-contents corrections. + +## [0.6.1] + +### Added + +- `update_in_place`. + +### Changed + +- The persistence queue is optimized. + +### Fixed + +- `insert` with an already-existing key. +- A `use rkyv::Archive` import was required for some declarations. +- `wait_for_ops`. + +## [0.5.6] + +### Changed + +- Updated `indexset`. + +## [0.5.5] + +### Fixed + +- Array-typed fields. + +### Changed + +- Moves to the newer Rust edition. + +## [0.5.4] + +### Added + +- Unsized index space, so index keys are no longer limited to fixed-width + types. +- `SystemInfo` for the table and its indexes. +- `where_by` on `SelectBuilder` for any column, indexed or not. +- Float columns are usable in indexes, including ranges. + +### Fixed + +- Re-reading a table from file. +- Index difference logic for `update` queries. + +## [0.5.1] + +### Changed + +- Persistence I/O is asynchronous. + +## [0.5.0] + +### Added + +- `select_where_{field}` queries for selecting data ranges. +- `count` on the table. +- Persist sync logic. + +### Changed + +- Non-unique indexes are backed by `IndexMultiMap`. + +### Fixed + +- Secondary index left inconsistent after an update. +- Diff logic for a full-row update. + ## [0.4.1] ### Added diff --git a/Cargo.toml b/Cargo.toml index e32eb50c..095500bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["codegen", "dsl", "examples", "performance_measurement", "performance [package] name = "worktable" -version = "1.0.0-beta.19" +version = "1.9.0-alpha1" edition = "2024" authors = ["Handy-caT"] license = "MIT" @@ -15,9 +15,32 @@ keywords = ["database", "embedded", "in-memory", "index", "storage"] categories = ["database-implementations", "data-structures", "caching"] [features] -default = ["wti-predictable-search"] -perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"] -s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktable_codegen/s3-support"] +default = ["std", "wti-std-search", "vanilla-index"] +# `futures/std` belongs here rather than on the dependency +# line: written as `features = ["std"]` beside the version, a +# `--no-default-features` build of this crate would still turn them on. The +# s3 tests reach `futures::io`, whose traits live behind that feature. +std = ["nagoya/std", "ps-reclaim/std", "congee/std", "indexset/std", "futures/std", "worktable_codegen/std", "dep:arc-swap", "dep:convert_case", "dep:eyre", "dep:worktable_dsl", "uuid/std", "psc-nanoid/std"] +# The tokio backend for `Runtime`, selectable with `runtime: tokio` in a +# schema. **Off, and it stays off.** Getting tokio out of the normal dependency +# graph is the work this builds on: it used to arrive through the `tokio::` +# paths `worktable!` emitted into consumer crates, so a runtime was part of the +# macro's contract whether a consumer ran one or not. The check is +# `cargo tree -e normal -i tokio` printing nothing in the default feature set. +tokio-runtime = ["dep:tokio", "std"] +# The upstream IndexSet backend, selectable per index with `using indexset`. +# Optional because it reaches `crossbeam-utils`, `parking_lot` and `serde`, +# none of which build without `std`, and because it is one backend of four +# rather than something the table needs. +vanilla-index = ["dep:vanilla_indexset", "std"] +perf_measurements = ["std", "dep:performance_measurement", "dep:performance_measurement_codegen"] +# Test-only, and empty on purpose: it compiles the four-backend arms of +# `tests/worktable/runtime_backends.rs`, which declare `runtime:` on a table. +# Off by default because the DSL keyword and the `Runtime` trait land +# separately, and a default-on flag would make this branch red until they do. +# The arm that runs today declares no runtime and is not behind this flag. +runtime-backends = [] +s3-support = ["std", "data_bucket/s3-support", "dep:blake3", "dep:rusty-s3", "dep:url", "dep:ureq", "dep:walkdir", "worktable_codegen/s3-support"] # Moves unique WorkTablesIndex structural CDC work out of the table mutation # path and into the background persistence worker. The persisted page format # is unchanged, so stores remain readable with or without this feature. @@ -25,7 +48,7 @@ logical-index-persistence = ["worktable_codegen/logical-index-persistence"] wti-hybrid-search = ["indexset/wt-slice-binary-search"] wti-predictable-search = ["indexset/custom-binary-search"] wti-std-search = ["indexset/std-binary-search"] -wti-superslice-search = ["indexset/superslice-binary-search"] +wti-superslice-search = ["std", "indexset/superslice-binary-search"] # Compatibility no-op: immutable row publication is mandatory for the safe # generated API, including `default-features = false` builds. versioned-row-publication = ["worktable_codegen/versioned-row-publication"] @@ -35,54 +58,133 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"] [dependencies] # Read-mostly snapshots own page Arcs while the fixed directory supplies a # pointer-only fast path. Publication is append-only and asserted at each swap. -arc-swap = "1" +arc-swap = { version = "1", default-features = false, optional = true } async-trait = "0.1" -arctic = { package = "arctic-wt", version = "^0.1, >=0.1.9", default-features = false, features = ["smr-ps-reclaim"] } -congee = { package = "congee-wt", version = "^0.4, >=0.4.4" } -convert_case = "0.6" -crc32fast = "1" -data_bucket = { version = "^0.5, >=0.5.7" } -derive_more = { version = "2", features = ["from", "error", "display", "debug", "into"] } -eyre = "0.6" -fastrand = "2" -futures = "0.3" -indexset = { package = "WorkTablesIndex", version = "^0.0, >=0.0.12", default-features = false, features = ["concurrent", "cdc", "multimap"] } -vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"] } +arctic = { package = "arctic-wt", version = "^0.1", default-features = false, features = ["smr-ps-reclaim"] } +blake3 = { version = "1", optional = true } +# `default-features = false`, or its `std` turns `ps-reclaim/std` on and this +# crate links the standard library through a dependency. That matters for +# EKOPathRS bootstrapping, which is what `no_std` here is for, not for any +# bare-metal target: we ship on Linux, macOS and Windows. +congee = { package = "congee-wt", version = "^0.4", default-features = false } +convert_case = { version = "0.6", default-features = false, optional = true } +crc32fast = { version = "1", default-features = false } +# 0.7 supplies the v3 row directory, integrity checks and configurable stride. +data_bucket = { version = "^0.7" } +derive_more = { version = "2", default-features = false, features = ["from", "error", "display", "debug", "into"] } +eyre = { version = "0.6", optional = true } +hashbrown = "0.15" +futures = { version = "0.3", default-features = false, features = ["alloc"] } +# The file traits and the host implementation. `std` here; the crate takes it +# without default features wherever the filesystem is not involved. +# The current 0.1 runtime supplies cancellation/panic handling and worker +# wake ownership. Runtime::with_tuning marks pool workers correctly. Compatible +# updates remain open; update ps-st3 in existing lockfiles for scheduler fixes. +# +# It also re-exports `Tuning`, which is why `ps-st3` is no longer a direct +# dependency here: it was named for that one type. +nagoya = { version = "^0.1", default-features = false } +indexset = { package = "WorkTablesIndex", version = "^0.0", default-features = false, features = ["concurrent", "cdc", "multimap"] } +# Pulls `serde` and `serde_core` into every build, and nothing here uses them. +# They are not reachable from this line: `indexset` 0.15 declares `ftree` with +# `features = ["serde"]` unconditionally, so no `default-features = false` here +# removes them and forking `ftree` would not either. This is one of four +# selectable index backends and making it optional is the fix. +vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"], optional = true } # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "^0.12", features = ["concurrent", "cdc", "multimap"] } -log = "0.4" -ordered-float = "5" -parking_lot = "0.12" +log = { version = "0.4", default-features = false } +ordered-float = { version = "5", default-features = false } +# The house fork, not vanilla: `parking_lot_lite_hack` is what every other +# consumer here takes. Same crate name via `package`, so no call site changes. +# +# Vanilla is still in the default-feature graph, and no longer for the reason +# it was: it used to arrive through tokio, and now arrives through `indexset +# 0.15` behind the `vanilla-index` default feature. So it is gone from a +# `--no-default-features` build and present in a normal one, until that backend +# is deselected. +# +# FairMutex first shipped in 0.12.8. Keep the compatible 0.12 line open so +# Cargo selects the current fork release and WorkTablesIndex can share it. +parking_lot = { package = "parking_lot_lite_hack", version = "^0.12", default-features = false } performance_measurement = { path = "performance_measurement", version = "^0.1", optional = true } performance_measurement_codegen = { path = "performance_measurement/codegen", version = "^0.1", optional = true } -prettytable-rs = "0.10" -psc-nanoid = { version = "3", features = ["rkyv", "packed"] } -rkyv = { version = "0.8", features = ["uuid-1"] } -reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] } -ps-reclaim = { version = "^0.1, >=0.1.3" } -rustc-hash = "2" +psc-nanoid = { version = "^3.2", default-features = false, features = ["rkyv", "packed"] } +rkyv = { version = "0.8", default-features = false, features = ["alloc", "bytecheck", "uuid-1"] } +# A blocking HTTP client, not an async one, and that is the point. +# +# `rusty-s3` only signs: `action.sign(..)` returns a presigned URL and the four +# calls here are a PUT and three GETs against those URLs. No streaming, no +# multipart, no auth headers. `reqwest` was doing that through hyper, h2, tower +# and tokio, which cost 91 crates and — worse — dragged tokio into a build that +# otherwise has none, so an S3 write from the nagoya persistence worker panicked +# with "there is no reactor running". +# +# The persistence worker is already on its own thread and does one request at a +# time. Async HTTP exists to multiplex many connections onto few threads, which +# is not this. A blocking call is the correct shape here rather than a +# concession, and it removes the reactor requirement instead of satisfying it. +ureq = { version = "2", optional = true, default-features = false, features = ["tls"] } +# `spin` rather than `std`: without it this crate reaches for `thread_local!`, +# which needs a platform. See its `slot` module for what it falls back to. +ps-reclaim = { version = "^0.1", default-features = false, features = ["spin"] } +rustc-hash = { version = "2", default-features = false } rusty-s3 = { version = "0.10", optional = true } smart-default = "0.7" -tokio = { version = "1", features = ["full"] } -tracing = "0.1" +# Optional, off by default, and reachable only through the `tokio-runtime` +# feature. See that feature for why this is not a plain dependency. The feature +# list is what `TokioRt` calls and no more: `full` would put an I/O driver and +# a process reaper in the graph for a backend that only schedules. +tokio = { version = "1", default-features = false, features = [ + "rt-multi-thread", + "sync", + "time", +], optional = true } +tracing = { version = "0.1", default-features = false } url = { version = "2", optional = true } -uuid = { version = "1", features = ["v4", "v7"] } +uuid = { version = "^1", default-features = false, features = ["v4", "v7"] } walkdir = { version = "2", optional = true } # These pre-release workspace crates move as one train. The explicit caret # keeps the dependency policy consistent while the local path selects this # checkout during validation. -worktable_codegen = { path = "codegen", version = "^1.0.0-beta.19" } +worktable_codegen = { path = "codegen", version = "^1.9.0-alpha1" } # Re-exported below. Each generated table carries its declaration as a const # whose documentation says to read it with `worktable_dsl::Schema::parse`; that # instruction is only true if a plain `worktable` dependency can reach the # crate. -worktable_dsl = { path = "dsl", version = "^1.0.0-beta.18.1" } +worktable_dsl = { path = "dsl", version = "^1.0.0-beta.19", optional = true } + +[target.'cfg(unix)'.dependencies] +libc = { version = "^0.2", default-features = false } + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "^0.61", default-features = false, features = ["Win32_Foundation", "Win32_System_SystemInformation"] } [dev-dependencies] +fastrand = "2" chrono = "0.4" +# For one test, and only one: the flavor registry lives here and the DSL +# carries a mirror of it so the parser does not have to depend on the runtime +# crate. A dev-dependency is what lets a test compare the two lists without +# putting the mirror's `syn` and `proc-macro2` into anybody's runtime graph. +worktable_dsl = { path = "dsl" } criterion = { version = "0.5", features = ["async_tokio"] } rand = "0.9" +# A dev-dependency, and that is the whole of the no_std claim. It was a normal +# dependency with every feature on, so `cargo tree --no-default-features -e +# normal -i tokio` found it linked into a build that had asked for no std at +# all. `cargo check --no-default-features` passed the entire time, because it +# only ever proved this crate's own source was std-free and never its closure. +# What kept it there was not the engine: it was six `tokio::` paths emitted by +# `worktable!` into consumer crates, which made the runtime part of the macro's +# contract. Those go through `worktable::prelude` now. +tokio = { version = "1", features = ["full"] } tracing-subscriber = "0.3" +# Drives `tests/ui.rs`. Half of what `worktable!` promises is a refusal, and a +# refusal is only testable by compiling something that must not compile. The +# expected diagnostic is a committed `.stderr` beside each case, so the test +# asserts on the message and not merely on failure. +trybuild = "1" # Only under `--cfg loom`, so a normal build and a normal `cargo test` never # resolve it. See src/partition/loom_tests.rs for how to run the models. diff --git a/README.md b/README.md index e13a8cde..f7d15787 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ # WorkTable +Read the [user guide](docs/wt-user-guide.typ) for features and Rust callsites, or +[Why WorkTables](docs/why-worktables.typ) for the design and measured examples. +Typst is the maintained source. Run `sh scripts/build-guides.sh` to build both PDFs. + +Generated mutable paged tables expose `table.vacuum_with_pacing(VacuumPacing { +batch_pages: 64, ..Default::default() })` for a caller-selected vacuum policy. +`table.vacuum()` retains the default policy. This is a Rust API, with no new DSL +syntax. Zero batch pages disables automatic pacing; positive values wait for +quiet foreground periods and release exclusion between source-page batches. + *Absolutely not a database.* Embedded table storage for Rust. Declare a table with the `worktable!` macro and get a @@ -16,6 +26,27 @@ from a macro, and that persisting it is one feature flag away. cargo add worktable@1.0.0-beta.5 ``` +## New in 1.9 + +- **`no_std`.** `default-features = false` builds the library and generated calls without + Rust std. Allocation and OS services remain available. Hosted persistence, + background vacuum and runtime thread creation require `std`. +- **Columnar fields and indexes.** `columnar` on a column, `columnar_indexes` with + `cluster_by`, so a scan over one field reads only that field's bytes. +- **Explicit owned runtime execution.** `runtime: nagoya()` or `runtime: tokio` selects the default for `execute_async().await`. Named profiles schedule owned selects and annotated mutations on `Arc`; ordinary borrowed operations keep their callsite execution. +- **`page_size` on a persisted table**, at any size above a 512-byte floor. +- **The default index backend is `arctic`**, not `worktables_index`. Arctic cannot + key an optional or variable-width column, so an index over `String optional` + must now say `using worktables_index`. Only `congee` still requires `persist` + to be stated explicitly. +- **`using fxhash`, a hash index**, on `vec: true` tables only. Worth 4.9x on + build and 4.0x on lookup at a million rows against the default. It is refused + on a paged table, because a paged table generates a range select per index and + writes each persisted index to disk as sorted pages, and a hash map can do + neither. A table using it has no `range` or `range_by_` methods at all — they + are not generated, so asking for one is a compile error rather than a method + that cannot answer. + ## What you get | | | @@ -28,7 +59,7 @@ cargo add worktable@1.0.0-beta.5 | **Generated queries** | `select`, `insert`, `insert_many`, `upsert`, `update`, `delete` and a `select_all` query builder on every table, plus the custom update/delete queries you declare. | | **Paged in-memory storage** | Records live in `DataPages` with a free list for reuse. `rkyv` gives zero-copy access to archived rows. | | **Concurrency** | Lock-free concurrent indexes with change-data-capture, plus a row-level `LockMap` for ordered access. | -| **Optional persistence** | `PersistedWorkTable` writes to local disk; the `s3-support` feature syncs that to S3. Both opt-in, so a purely in-memory table pays for neither. | +| **Optional persistence** | `PersistedWorkTable` writes to local disk; the `s3-support` feature adds database-wide S3 generations and a queryable generated system catalog. Both are opt-in, so a purely in-memory table pays for neither. | | **Schema migration** | `worktable_version!` and `migration_engine!` version a table's schema and generate migrations between versions. See [docs/migration.md](docs/migration.md). | | **Memory accounting** | `MemStat` estimates live heap; resident benchmarks measure allocator and SMR overhead. | @@ -50,12 +81,39 @@ exported from the crate root; the prelude carries `DiskPersistenceEngine`, `ReadOnlyPersistenceEngine`, the space and table-of-contents types, and the operation-log types (`InsertOperation`, `UpdateOperation`, `DeleteOperation`, `AcknowledgeOperation`). -S3 support layers *on top of* the disk engine rather than replacing it. -`S3SyncDiskPersistenceEngine` wraps a `DiskPersistenceEngine` and syncs it. +The recommended S3 path is database-wide. Create one `S3Database`, clone that handle +into each persisted table's `DatabaseS3DiskConfig`, and generate the table-specific +engine alias with `database_s3_persistence!(TableName)`. DataBucket stages immutable +content-addressed page segments and WorkTable supplies the domain's generated system +catalog. A conditional 160-byte head publishes the new generation only after its pages +and catalog checkpoint are durable. One isolated page mutation measured 33,016 bytes; +the same mutation with a catalog larger than one page measured 49,544 bytes. The 4 MiB +segment size is a coalescing ceiling, not a write minimum. + +The older `s3_sync_persistence!` callsite remains available for existing per-table +manifests. New databases should use the shared domain so tables commit against one +catalog and restore through catalog page mappings. + +```rust +use worktable::{database_s3_persistence, DatabaseS3DiskConfig, S3Database}; + +database_s3_persistence!(OrderWorkTable); + +let database = S3Database::open_s3(domain_id, writer_epoch, s3_config)?; +let engine = OrderDatabaseS3PersistenceEngine::new(DatabaseS3DiskConfig { + disk: DiskConfig::new_with_table_name(dir, "orders", OrderWorkTable::version()), + database: database.clone(), +}).await?; +let orders = OrderWorkTable::load(engine).await?; + +for table in database.catalog().system_tables() { + println!("{}: {} rows", table.name.as_str(), table.row_count); +} +``` ```toml [dependencies] -worktable = { version = "=1.0.0-beta.5", features = ["s3-support"] } # S3 sync, optional +worktable = { version = "^1.9.0-alpha1", features = ["s3-support"] } # S3 sync, optional ``` Persisted indexes default to WorkTablesIndex. Vanilla IndexSet can be selected explicitly with `using indexset` while retaining the existing disk/S3 representation. Congee and Arctic persistence is experimental and uses their native checkpoint/WAL adapters; declarations using either backend must state `persist: true` or `persist: false` explicitly. The full syntax and capability matrix are documented in [Per-index backends with `using`](docs/index-backend-dsl-proposal.md). @@ -115,7 +173,7 @@ consistent with moved rows, but it does not truncate `.wt.data`. Use observe physical growth and decide when to snapshot/rebuild or run future offline compaction. -WorkTablesIndex uses its predictable branch-based node search by default in WorkTable. This avoids a measured regression for sequential numeric-key workloads. Alternative search policies remain compile-time feature gates: disable WorkTable's default features and enable one of `wti-hybrid-search`, `wti-std-search`, or `wti-superslice-search` (plus any other features such as `s3-support`). Prefer one search feature for an unambiguous build. If Cargo feature unification enables several, WorkTablesIndex applies the documented deterministic precedence rather than rejecting the graph. +WorkTablesIndex uses the standard slice binary search by default in WorkTable. At the default node width, the isolated 200,000-key matrix measured randomized lookup at 45.4 ns with this policy and 101.9 ns with the predictable policy. Predictable search remains useful for ordered writes: the alternating table A/B measured about 12-14% less persisted insert-and-drain time, while standard search was faster for four-client in-memory insertion. Alternative search policies remain compile-time feature gates: disable WorkTable's default features and enable one of `wti-predictable-search`, `wti-hybrid-search`, or `wti-superslice-search` (plus any other features such as `s3-support`). Prefer one search feature for an unambiguous build. If Cargo feature unification enables several, WorkTablesIndex applies the documented deterministic precedence rather than rejecting the graph. ## Concurrent read/write publication @@ -158,7 +216,7 @@ provides the page and link primitives its data layout uses: `PageId`, `Link`, backend, and those types appear throughout the in-memory paging, the indexes, the memory accounting and the on-disk format alike. -WorkTable re-exports it (`pub use data_bucket;`) and pins an exact version. **Take it +WorkTable re-exports it (`pub use data_bucket;`) and uses a compatible caret requirement. **Take it through that re-export rather than depending on it separately.** A second copy in your graph gives you two incompatible sets of the same types, and the resulting error names two different `data_bucket` paths while looking like something else entirely. diff --git a/benches/cases/full_featured.rs b/benches/cases/full_featured.rs index 0e5b222d..69c95bfa 100644 --- a/benches/cases/full_featured.rs +++ b/benches/cases/full_featured.rs @@ -19,7 +19,7 @@ fn insert(c: &mut Criterion) { another: format!("another_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(black_box(row))) + nagoya::block_on(table.insert(black_box(row))) }, BatchSize::SmallInput, ) @@ -37,7 +37,7 @@ fn select_by_pk(c: &mut Criterion) { another: format!("another_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -60,7 +60,7 @@ fn select_by_unique_index(c: &mut Criterion) { another: format!("another_{}", i), something: i, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("full_featured_select_by_val1", |b| { @@ -82,7 +82,7 @@ fn select_by_non_unique_index(c: &mut Criterion) { another: format!("cat_{}", i % 10), something: i, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("full_featured_select_by_another", |b| { @@ -196,7 +196,7 @@ fn in_place_update(c: &mut Criterion) { another: "test".to_string(), something: 0, }; - futures::executor::block_on(table.insert(row)).unwrap().into() + nagoya::block_on(table.insert(row)).unwrap().into() }; c.bench_function("full_featured_in_place_update_val", |b| { @@ -219,7 +219,7 @@ fn delete(c: &mut Criterion) { another: format!("temp_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: FullFeaturedPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -242,7 +242,7 @@ fn delete_by_index_query(c: &mut Criterion) { another: another.clone(), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); another }, |another: String| rt.block_on(async { table.delete_by_another(another).await.unwrap() }), @@ -319,7 +319,7 @@ fn batch_insert(c: &mut Criterion) { another: format!("another_{}", i), something: i as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -347,7 +347,7 @@ fn batch_select_pk(c: &mut Criterion) { another: format!("another_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/benches/cases/non_unique_index.rs b/benches/cases/non_unique_index.rs index 7e5f2038..c2589030 100644 --- a/benches/cases/non_unique_index.rs +++ b/benches/cases/non_unique_index.rs @@ -33,7 +33,7 @@ fn select_by_pk(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -54,7 +54,7 @@ fn select_by_non_unique_index(c: &mut Criterion) { value: fastrand::u64(..), category: i % 10, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("non_unique_index_select_by_category", |b| { @@ -108,7 +108,7 @@ fn delete(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: NonUniqueIndexPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -176,7 +176,7 @@ fn batch_insert(c: &mut Criterion) { value: i as u64, category: (i % 10) as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -202,7 +202,7 @@ fn batch_select_pk(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/benches/cases/nonunique_arctic_vs_wti.rs b/benches/cases/nonunique_arctic_vs_wti.rs index 6ea1d8a7..9d4e19ed 100644 --- a/benches/cases/nonunique_arctic_vs_wti.rs +++ b/benches/cases/nonunique_arctic_vs_wti.rs @@ -84,7 +84,7 @@ fn select_by_key(c: &mut Criterion) { for (fan_out, keys) in SHAPES { group.throughput(Throughput::Elements(fan_out)); - let table = futures::executor::block_on(populated_wti(fan_out, keys)); + let table = nagoya::block_on(populated_wti(fan_out, keys)); group.bench_with_input(BenchmarkId::new("wti_string", fan_out), &fan_out, |b, _| { b.iter(|| { let key = string_key(fastrand::u64(0..keys)); @@ -92,7 +92,7 @@ fn select_by_key(c: &mut Criterion) { }) }); - let table = futures::executor::block_on(populated_arctic(fan_out, keys)); + let table = nagoya::block_on(populated_arctic(fan_out, keys)); group.bench_with_input(BenchmarkId::new("arctic_u128", fan_out), &fan_out, |b, _| { b.iter(|| { let key = hash_of(fastrand::u64(0..keys)); @@ -108,7 +108,7 @@ fn insert(c: &mut Criterion) { for (fan_out, keys) in SHAPES { // Steady state: the table already holds `fan_out` rows per key and // each measured insert lands on an existing key. - let table = futures::executor::block_on(populated_wti(fan_out, keys)); + let table = nagoya::block_on(populated_wti(fan_out, keys)); group.bench_with_input(BenchmarkId::new("wti_string", fan_out), &fan_out, |b, _| { b.iter_batched( || WtiStringAdjacencyRow { @@ -116,12 +116,12 @@ fn insert(c: &mut Criterion) { source: string_key(fastrand::u64(0..keys)), payload: u64::MAX, }, - |row| futures::executor::block_on(table.insert(black_box(row))).unwrap(), + |row| nagoya::block_on(table.insert(black_box(row))).unwrap(), BatchSize::SmallInput, ) }); - let table = futures::executor::block_on(populated_arctic(fan_out, keys)); + let table = nagoya::block_on(populated_arctic(fan_out, keys)); group.bench_with_input(BenchmarkId::new("arctic_u128", fan_out), &fan_out, |b, _| { b.iter_batched( || ArcticHashAdjacencyRow { @@ -129,7 +129,7 @@ fn insert(c: &mut Criterion) { source: hash_of(fastrand::u64(0..keys)), payload: u64::MAX, }, - |row| futures::executor::block_on(table.insert(black_box(row))).unwrap(), + |row| nagoya::block_on(table.insert(black_box(row))).unwrap(), BatchSize::SmallInput, ) }); diff --git a/benches/cases/partition_routing.rs b/benches/cases/partition_routing.rs index 7d59b03a..cee7a451 100644 --- a/benches/cases/partition_routing.rs +++ b/benches/cases/partition_routing.rs @@ -19,6 +19,7 @@ use worktable::worktable; worktable!( name: Route, partition_by: symbol_id: u16, + partition_max_size: u64, columns: { exchange_id: u8 primary_key, bid: f64, @@ -46,7 +47,7 @@ async fn populated() -> RoutePartitions { /// The four ways to reach a partition, single threaded, one hot key. fn lookup(c: &mut Criterion) { - let routes = futures::executor::block_on(populated()); + let routes = nagoya::block_on(populated()); let cached = routes.partition(7).unwrap(); let mut group = c.benchmark_group("partition_lookup"); @@ -87,7 +88,7 @@ fn contended(c: &mut Criterion, name: &str, same_key: bool) { for api in ["partition_ref", "pinned_get", "partition_arc"] { group.bench_with_input(BenchmarkId::new(api, threads), &threads, |b, &threads| { b.iter_custom(|iters| { - let routes = Arc::new(futures::executor::block_on(populated())); + let routes = Arc::new(nagoya::block_on(populated())); let go = Arc::new(AtomicBool::new(false)); let workers: Vec<_> = (0..threads) @@ -151,7 +152,7 @@ fn distinct_key_readers(c: &mut Criterion) { /// Accounting over 500 partitions. These were routed through `system_info`, /// which copied every data page, and through `keys` then `partition` per key. fn metrics(c: &mut Criterion) { - let routes = futures::executor::block_on(populated()); + let routes = nagoya::block_on(populated()); let mut group = c.benchmark_group("partition_metrics"); group.bench_function("memory_total", |b| b.iter(|| black_box(routes.memory_total()))); group.bench_function("rows_by_key", |b| b.iter(|| black_box(routes.rows_by_key()))); diff --git a/benches/cases/simple.rs b/benches/cases/simple.rs index a035b53a..afbcc353 100644 --- a/benches/cases/simple.rs +++ b/benches/cases/simple.rs @@ -15,7 +15,7 @@ fn insert(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(black_box(row))) + nagoya::block_on(table.insert(black_box(row))) }, BatchSize::SmallInput, ) @@ -30,7 +30,7 @@ fn select_by_pk(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -82,7 +82,7 @@ fn delete(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: SimplePrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -146,7 +146,7 @@ fn batch_insert(c: &mut Criterion) { id: table.get_next_pk().into(), value: i as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -171,7 +171,7 @@ fn batch_select_pk(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/benches/cases/unique_index.rs b/benches/cases/unique_index.rs index 7509ae63..73144ff2 100644 --- a/benches/cases/unique_index.rs +++ b/benches/cases/unique_index.rs @@ -52,7 +52,7 @@ fn select_by_pk(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -73,7 +73,7 @@ fn select_by_unique_index(c: &mut Criterion) { test: i, another: i as u64, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("unique_index_select_by_test", |b| { @@ -93,7 +93,7 @@ fn select_by_unique_index_range(c: &mut Criterion) { test: i, another: i as u64, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("unique_index_select_by_test_range", |b| { @@ -109,8 +109,8 @@ fn art_primary_key_ranges(c: &mut Criterion) { let congee = CongeeRangeBenchmarkWorkTable::default(); let arctic = ArcticRangeBenchmarkWorkTable::default(); for id in 0..ROWS { - futures::executor::block_on(congee.insert(CongeeRangeBenchmarkRow { id, value: id })).unwrap(); - futures::executor::block_on(arctic.insert(ArcticRangeBenchmarkRow { id, value: id })).unwrap(); + nagoya::block_on(congee.insert(CongeeRangeBenchmarkRow { id, value: id })).unwrap(); + nagoya::block_on(arctic.insert(ArcticRangeBenchmarkRow { id, value: id })).unwrap(); } let mut group = c.benchmark_group("art_primary_key_single_row_range"); @@ -173,7 +173,7 @@ fn delete(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: UniqueIndexPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -241,7 +241,7 @@ fn batch_insert(c: &mut Criterion) { test: i as i64, another: i as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -267,7 +267,7 @@ fn batch_select_pk(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index cab40835..2db7b380 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "worktable_codegen" -version = "1.0.0-beta.19" +version = "1.9.0-alpha1" edition = "2024" license = "MIT" description = "Proc-macro companion crate for worktable: the worktable! macro and its derives." @@ -9,6 +9,10 @@ repository = "https://github.com/pathscale/WorkTable" [features] s3-support = [] logical-index-persistence = [] +# Mirrors the consuming crate's `std`. The macro emits a runtime type name +# that only exists in a std build, so the emitter has to know which build it is +# expanding into. Forwarded from `worktable` so the two cannot disagree. +std = [] # Compatibility no-op retained for downstream manifests. versioned-row-publication = [] @@ -20,13 +24,20 @@ proc-macro = true [dependencies] # The schema language, extracted so consumers other than this macro can read # a declaration. See its crate docs for why that needed a separate crate. -# Name the pre-release floor explicitly while retaining the workspace's caret -# policy. beta.18.1 is the first DSL artifact that exports every validator this -# code generator calls; beta.18 was published before that API landed. -worktable_dsl = { path = "../dsl", version = "^1.0.0-beta.18.1" } -rkyv = { version = "0.8" } +# Name the reviewed prerelease while accepting compatible schema-model and +# validator updates. Release checks verify the resolved generated API. +worktable_dsl = { path = "../dsl", version = "^1.0.0-beta.19" } +# Test-only. As a normal dependency this proc-macro crate put `rkyv` with its +# default features into the graph, which turned on `rkyv/std` for the target +# build too and dragged `ptr_meta` with it. The generated code names `rkyv` +# paths through `quote!`, which needs no dependency here at all. +# +# See the `[dev-dependencies]` section below. syn = { version = "2", features = ["full"] } quote = "1" proc-macro2 = "1" convert_case = "0.6" indexmap = "2" + +[dev-dependencies] +rkyv = { version = "0.8" } diff --git a/codegen/src/common/name_generator.rs b/codegen/src/common/name_generator.rs index 7cdaa9fb..b1865a12 100644 --- a/codegen/src/common/name_generator.rs +++ b/codegen/src/common/name_generator.rs @@ -96,6 +96,16 @@ impl WorktableNameGenerator { Ident::new(format!("{}Index", self.name).as_str(), Span::mixed_site()) } + /// The alias the generated code names its runtime through. + /// + /// One name per table rather than the concrete type repeated at every site + /// that needs it: the sync primitives, the timers and the vacuum spawn all + /// have to agree, and a table that resolved its runtime twice could get two + /// answers. + pub fn get_runtime_type_ident(&self) -> Ident { + Ident::new(format!("{}Runtime", self.name).as_str(), Span::mixed_site()) + } + pub fn get_page_size_const_ident(&self) -> Ident { let upper_snake_case_name = self.name.from_case(Case::Pascal).to_case(Case::UpperSnake); Ident::new( @@ -104,6 +114,12 @@ impl WorktableNameGenerator { ) } + /// Payload budget for index and metadata pages, independent of row slots. + pub fn get_disk_page_capacity(&self) -> proc_macro2::TokenStream { + let page_size = self.get_page_size_const_ident(); + quote::quote! { (#page_size - worktable::prelude::GENERAL_HEADER_SIZE) } + } + pub fn get_page_inner_size_const_ident(&self) -> Ident { let upper_snake_case_name = self.name.from_case(Case::Pascal).to_case(Case::UpperSnake); Ident::new( diff --git a/codegen/src/database_s3_persistence/mod.rs b/codegen/src/database_s3_persistence/mod.rs new file mode 100644 index 00000000..f91f87b9 --- /dev/null +++ b/codegen/src/database_s3_persistence/mod.rs @@ -0,0 +1,49 @@ +use proc_macro2::TokenStream; +use quote::quote; +use syn::parse::{Parse, ParseStream}; +use syn::{Ident, Result}; + +use crate::common::name_generator::WorktableNameGenerator; + +struct Input { + table_name: Ident, +} + +impl Parse for Input { + fn parse(input: ParseStream) -> Result { + Ok(Self { + table_name: input.parse()?, + }) + } +} + +pub fn expand(input: TokenStream) -> Result { + let input: Input = syn::parse2(input)?; + let name = input.table_name.to_string(); + let base = name.strip_suffix("WorkTable").unwrap_or(&name).to_string(); + let output_name = format!("{base}DatabaseS3PersistenceEngine"); + let names = WorktableNameGenerator::from_table_name(base); + let output = Ident::new(&output_name, input.table_name.span()); + let primary_key = names.get_primary_key_type_ident(); + let space_primary_index = names.get_space_primary_index_ident(); + let space_secondary_index = names.get_space_secondary_index_ident(); + let secondary_events = names.get_space_secondary_index_events_ident(); + let available_indexes = names.get_available_indexes_ident(); + let inner_size = names.get_page_inner_size_const_ident(); + let page_size = names.get_page_size_const_ident(); + + Ok(quote! { + pub type #output = worktable::prelude::DatabaseS3PersistenceEngine< + worktable::prelude::SpaceData< + <<#primary_key as worktable::prelude::TablePrimaryKey>::Generator as worktable::prelude::PrimaryKeyGeneratorState>::State, + { #inner_size }, + { #page_size as u32 }, + >, + #space_primary_index, + #space_secondary_index, + #primary_key, + #secondary_events, + #available_indexes, + >; + }) +} diff --git a/codegen/src/generators/columnar.rs b/codegen/src/generators/columnar.rs new file mode 100644 index 00000000..49577a25 --- /dev/null +++ b/codegen/src/generators/columnar.rs @@ -0,0 +1,524 @@ +use convert_case::{Case, Casing}; +use proc_macro2::{Ident, Literal, Span, TokenStream}; +use quote::{format_ident, quote}; + +use crate::common::model::{ColumnCompression, Columns}; +use crate::common::name_generator::{WorktableNameGenerator, is_float}; + +fn data_ident(table: &Ident) -> Ident { + format_ident!("{}ColumnarData", table) +} + +fn column_field(field: &Ident) -> Ident { + format_ident!("column_{}", field) +} + +fn index_field(index: &Ident) -> Ident { + format_ident!("columnar_index_{}", index) +} + +fn slot_id_type(columns: &Columns) -> Ident { + Ident::new(columns.column_slot_id.type_name(), Span::mixed_site()) +} + +fn compression_variant(compression: ColumnCompression) -> Ident { + Ident::new( + &compression.name().from_case(Case::Snake).to_case(Case::Pascal), + Span::mixed_site(), + ) +} + +fn key_type(columns: &Columns, fields: &[Ident]) -> TokenStream { + let fields = fields.iter().map(|field| { + let ty = columns.columns_map.get(field).expect("validated columnar index field"); + if is_float(&ty.to_string()) { + quote! { OrderedFloat<#ty> } + } else { + quote! { #ty } + } + }); + quote! { (#(#fields,)*) } +} + +fn row_key(columns: &Columns, fields: &[Ident], row: TokenStream) -> TokenStream { + let fields = fields.iter().map(|field| { + let ty = columns.columns_map.get(field).expect("validated columnar index field"); + if is_float(&ty.to_string()) { + quote! { OrderedFloat(#row.#field) } + } else { + quote! { #row.#field.clone() } + } + }); + quote! { (#(#fields,)*) } +} + +pub(crate) fn index_struct_field(table: &Ident, columns: &Columns, persisted: bool) -> TokenStream { + if columns.columnar_fields.is_empty() { + return quote! {}; + } + let data = data_ident(table); + let skip = persisted.then(|| quote! { #[index(skip)] }); + quote! { + #skip + columnar: ParkingRwLock<#data>, + #skip + columnar_publication: ParkingRwLock<()> + } +} + +pub(crate) fn index_default_field(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { + columnar: ParkingRwLock::new(Default::default()), + columnar_publication: ParkingRwLock::new(()), + } + } +} + +pub(crate) fn save_row(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { + if let Err(bits) = self.columnar.write().save_row(&row) { + return Err(IndexError::ColumnSlotIdExhausted { + bits, + inserted_already: inserted_indexes.clone(), + }); + } + } + } +} + +pub(crate) fn publication_guard(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { + fn row_publication(&self) -> Option> { + Some(self.columnar_publication.read()) + } + } + } +} + +pub(crate) fn table_publication_guard(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { let _publication = self.0.indexes.columnar_publication.read(); } + } +} + +pub(crate) fn save_row_cdc(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { + if let Err(bits) = self.columnar.write().save_row(&row) { + return (partial_events, Err(IndexError::ColumnSlotIdExhausted { + bits, + inserted_already: inserted_indexes.clone(), + })); + } + } + } +} + +pub(crate) fn reinsert_row(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { + if let Err(bits) = self.columnar.write().replace_row(&row_old, &row_new) { + return Err(IndexError::ColumnSlotIdExhausted { + bits, + inserted_already: inserted_indexes.clone(), + }); + } + } + } +} + +pub(crate) fn reinsert_row_cdc(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { + if let Err(bits) = self.columnar.write().replace_row(&row_old, &row_new) { + return (partial_events, Err(IndexError::ColumnSlotIdExhausted { + bits, + inserted_already: inserted_indexes.clone(), + })); + } + } + } +} + +pub(crate) fn delete_row(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { self.columnar.write().delete_row(&row); } + } +} + +pub(crate) fn mark_dirty(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { self.columnar.write().mark_dirty(); } + } +} + +pub(crate) fn table_mark_dirty(columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + quote! {} + } else { + quote! { self.0.indexes.columnar.write().mark_dirty(); } + } +} + +pub(crate) fn definitions(table: &Ident, columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + return quote! {}; + } + + let names = WorktableNameGenerator::from_table_name(table.to_string()); + let row = names.get_row_type_ident(); + let pk = names.get_primary_key_type_ident(); + let data = data_ident(table); + let slot_id = slot_id_type(columns); + + let column_fields = columns.columnar_fields.iter().map(|(field, _)| { + let storage = column_field(field); + let ty = columns.columns_map.get(field).expect("columnar field exists"); + quote! { #storage: ColumnarColumn<#ty>, } + }); + let column_defaults = columns.columnar_fields.iter().map(|(field, config)| { + let storage = column_field(field); + let chunk_rows = Literal::usize_unsuffixed(config.chunk_rows.expect("columnar defaults applied")); + let compression = compression_variant(config.compression); + quote! { #storage: ColumnarColumn::new(#chunk_rows, ColumnCompression::#compression), } + }); + let index_fields = columns.columnar_indexes.values().map(|index| { + let field = index_field(&index.name); + let ty = key_type(columns, &index.cluster_by); + quote! { #field: ClusteredColumnarIndex<#ty, #slot_id>, } + }); + let index_defaults = columns.columnar_indexes.values().map(|index| { + let field = index_field(&index.name); + quote! { #field: Default::default(), } + }); + + let set_columns = columns.columnar_fields.keys().map(|field| { + let storage = column_field(field); + quote! { self.#storage.set(slot_id, row.#field.clone()); } + }); + let remove_columns = columns.columnar_fields.keys().map(|field| { + let storage = column_field(field); + quote! { self.#storage.remove(slot_id); } + }); + let insert_indexes = columns.columnar_indexes.values().map(|index| { + let field = index_field(&index.name); + let key = row_key(columns, &index.cluster_by, quote! { row }); + quote! { self.#field.insert(#key, slot_id); } + }); + let delete_indexes = columns.columnar_indexes.values().map(|index| { + let field = index_field(&index.name); + let key = row_key(columns, &index.cluster_by, quote! { row }); + quote! { self.#field.remove(&#key, slot_id); } + }); + let replace_remove_indexes = columns.columnar_indexes.values().map(|index| { + let field = index_field(&index.name); + let key = row_key(columns, &index.cluster_by, quote! { row_old }); + quote! { self.#field.remove(&#key, slot_id); } + }); + let replace_insert_indexes = columns.columnar_indexes.values().map(|index| { + let field = index_field(&index.name); + let key = row_key(columns, &index.cluster_by, quote! { row_new }); + quote! { self.#field.insert(#key, slot_id); } + }); + let replace_columns = columns.columnar_fields.keys().map(|field| { + let storage = column_field(field); + quote! { self.#storage.set(slot_id, row_new.#field.clone()); } + }); + + quote! { + #[derive(Debug, MemStat)] + struct #data { + next_slot_position: Option, + free_slot_ids: worktable::prelude::BTreeSet<#slot_id>, + slot_generations: Vec, + incarnation: u64, + slots_high_water: usize, + dirty: bool, + slots: worktable::prelude::BTreeMap<#pk, (#slot_id, u64)>, + primary_keys: ColumnarColumn<#pk>, + #(#column_fields)* + #(#index_fields)* + } + + impl Default for #data { + fn default() -> Self { + Self { + next_slot_position: Some(0), + free_slot_ids: Default::default(), + slot_generations: Default::default(), + incarnation: next_columnar_incarnation(), + slots_high_water: 0, + dirty: true, + slots: Default::default(), + primary_keys: ColumnarColumn::new(65_536, ColumnCompression::None), + #(#column_defaults)* + #(#index_defaults)* + } + } + } + + impl #data { + fn allocate_slot(&mut self) -> Result<(#slot_id, u64), u8> { + if let Some(slot_id) = self.free_slot_ids.pop_first() { + let generation = self.slot_generations[slot_id.slot()]; + return Ok((slot_id, generation)); + } + let position = self + .next_slot_position + .ok_or(<#slot_id as ColumnSlotId>::BITS)?; + let slot_id = <#slot_id as ColumnSlotId>::try_from_position(position) + .ok_or(<#slot_id as ColumnSlotId>::BITS)?; + self.next_slot_position = position.checked_add(1); + let slot = slot_id.slot(); + if self.slot_generations.len() <= slot { + self.slot_generations.resize(slot + 1, 0); + } + Ok((slot_id, self.slot_generations[slot])) + } + + fn save_row(&mut self, row: &#row) -> Result<(), u8> { + let primary_key = row.get_primary_key(); + let (slot_id, _) = if let Some(slot) = self.slots.get(&primary_key).copied() { + slot + } else { + let slot = self.allocate_slot()?; + self.slots.insert(primary_key.clone(), slot); + self.primary_keys.set(slot.0, primary_key); + self.slots_high_water = self.slots_high_water.max(self.slots.len()); + slot + }; + #(#set_columns)* + #(#insert_indexes)* + Ok(()) + } + + fn delete_row(&mut self, row: &#row) { + let primary_key = row.get_primary_key(); + let Some((slot_id, generation)) = self.slots.remove(&primary_key) else { + return; + }; + #(#delete_indexes)* + #(#remove_columns)* + self.primary_keys.remove(slot_id); + if let Some(next_generation) = generation.checked_add(1) { + self.slot_generations[slot_id.slot()] = next_generation; + self.free_slot_ids.insert(slot_id); + } + } + + fn replace_row(&mut self, row_old: &#row, row_new: &#row) -> Result<(), u8> { + let old_primary_key = row_old.get_primary_key(); + let new_primary_key = row_new.get_primary_key(); + if old_primary_key != new_primary_key { + self.delete_row(row_old); + return self.save_row(row_new); + } + let Some((slot_id, _)) = self.slots.get(&old_primary_key).copied() else { + return self.save_row(row_new); + }; + #(#replace_remove_indexes)* + #(#replace_columns)* + #(#replace_insert_indexes)* + Ok(()) + } + + fn row_ref(&self, slot_id: #slot_id) -> Option> { + let primary_key = self.primary_keys.get(slot_id)?.clone(); + let (current_slot, generation) = self.slots.get(&primary_key).copied()?; + (current_slot == slot_id).then(|| { + ColumnarRowRef::__new(primary_key, slot_id, generation, self.incarnation) + }) + } + + fn validates(&self, row_ref: &ColumnarRowRef<#pk, #slot_id>) -> bool { + row_ref.__incarnation() == self.incarnation + && self.slots.get(row_ref.primary_key()).copied() + == Some((row_ref.__slot_id(), row_ref.__generation())) + } + + fn mark_dirty(&mut self) { + self.dirty = true; + } + } + } +} + +pub(crate) fn table_methods(table: &Ident, columns: &Columns) -> TokenStream { + if columns.columnar_fields.is_empty() { + return quote! {}; + } + + let names = WorktableNameGenerator::from_table_name(table.to_string()); + let row = names.get_row_type_ident(); + let pk = names.get_primary_key_type_ident(); + let data = data_ident(table); + let slot_id = slot_id_type(columns); + let row_ref = quote! { ColumnarRowRef<#pk, #slot_id> }; + + let field_methods = columns.columnar_fields.iter().map(|(field, _)| { + let storage = column_field(field); + let scan = format_ident!("columnar_scan_{}", field); + let project = format_ident!("columnar_project_{}", field); + let ty = columns.columns_map.get(field).expect("columnar field exists"); + quote! { + pub fn #scan(&self) -> Result, WorkTableError> { + loop { + self.ensure_columnar_current()?; + let columnar = self.0.indexes.columnar.read(); + if !columnar.dirty { + return Ok(columnar.#storage.iter::<#slot_id>() + .filter_map(|(slot_id, value)| { + columnar.row_ref(slot_id).map(|row_ref| (row_ref, value.clone())) + }) + .collect()); + } + } + } + + pub fn #project(&self, rows: &[#row_ref]) -> Result, WorkTableError> { + loop { + self.ensure_columnar_current()?; + let columnar = self.0.indexes.columnar.read(); + if !columnar.dirty { + return Ok(rows.iter().filter_map(|row_ref| { + columnar.validates(row_ref).then(|| { + columnar.#storage.get(row_ref.__slot_id()) + .cloned() + .map(|value| (row_ref.clone(), value)) + }).flatten() + }).collect()); + } + } + } + } + }); + + let index_methods = columns.columnar_indexes.values().map(|index| { + let storage = index_field(&index.name); + let select = format_ident!("columnar_select_{}", index.name); + let scan = format_ident!("columnar_scan_{}", index.name); + let args = index.cluster_by.iter().map(|field| { + let ty = columns.columns_map.get(field).expect("columnar index field exists"); + quote! { #field: #ty } + }); + let key_fields = index.cluster_by.iter().map(|field| { + let ty = columns.columns_map.get(field).expect("columnar index field exists"); + if is_float(&ty.to_string()) { + quote! { OrderedFloat(#field) } + } else { + quote! { #field } + } + }); + quote! { + pub fn #select(&self, #(#args),*) -> Result, WorkTableError> { + let key = (#(#key_fields,)*); + loop { + self.ensure_columnar_current()?; + let columnar = self.0.indexes.columnar.read(); + if !columnar.dirty { + return Ok(columnar.#storage.exact(&key).into_iter() + .filter_map(|slot_id| columnar.row_ref(slot_id)) + .collect()); + } + } + } + + pub fn #scan(&self) -> Result, WorkTableError> { + loop { + self.ensure_columnar_current()?; + let columnar = self.0.indexes.columnar.read(); + if !columnar.dirty { + return Ok(columnar.#storage.ordered_slot_ids().into_iter() + .filter_map(|slot_id| columnar.row_ref(slot_id)) + .collect()); + } + } + } + } + }); + + quote! { + fn ensure_columnar_current(&self) -> Result<(), WorkTableError> { + if !self.0.indexes.columnar.read().dirty { + return Ok(()); + } + // Publication gate must precede the replica lock. Writers hold its + // read side through both secondary maintenance and the primary + // pointer/visibility update, so a rebuild cannot snapshot that gap. + let _publication = self.0.indexes.columnar_publication.write(); + let mut columnar = self.0.indexes.columnar.write(); + if !columnar.dirty { + return Ok(()); + } + let rows: Vec<#row> = { + let read_guard = self.0.data.read_guard(); + self.0.primary_index.pk_map.iter_values().filter_map(|(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }).collect() + }; + // Preserve primary-key/slot identity across rebuilding derived values. + let mut rebuilt: #data = Default::default(); + rebuilt.next_slot_position = columnar.next_slot_position; + rebuilt.free_slot_ids = core::mem::take(&mut columnar.free_slot_ids); + rebuilt.slot_generations = core::mem::take(&mut columnar.slot_generations); + rebuilt.incarnation = columnar.incarnation; + rebuilt.slots_high_water = columnar.slots_high_water; + rebuilt.slots = core::mem::take(&mut columnar.slots); + rebuilt.primary_keys = core::mem::replace( + &mut columnar.primary_keys, + ColumnarColumn::new(65_536, ColumnCompression::None), + ); + for row in &rows { + rebuilt.save_row(row).map_err(WorkTableError::ColumnSlotIdExhausted)?; + } + rebuilt.dirty = false; + *columnar = rebuilt; + Ok(()) + } + + pub fn columnar_slots_in_use(&self) -> usize { + self.0.indexes.columnar.read().slots.len() + } + + pub fn columnar_slots_high_water(&self) -> usize { + self.0.indexes.columnar.read().slots_high_water + } + + /// Returns whether a fallback mutation has invalidated the derived + /// columnar replica. The next columnar read rebuilds it automatically. + pub fn columnar_is_dirty(&self) -> bool { + self.0.indexes.columnar.read().dirty + } + + /// Rebuilds a dirty derived columnar replica at an application-chosen + /// point instead of charging the first later columnar reader. + pub fn rebuild_columnar(&self) -> Result<(), WorkTableError> { + self.ensure_columnar_current() + } + + #(#field_methods)* + #(#index_methods)* + } +} diff --git a/codegen/src/generators/dense_table.rs b/codegen/src/generators/dense_table.rs new file mode 100644 index 00000000..9b373b1a --- /dev/null +++ b/codegen/src/generators/dense_table.rs @@ -0,0 +1,455 @@ +//! The typed facade over [`worktable::partition::DenseRows`]. +//! +//! Emitted instead of the full table as a partition payload when +//! `partition_max_size` is narrow. The storage lives in the library, so what is +//! generated here is the part that needs the row type: the key projection, the +//! per-column updates, and the three methods the router calls. +//! +//! It is *only* a partition payload. A dense table addressed by position is a +//! `Vec` with extra steps unless something upstream guarantees the keys are +//! dense and small, and `partition_by` is that guarantee: the routing key does +//! the spreading, and the inner key only has to separate the handful of rows +//! inside one partition. + +use proc_macro2::{Ident, Literal, TokenStream}; +use quote::{format_ident, quote}; +use syn::Error; +use worktable_dsl::model::{Columns, Operation, PartitionMaxSize}; + +/// The `queries:` block, in the shape this generator needs it. +/// +/// Lifted out of the model before the table generators consume it, because the +/// dense payload is emitted after them and `Queries` is not `Clone`. +#[derive(Debug, Default)] +pub struct DenseQueries { + /// `update (columns) by `. + pub updates: Vec<(Ident, Operation)>, + /// `delete () by `. + pub deletes: Vec<(Ident, Operation)>, + /// `in_place (columns) by `. Refused: see [`expand`]. + pub in_place: Vec<(Ident, Operation)>, +} + +impl DenseQueries { + /// Read what a dense payload needs out of a parsed `queries:` block. + pub fn from_model(queries: Option<&worktable_dsl::model::Queries>) -> Self { + let Some(queries) = queries else { + return Self::default(); + }; + let lift = |map: &indexmap::IndexMap| { + map.iter().map(|(name, op)| (name.clone(), op.clone())).collect() + }; + Self { + updates: lift(&queries.updates), + deletes: lift(&queries.deletes), + in_place: lift(&queries.in_place), + } + } + + fn is_empty(&self) -> bool { + self.updates.is_empty() && self.deletes.is_empty() && self.in_place.is_empty() + } +} + +/// Primary key types a position-addressed table can take. +/// +/// Unsigned only, and for the same reason `partition_by`'s own key is: the key +/// *is* an index. A signed or floating key has no position to be, and a +/// `String` key would have to be hashed, which is the tree this shape exists to +/// delete. +const DENSE_KEY_TYPES: [&str; 5] = ["u8", "u16", "u32", "u64", "usize"]; + +/// The name of the generated payload type. +/// +/// `Dense` and not `Micro`: "micro-partition" is Snowflake's word for a 50 to +/// 500 MB automatic columnar unit, and using it for a 23-row partition would +/// mean something different to everyone who has met the term before. +pub fn type_ident(name: &Ident) -> Ident { + format_ident!("{}DenseTable", name) +} + +/// Check that this declaration can be addressed by position. +/// +/// Separate from [`expand`] because the router has to refuse before either +/// table is generated: a declaration that cannot be dense has to say so once, +/// naming the column, rather than failing inside an expansion. +pub fn validate(name: &Ident, columns: &Columns, max_size: PartitionMaxSize) -> syn::Result<(Ident, TokenStream)> { + let rows = max_size.rows().expect("only a dense width reaches here"); + + if columns.primary_keys.len() != 1 { + return Err(Error::new( + name.span(), + format!( + "`partition_max_size: {}` addresses rows by position, so the primary key has to be \ + one unsigned column. This table declares {} primary key columns. Use \ + `partition_max_size: u64` for a full table per partition, which takes a composite \ + key.", + max_size.type_name(), + columns.primary_keys.len() + ), + )); + } + + let pk = columns.primary_keys.first().expect("checked above").clone(); + let pk_type = columns + .columns_map + .get(&pk) + .expect("the primary key is a column") + .clone(); + let pk_text = pk_type.to_string().replace(' ', ""); + + if !DENSE_KEY_TYPES.contains(&pk_text.as_str()) { + return Err(Error::new( + pk.span(), + format!( + "`{pk}: {pk_text}` cannot address a row by position: `partition_max_size: {}` means \ + the key indexes the partition directly, so it must be one of {}. Either give the \ + partition an unsigned key, or use `partition_max_size: u64`, which keeps the full \ + table and its index and takes a key of any type.", + max_size.type_name(), + DENSE_KEY_TYPES.join(", ") + ), + )); + } + + // A key narrower than the cap cannot reach it, which is not an error but is + // always a mistake worth naming: `partition_max_size: u16` beside a `u8` + // key declares 65,536 rows and can hold 256. + let key_span = 1u64 << (8 * key_bytes(&pk_text).unwrap_or(8)); + if key_bytes(&pk_text).is_some() && key_span < rows { + return Err(Error::new( + pk.span(), + format!( + "`partition_max_size: {}` declares {rows} rows a partition, but `{pk}: {pk_text}` \ + only counts to {key_span}, so {} of those rows are unreachable. Declare \ + `partition_max_size: {}` to match the key.", + max_size.type_name(), + rows - key_span, + pk_text + ), + )); + } + + Ok((pk, pk_type)) +} + +/// Bytes in a fixed-width unsigned type, or `None` for `usize`, whose width is +/// the target's rather than the declaration's. +fn key_bytes(name: &str) -> Option { + match name { + "u8" => Some(1), + "u16" => Some(2), + "u32" => Some(4), + "u64" => Some(8), + _ => None, + } +} + +/// Generate `DenseTable`. +/// +/// `row_ident` is the row the paged or `Vec` generator already emitted: the +/// dense payload reuses it rather than declaring a parallel one, so a caller +/// carries one row type whichever shape the partition has. +pub fn expand( + name: &Ident, + columns: &Columns, + max_size: PartitionMaxSize, + queries: &DenseQueries, +) -> syn::Result { + let (pk, pk_type) = validate(name, columns, max_size)?; + let rows = max_size.rows().expect("only a dense width reaches here"); + + let row_ident = format_ident!("{}Row", name); + let table = type_ident(name); + let cap = Literal::usize_suffixed(usize::try_from(rows).expect("a cap is at most 65,536")); + + let per_column = columns + .columns_map + .iter() + .filter(|(column, _)| **column != pk) + .map(|(column, ty)| { + let setter = format_ident!("update_{}", column); + let doc = format!( + "Set `{column}` on the row at `{pk}`, in place.\n\n\ + Returns the previous value, or `None` if that key holds no row. \ + The row is never cloned: at a wide row that is the difference \ + between touching one field and copying the row twice." + ); + quote! { + #[doc = #doc] + pub fn #setter(&self, #pk: &#pk_type, value: #ty) -> Option<#ty> { + let at = Self::at(#pk)?; + self.inner.update(at, |row| core::mem::replace(&mut row.#column, value)) + } + } + }) + .collect::>(); + + let query_methods = gen_queries(name, columns, &pk, &pk_type, queries)?; + + let table_doc = format!( + "One partition of [`{name}Partitions`], addressed by position.\n\n\ + `{pk}` is not looked up, it *is* the row's position, so this table has no \ + primary index at all. `partition_max_size` declares {rows} rows here; the row \ + vector still grows only to the highest key used, so the declared width is a \ + bound rather than a reservation.\n\n\ + Every method takes `&self`, because `partition_or_create` hands out an `Arc`. \ + Writes serialise per partition. See `worktable::partition::DenseRows` for what \ + this drops relative to a full table and why each is safe to drop at this size." + ); + + Ok(quote! { + #[doc = #table_doc] + #[derive(Debug)] + pub struct #table { + inner: worktable::partition::DenseRows<#row_ident>, + } + + impl Default for #table { + fn default() -> Self { + Self { inner: worktable::partition::DenseRows::new(#cap) } + } + } + + impl #table { + /// Rows one partition holds, as `partition_max_size` declared it. + pub const MAX_ROWS: usize = #cap; + + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// The key as a position, or `None` if it does not fit one. + /// + /// Only a 32-bit target with a key above `u32::MAX` fails here, and + /// a cap is at most 65,536, so such a key is out of range anyway. + #[inline] + fn at(#pk: &#pk_type) -> Option { + usize::try_from(*#pk).ok() + } + + /// The position a key names, refusing one that does not fit. + #[inline] + fn at_checked(#pk: &#pk_type) -> Result { + Self::at(#pk).ok_or_else(|| { + worktable::partition::DenseError::out_of_range(*#pk as u64, Self::MAX_ROWS) + }) + } + + /// Insert, refusing a key that is occupied or out of range. + pub fn insert(&self, row: #row_ident) -> Result<(), worktable::partition::DenseError> { + let at = Self::at_checked(&row.#pk)?; + self.inner.insert(at, row) + } + + /// Insert, or replace the row this key already names. + pub fn upsert( + &self, + row: #row_ident, + ) -> Result, worktable::partition::DenseError> { + let at = Self::at_checked(&row.#pk)?; + self.inner.upsert(at, row) + } + + /// The row this key names, cloned out. + /// + /// One bounds check and one load. There is no tree to descend and + /// nothing to hash. + #[must_use] + pub fn select(&self, #pk: &#pk_type) -> Option<#row_ident> { + self.inner.get(Self::at(#pk)?) + } + + /// Whether this key holds a row. + #[must_use] + pub fn contains(&self, #pk: &#pk_type) -> bool { + Self::at(#pk).is_some_and(|at| self.inner.contains(at)) + } + + /// Replace the whole row this key names, returning the old one. + /// + /// `None` means the key held nothing, and nothing was written: this + /// updates, it does not insert. `upsert` is the one that does both. + pub fn update( + &self, + row: #row_ident, + ) -> Result, worktable::partition::DenseError> { + let at = Self::at_checked(&row.#pk)?; + Ok(self.inner.update(at, |slot| core::mem::replace(slot, row))) + } + + #(#per_column)* + + #(#query_methods)* + + /// Take the row this key names out. + /// + /// Nothing shifts. A position is a key, so compacting would + /// renumber every row above it. + pub fn delete(&self, #pk: &#pk_type) -> Option<#row_ident> { + self.inner.remove(Self::at(#pk)?) + } + + /// Every row present, ascending by key. + #[must_use] + pub fn select_all(&self) -> worktable::prelude::Vec<#row_ident> { + self.inner.iter().into_iter().map(|(_, row)| row).collect() + } + + /// Rows present. Does not take the lock. + #[must_use] + pub fn row_count(&self) -> usize { + self.inner.row_count() + } + + /// Rows present, under the name every other table uses. + #[must_use] + pub fn len(&self) -> usize { + self.inner.row_count() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + /// Slots allocated, present or not. + /// + /// One past the highest key ever inserted, and **not** the declared + /// cap. This is the figure that explains the shape's memory, so it + /// is exposed rather than inferred. + #[must_use] + pub fn slots(&self) -> usize { + self.inner.slots() + } + + /// Row bytes, which here is the whole table. + /// + /// `slots * size_of::>()`. There is no index to add, + /// which is the point: at 23 rows the index is most of what a full + /// table costs. A column owning a heap allocation is not counted, + /// the same gap the paged table's figure has. + #[must_use] + pub fn used_bytes(&self) -> u64 { + (self.inner.slots() * core::mem::size_of::>()) as u64 + } + } + }) +} + +/// Generate one method per `queries:` entry. +/// +/// The method names and the `Query` argument structs are the paged +/// table's: a partitioned declaration still generates the full table beside the +/// dense payload, so the query structs already exist and a caller keeps the +/// same call. What differs is the signature, deliberately, the same way every +/// other pair of shapes in this crate differs: there is no `.await` and no +/// `WorkTableError`, so moving a call between them fails to compile rather than +/// quietly changing what it guarantees. +fn gen_queries( + name: &Ident, + columns: &Columns, + pk: &Ident, + pk_type: &TokenStream, + queries: &DenseQueries, +) -> syn::Result> { + if queries.is_empty() { + return Ok(Vec::new()); + } + + // `in_place` exists on the paged table because a write there is async and + // has to hold a column across a suspension point. Nothing here is async and + // `update` is already in place, so generating both would be two names for + // one method. + if let Some((query, _)) = queries.in_place.first() { + return Err(Error::new( + query.span(), + format!( + "`in_place {query}` has no meaning on a dense partition: every update here is \ + already in place, because there is no page to rewrite and no await to hold a \ + column across. Declare it as `update {query}`, or use `partition_max_size: u64` \ + for the full table." + ), + )); + } + + let mut out = Vec::new(); + + for (query, op) in &queries.updates { + by_must_be_the_key(pk, query, op, "update")?; + let method = format_ident!("update_{}", snake(query)); + let query_ty = format_ident!("{}Query", query); + let fields = &op.columns; + for column in fields { + if !columns.columns_map.contains_key(column) { + return Err(Error::new(column.span(), format!("no column `{column}`"))); + } + } + let doc = format!( + "`update {query}`, by position.\n\n\ + Edits {} in place on the row at `{pk}`, without cloning the row. \ + `None` means that key holds no row and nothing was written.\n\n\ + The paged table's method of this name is `async` and returns \ + `Result<(), WorkTableError>`. This one is neither, so a call does not \ + move silently between the two shapes.", + fields.iter().map(|f| format!("`{f}`")).collect::>().join(", ") + ); + out.push(quote! { + #[doc = #doc] + pub fn #method(&self, row: #query_ty, #pk: &#pk_type) -> Option<()> { + let at = Self::at(#pk)?; + self.inner.update(at, |target| { + #(target.#fields = row.#fields;)* + }) + } + }); + } + + for (query, op) in &queries.deletes { + by_must_be_the_key(pk, query, op, "delete")?; + let method = format_ident!("delete_{}", snake(query)); + let row_ident = format_ident!("{}Row", name); + let doc = format!( + "`delete {query}`, by position.\n\n\ + Takes the row at `{pk}` out and returns it. Nothing shifts: a position \ + is a key, so compacting would renumber every row above it." + ); + out.push(quote! { + #[doc = #doc] + pub fn #method(&self, #pk: &#pk_type) -> Option<#row_ident> { + self.inner.remove(Self::at(#pk)?) + } + }); + } + + Ok(out) +} + +/// A dense partition has no secondary index, so a query can only be keyed by +/// the position. +/// +/// Refused rather than scanned. A scan of at most 65,536 rows would work and +/// would be the wrong thing to generate silently: the declaration asks for a +/// keyed operation and would get a linear one, which is the sort of quiet +/// downgrade the rest of this crate refuses. +fn by_must_be_the_key(pk: &Ident, query: &Ident, op: &Operation, kind: &str) -> syn::Result<()> { + if op.by == *pk { + return Ok(()); + } + Err(Error::new( + op.by.span(), + format!( + "`{kind} {query} ... by {}` needs an index on `{}`, and a dense partition has none: \ + the only key it can address a row by is its position, which is `{pk}`. Key the query \ + by `{pk}`, or use `partition_max_size: u64` for the full table and its indexes.", + op.by, op.by + ), + )) +} + +/// `TopPrice` to `top_price`, the same casing the paged table's methods use. +fn snake(name: &Ident) -> String { + use convert_case::{Case, Casing as _}; + name.to_string().from_case(Case::Pascal).to_case(Case::Snake) +} diff --git a/codegen/src/generators/in_memory/index/cdc.rs b/codegen/src/generators/in_memory/index/cdc.rs index ed85e087..3394f3f5 100644 --- a/codegen/src/generators/in_memory/index/cdc.rs +++ b/codegen/src/generators/in_memory/index/cdc.rs @@ -311,7 +311,7 @@ impl InMemoryGenerator { fn process_difference_remove_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { #(#process_difference_rows)* (#events_ident { @@ -381,7 +381,7 @@ impl InMemoryGenerator { fn process_difference_insert_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; let mut partial_events = #events_ident::default(); diff --git a/codegen/src/generators/in_memory/index/info.rs b/codegen/src/generators/in_memory/index/info.rs index 782db9fb..24a84490 100644 --- a/codegen/src/generators/in_memory/index/info.rs +++ b/codegen/src/generators/in_memory/index/info.rs @@ -32,6 +32,14 @@ impl InMemoryGenerator { quote! { self.#index_field_name.capacity() }, quote! { self.#index_field_name.node_count() }, ), + // Refused before any generator runs (`worktable/mod.rs`), so this + // is unreachable. It emits a refusal rather than panicking + // because a future path that reaches it should fail at the + // declaration, not inside the macro. + crate::common::model::IndexBackend::FxHash => ( + quote! { compile_error!("`using fxhash` cannot back a paged table") }, + quote! { compile_error!("`using fxhash` cannot back a paged table") }, + ), crate::common::model::IndexBackend::Congee | crate::common::model::IndexBackend::Arctic => ( // Neither ART exposes allocator capacity or internal // node counts through its stable public API. diff --git a/codegen/src/generators/in_memory/index/mod.rs b/codegen/src/generators/in_memory/index/mod.rs index b7024fc2..3b50d2a7 100644 --- a/codegen/src/generators/in_memory/index/mod.rs +++ b/codegen/src/generators/in_memory/index/mod.rs @@ -12,6 +12,7 @@ use quote::quote; impl InMemoryGenerator { /// Generates index type and it's impls. pub fn gen_index_def(&mut self) -> syn::Result { + let columnar_def = crate::generators::columnar::definitions(&self.name, &self.columns); let type_def = self.gen_type_def()?; let impl_def = self.gen_secondary_index_impl_def(); let info_def = self.gen_secondary_index_info_impl_def(); @@ -24,6 +25,7 @@ impl InMemoryGenerator { let available_indexes = self.gen_available_indexes(); Ok(quote! { + #columnar_def #type_def #impl_def #info_def @@ -101,11 +103,13 @@ impl InMemoryGenerator { #[derive(Debug, MemStat)] } }; + let columnar_field = crate::generators::columnar::index_struct_field(&self.name, &self.columns, false); Ok(quote! { #derive pub struct #ident { - #(#index_rows),* + #(#index_rows,)* + #columnar_field } }) } @@ -152,6 +156,11 @@ impl InMemoryGenerator { get_index_page_size_from_data_length::<#t>(#const_name) ), }, + // Unreachable: refused in `worktable/mod.rs` before any + // generator runs. + crate::common::model::IndexBackend::FxHash => quote! { + #i: compile_error!("`using fxhash` cannot back a paged table"), + }, crate::common::model::IndexBackend::Congee | crate::common::model::IndexBackend::Arctic => { quote! { #i: Default::default(), } @@ -175,12 +184,14 @@ impl InMemoryGenerator { Ok::<_, syn::Error>(res) }) .collect::, syn::Error>>()?; + let columnar_field = crate::generators::columnar::index_default_field(&self.columns); Ok(quote! { impl Default for #index_type_ident { fn default() -> Self { Self { #(#index_rows)* + #columnar_field } } } @@ -205,11 +216,22 @@ impl InMemoryGenerator { } } else { quote! { - #[derive(Debug, Clone, Copy, MoreDisplay, PartialEq, PartialOrd, Ord, Hash, Eq)] + #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Ord, Hash, Eq)] pub enum #avt_type_ident { #(#indexes)* } + // Delegated to `Debug` rather than derived. Every variant here + // is fieldless, so `Debug` prints exactly the variant name, + // which is what `derive_more::Display` produced. Deriving it + // put `::derive_more::` paths in the expansion and so put that + // crate into this macro's contract. + impl core::fmt::Display for #avt_type_ident { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Debug::fmt(self, f) + } + } + impl AvailableIndex for #avt_type_ident { fn to_string_value(&self) -> String { ToString::to_string(&self) diff --git a/codegen/src/generators/in_memory/index/usual.rs b/codegen/src/generators/in_memory/index/usual.rs index 2c90abc7..bdec9a24 100644 --- a/codegen/src/generators/in_memory/index/usual.rs +++ b/codegen/src/generators/in_memory/index/usual.rs @@ -15,6 +15,7 @@ impl InMemoryGenerator { let avt_index_ident = name_generator.get_available_indexes_ident(); let save_row_fn = self.gen_save_row_index_fn(); + let publication_guard = crate::generators::columnar::publication_guard(&self.columns); let reinsert_row_fn = self.gen_reinsert_row_index_fn(); let delete_row_fn = self.gen_delete_row_index_fn(); let process_difference_insert_fn = self.gen_process_difference_insert_index_fn(); @@ -24,6 +25,7 @@ impl InMemoryGenerator { quote! { impl TableSecondaryIndex<#row_type_ident, #avt_type_ident, #avt_index_ident> for #index_type_ident { #save_row_fn + #publication_guard #reinsert_row_fn #delete_row_fn #process_difference_insert_fn @@ -73,11 +75,13 @@ impl InMemoryGenerator { } }) .collect::>(); + let columnar_save = crate::generators::columnar::save_row(&self.columns); quote! { fn save_row(&self, row: #row_type_ident, link: Link) -> core::result::Result<(), IndexError<#available_index_ident>> { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; #(#save_rows)* + #columnar_save core::result::Result::Ok(()) } } @@ -151,6 +155,7 @@ impl InMemoryGenerator { (insert, remove) }) .unzip(); + let columnar_reinsert = crate::generators::columnar::reinsert_row(&self.columns); quote! { fn reinsert_row(&self, @@ -163,6 +168,7 @@ impl InMemoryGenerator { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; #(#insert_rows)* #(#remove_rows)* + #columnar_reinsert core::result::Result::Ok(()) } } @@ -196,10 +202,12 @@ impl InMemoryGenerator { } }) .collect::>(); + let columnar_delete = crate::generators::columnar::delete_row(&self.columns); quote! { fn delete_row(&self, row: #row_type_ident, link: Link) -> core::result::Result<(), IndexError<#available_index_ident>> { #(#delete_rows)* + #columnar_delete core::result::Result::Ok(()) } } @@ -240,14 +248,16 @@ impl InMemoryGenerator { quote! {} } }); + let columnar_dirty = crate::generators::columnar::mark_dirty(&self.columns); quote! { fn process_difference_remove( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { #(#process_difference_remove_rows)* + #columnar_dirty core::result::Result::Ok(()) } } @@ -299,15 +309,17 @@ impl InMemoryGenerator { quote! {} } }); + let columnar_dirty = crate::generators::columnar::mark_dirty(&self.columns); quote! { fn process_difference_insert( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { let mut inserted_indexes: Vec<#avt_index_ident> = vec![]; #(#process_difference_insert_rows)* + #columnar_dirty core::result::Result::Ok(()) } } diff --git a/codegen/src/generators/in_memory/locks.rs b/codegen/src/generators/in_memory/locks.rs index 78b40aec..1153c65c 100644 --- a/codegen/src/generators/in_memory/locks.rs +++ b/codegen/src/generators/in_memory/locks.rs @@ -24,7 +24,7 @@ impl InMemoryGenerator { .keys() .map(|i| { let name = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); - quote! { #name: Option>, } + quote! { #name: Option>, } }) .collect(); @@ -124,8 +124,8 @@ impl InMemoryGenerator { .collect(); quote! { - fn with_lock(id: u16) -> (Self, std::sync::Arc) { - let lock = std::sync::Arc::new(Lock::new(id)); + fn with_lock(id: u16) -> (Self, worktable::prelude::Arc) { + let lock = worktable::prelude::Arc::new(Lock::new(id)); ( Self { #(#rows),* @@ -154,9 +154,9 @@ impl InMemoryGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let lock = std::sync::Arc::new(Lock::new(id)); + fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* (set, lock) @@ -186,8 +186,8 @@ impl InMemoryGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> std::collections::HashSet> { - let mut set = std::collections::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { + let mut set = worktable::prelude::HashSet::new(); #(#rows)* set } diff --git a/codegen/src/generators/in_memory/primary_key.rs b/codegen/src/generators/in_memory/primary_key.rs index 8d58b84f..fcdd0d17 100644 --- a/codegen/src/generators/in_memory/primary_key.rs +++ b/codegen/src/generators/in_memory/primary_key.rs @@ -70,19 +70,68 @@ impl InMemoryGenerator { primary_key_backend_impl(self.columns.primary_index_backend, &ident, types)?; let borrowed_impl = gen_borrowed_primary_key_impl(&ident, types); + // `From` written out rather than derived. `derive_more::From` puts + // `::derive_more::` paths in its expansion, which made that crate part + // of this macro's contract: a consumer who never wrote `derive_more` + // had to declare it anyway. A newtype conversion is three lines. + // + // A composite key converts from the tuple, which is the shape + // `derive_more` produced for a multi-field tuple struct. + let from_impl = if types.len() == 1 { + let ty = &types[0]; + quote! { + impl From<#ty> for #ident { + fn from(value: #ty) -> Self { + Self(value) + } + } + } + } else { + let binding: Vec<_> = (0..types.len()) + .map(|i| syn::Ident::new(&format!("field{i}"), proc_macro2::Span::mixed_site())) + .collect(); + quote! { + impl From<(#(#types),*)> for #ident { + fn from((#(#binding),*): (#(#types),*)) -> Self { + Self(#(#binding),*) + } + } + } + }; + + // And the reverse direction, which was `derive_more::Into`. Same + // reasoning: it is one impl, and deriving it dragged the crate in. + let into_impl = if types.len() == 1 { + let ty = &types[0]; + quote! { + impl From<#ident> for #ty { + fn from(value: #ident) -> Self { + value.0 + } + } + } + } else { + let index: Vec<_> = (0..types.len()).map(syn::Index::from).collect(); + quote! { + impl From<#ident> for (#(#types),*) { + fn from(value: #ident) -> Self { + (#(value.#index),*) + } + } + } + }; + Ok(quote! { #[derive( Clone, #backend_derive - rkyv::Archive, + worktable::prelude::rkyv::Archive, Debug, Default, - rkyv::Deserialize, + worktable::prelude::rkyv::Deserialize, Hash, - rkyv::Serialize, - From, + worktable::prelude::rkyv::Serialize, Eq, - Into, PartialEq, PartialOrd, Ord, @@ -90,9 +139,13 @@ impl InMemoryGenerator { MemStat, #unsized_derive )] + #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(PartialEq, Eq, PartialOrd, Ord, Debug))] pub struct #ident(#(#types),*); + #from_impl + #into_impl + #borrowed_impl #backend_impl }) @@ -140,14 +193,14 @@ impl InMemoryGenerator { /// atomic of primitive. fn get_generator_from_type(type_: &TokenStream, i: &Ident) -> syn::Result { Ok(match type_.to_string().as_str() { - "u8" => quote! { std::sync::atomic::AtomicU8 }, - "u16" => quote! { std::sync::atomic::AtomicU16 }, - "u32" => quote! { std::sync::atomic::AtomicU32 }, - "u64" => quote! { std::sync::atomic::AtomicU64 }, - "i8" => quote! { std::sync::atomic::AtomicI8 }, - "i16" => quote! { std::sync::atomic::AtomicI16 }, - "i32" => quote! { std::sync::atomic::AtomicI32 }, - "i64" => quote! { std::sync::atomic::AtomicI64 }, + "u8" => quote! { core::sync::atomic::AtomicU8 }, + "u16" => quote! { core::sync::atomic::AtomicU16 }, + "u32" => quote! { core::sync::atomic::AtomicU32 }, + "u64" => quote! { core::sync::atomic::AtomicU64 }, + "i8" => quote! { core::sync::atomic::AtomicI8 }, + "i16" => quote! { core::sync::atomic::AtomicI16 }, + "i32" => quote! { core::sync::atomic::AtomicI32 }, + "i64" => quote! { core::sync::atomic::AtomicI64 }, // The accepted set is `worktable_dsl::AUTOINCREMENT_TYPES`, and the // arms above must stay equal to it. `check` uses that list to // answer "would the macro accept this", so a second copy drifting diff --git a/codegen/src/generators/in_memory/queries/delete.rs b/codegen/src/generators/in_memory/queries/delete.rs index 911f7e9a..df23a6c9 100644 --- a/codegen/src/generators/in_memory/queries/delete.rs +++ b/codegen/src/generators/in_memory/queries/delete.rs @@ -14,7 +14,13 @@ impl InMemoryGenerator { let table_ident = name_generator.get_work_table_ident(); let custom_deletes = if let Some(q) = &self.queries { + let profile = q.delete_runtime.clone(); let custom_deletes = self.gen_custom_deletes(q.deletes.clone()); + let custom_deletes = crate::generators::profile_dispatch::wrap( + custom_deletes, + profile.as_ref(), + &name_generator.get_row_type_ident(), + )?; quote! { #custom_deletes } @@ -38,6 +44,7 @@ impl InMemoryGenerator { let pk_ident = name_generator.get_primary_key_type_ident(); let delete_logic = self.gen_delete_logic(true); let full_row_lock = self.gen_full_lock_for_update(); + let publication = crate::generators::columnar::table_publication_guard(&self.columns); quote! { pub async fn delete(&self, pk: Pk) -> core::result::Result<(), WorkTableError> @@ -46,6 +53,7 @@ impl InMemoryGenerator { let pk: #pk_ident = pk.into(); let pending_lock = { #full_row_lock }; let _guard = pending_lock.into_guard_with_mutation(); + #publication #delete_logic @@ -58,6 +66,7 @@ impl InMemoryGenerator { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let pk_ident = name_generator.get_primary_key_type_ident(); let delete_logic = self.gen_delete_logic(false); + let publication = crate::generators::columnar::table_publication_guard(&self.columns); quote! { pub async fn delete_without_lock(&self, pk: Pk) -> core::result::Result<(), WorkTableError> @@ -65,6 +74,7 @@ impl InMemoryGenerator { { let pk: #pk_ident = pk.into(); let _mutation_guard = self.0.lock_manager.mutation_guard(&pk); + #publication #delete_logic core::result::Result::Ok(()) } @@ -87,7 +97,7 @@ impl InMemoryGenerator { #pk_ident, #secondary_events_ident > = Operation::Delete(DeleteOperation { - id: uuid::Uuid::now_v7().into(), + id: worktable::prelude::uuid::Uuid::now_v7().into(), secondary_keys_events, primary_key_events, link, @@ -186,7 +196,7 @@ impl InMemoryGenerator { quote! { pub async fn #name(&self, by: #type_) -> core::result::Result<(), WorkTableError> { let _bulk_mutation = self.0.lock_manager.bulk_mutation_guard(); - let pks = std::cell::RefCell::new(Vec::new()); + let pks = core::cell::RefCell::new(Vec::new()); self.iter_with(|row| { if row.#field == by { pks.borrow_mut().push(row.get_primary_key()); diff --git a/codegen/src/generators/in_memory/queries/in_place.rs b/codegen/src/generators/in_memory/queries/in_place.rs index 1c4c3e3c..06450463 100644 --- a/codegen/src/generators/in_memory/queries/in_place.rs +++ b/codegen/src/generators/in_memory/queries/in_place.rs @@ -11,7 +11,13 @@ impl InMemoryGenerator { let table_ident = name_generator.get_work_table_ident(); let custom_in_place = if let Some(q) = &self.queries { + let profile = q.in_place_runtime.clone(); let custom_in_place = self.gen_in_place_queries(q.in_place.clone()); + let custom_in_place = crate::generators::profile_dispatch::wrap( + custom_in_place, + profile.as_ref(), + &name_generator.get_row_type_ident(), + )?; quote! { #custom_in_place } @@ -66,12 +72,12 @@ impl InMemoryGenerator { let column_types = if types.len() == 1 { let t = types[0]; quote! { - &mut <#t as rkyv::Archive>::Archived + &mut <#t as worktable::prelude::rkyv::Archive>::Archived } } else { let types = types.iter().map(|t| { quote! { - &mut <#t as rkyv::Archive>::Archived + &mut <#t as worktable::prelude::rkyv::Archive>::Archived } }); quote! { @@ -94,13 +100,14 @@ impl InMemoryGenerator { } }; let custom_lock = self.gen_custom_lock_for_update(lock_ident); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); quote! { pub async fn #method_ident( &self, mut f: F, by: Pk, - ) -> eyre::Result<()> + ) -> worktable::prelude::eyre::Result<()> where #pk_type: From { let pk: #pk_type = by.into(); @@ -119,6 +126,7 @@ impl InMemoryGenerator { .map_err(WorkTableError::PagesError)? }; + #columnar_dirty Ok(()) } } diff --git a/codegen/src/generators/in_memory/queries/locks.rs b/codegen/src/generators/in_memory/queries/locks.rs index cf80019f..96522cd8 100644 --- a/codegen/src/generators/in_memory/queries/locks.rs +++ b/codegen/src/generators/in_memory/queries/locks.rs @@ -95,9 +95,9 @@ impl InMemoryGenerator { quote! { #[allow(clippy::mutable_key_type)] - pub fn #ident(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let new_lock = std::sync::Arc::new(Lock::new(id)); + pub fn #ident(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let new_lock = worktable::prelude::Arc::new(Lock::new(id)); #(#inner)* (set, new_lock) } @@ -124,7 +124,7 @@ impl InMemoryGenerator { // timeout, task abort) would otherwise leave the registered lock // held forever and hang every later operation on this key. let pending_lock = PendingLock::new(op_lock, self.0.lock_manager.clone(), pk.clone()); - futures::future::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; pending_lock } } @@ -153,7 +153,7 @@ impl InMemoryGenerator { // timeout, task abort) would otherwise leave the registered lock // held forever and hang every later operation on this key. let pending_lock = PendingLock::new(op_lock, self.0.lock_manager.clone(), pk.clone()); - futures::future::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; pending_lock } } diff --git a/codegen/src/generators/in_memory/queries/select.rs b/codegen/src/generators/in_memory/queries/select.rs index 42139128..744cd0ea 100644 --- a/codegen/src/generators/in_memory/queries/select.rs +++ b/codegen/src/generators/in_memory/queries/select.rs @@ -32,7 +32,7 @@ impl InMemoryGenerator { // Acquire the grace-period guard only when iteration starts. // Merely constructing and retaining a query builder must not // stall retired-link reclamation. - let iter = std::iter::once_with(move || { + let iter = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .iter_values() @@ -55,7 +55,7 @@ impl InMemoryGenerator { return None; } current_link = replacement; - std::hint::spin_loop(); + core::hint::spin_loop(); } None }) diff --git a/codegen/src/generators/in_memory/queries/type.rs b/codegen/src/generators/in_memory/queries/type.rs index 05ded8f3..12cbf007 100644 --- a/codegen/src/generators/in_memory/queries/type.rs +++ b/codegen/src/generators/in_memory/queries/type.rs @@ -46,20 +46,38 @@ impl InMemoryGenerator { .expect("should be valid because parsed from declaration"); let type_upper = map_to_uppercase(s); let type_upper = Ident::new(type_upper.as_str(), Span::mixed_site()); - Some(quote! { - #[from] - #type_upper(#type_ident), - }) + Some(( + quote! { + #type_upper(#type_ident), + }, + // Written out rather than derived. `derive_more::From` + // generates `::derive_more::` paths inside its expansion, + // which makes that crate part of this macro's contract: + // a consumer who never wrote `derive_more` still had to + // declare it to compile a table. One newtype variant per + // type is a two-line impl, so the dependency bought + // nothing that could not be spelled here. + quote! { + impl From<#type_ident> for #avt_type_ident { + fn from(value: #type_ident) -> Self { + Self::#type_upper(value) + } + } + }, + )) }) .collect(); + let (rows, from_impls): (Vec<_>, Vec<_>) = rows.into_iter().flatten().unzip(); if !rows.is_empty() { Ok(quote! { - #[derive(Clone, Debug, From, PartialEq)] + #[derive(Clone, Debug, PartialEq)] #[non_exhaustive] pub enum #avt_type_ident { #(#rows)* } + + #(#from_impls)* }) } else { Ok(quote! { @@ -122,7 +140,8 @@ impl InMemoryGenerator { Ok::<_, syn::Error>(quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, Clone, rkyv::Serialize)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, Clone, worktable::prelude::rkyv::Serialize)] + #[rkyv(crate = worktable::prelude::rkyv)] #[repr(C)] pub struct #ident { #(#rows)* diff --git a/codegen/src/generators/in_memory/queries/update.rs b/codegen/src/generators/in_memory/queries/update.rs index 04af1554..3001fdd3 100644 --- a/codegen/src/generators/in_memory/queries/update.rs +++ b/codegen/src/generators/in_memory/queries/update.rs @@ -10,7 +10,13 @@ use quote::quote; impl InMemoryGenerator { pub fn gen_query_update_impl(&mut self) -> syn::Result { let custom_updates = if let Some(q) = &self.queries { + let profile = q.update_runtime.clone(); let custom_updates = self.gen_custom_updates(q.updates.clone()); + let custom_updates = crate::generators::profile_dispatch::wrap( + custom_updates, + profile.as_ref(), + &WorktableNameGenerator::from_table_name(self.name.to_string()).get_row_type_ident(), + )?; quote! { #custom_updates @@ -42,7 +48,7 @@ impl InMemoryGenerator { .keys() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -59,6 +65,7 @@ impl InMemoryGenerator { let persist_call = self.gen_persist_call(); let persist_op = self.gen_persist_op(); let full_row_lock = self.gen_full_lock_for_update(); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); // A full-row `update(row)` replaces EVERY column, so it inherently // rewrites every secondary index. The in-place fast path only applies // when no updated field is indexed (it emits no index diff), so a @@ -72,10 +79,10 @@ impl InMemoryGenerator { let full_row_in_place_eligible = !self.columns.is_sized && self.columns.indexes.is_empty(); let update_body = if self.columns.is_sized { quote! { - let mut bytes = rkyv::to_bytes::(&row) + let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; let mut archived_row = unsafe { - rkyv::access_unchecked_mut::<<#row_ident as rkyv::Archive>::Archived>(&mut bytes[..]) + worktable::prelude::rkyv::access_unchecked_mut::<<#row_ident as worktable::prelude::rkyv::Archive>::Archived>(&mut bytes[..]) .unseal_unchecked() }; @@ -85,6 +92,7 @@ impl InMemoryGenerator { #data_write #diff_process_remove + #columnar_dirty #persist_call @@ -99,6 +107,7 @@ impl InMemoryGenerator { self.0.data.update_in_place::<{ #const_name }>(row.clone(), link).is_ok() }; if in_place_ok { + #columnar_dirty return core::result::Result::Ok(()); } drop(_guard); @@ -273,8 +282,8 @@ impl InMemoryGenerator { let avt_type_ident = name_generator.get_available_type_ident(); quote! { if let core::result::Result::Err(e) = #write { - let mut reversed_diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = - std::collections::HashMap::new(); + let mut reversed_diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = + worktable::prelude::HashMap::new(); for (key, diff) in diffs { reversed_diffs.insert(key, Difference { old: diff.new, new: diff.old }); } @@ -440,6 +449,7 @@ impl InMemoryGenerator { #primary_key_ident, #secondary_events_ident > = Operation::Update(UpdateOperation { + retired_link: None, id: op_id, primary_key_events: vec![], secondary_keys_events, @@ -460,7 +470,7 @@ impl InMemoryGenerator { let row_old = self.0.data.select_non_ghosted(link)?; let row_new = row.clone(); let updated_bytes: Vec = vec![]; - let mut diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = std::collections::HashMap::new(); + let mut diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = worktable::prelude::HashMap::new(); } } else { quote! { @@ -517,7 +527,7 @@ impl InMemoryGenerator { // Create AcknowledgeOperation with all events let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(uuid::Uuid::now_v7()), + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), primary_key_events: vec![], // Updates don't modify primary key secondary_keys_events: merged_events, }); @@ -525,6 +535,28 @@ impl InMemoryGenerator { Err(WorkTableError::AlreadyExists(at.to_string_value())) } + IndexError::ColumnSlotIdExhausted { + bits, + inserted_already, + } => { + let (rollback_secondary_events, _): (#secondary_events_ident, _) = self.0.indexes.delete_from_indexes_cdc( + row_new.merge(row_old.clone()), + link, + inserted_already + ); + + let mut merged_events = secondary_events.clone(); + merged_events.extend(rollback_secondary_events); + + let ack_op = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events: merged_events, + }); + self.1.apply_operation(ack_op); + + Err(WorkTableError::ColumnSlotIdExhausted(bits)) + } IndexError::NotFound => Err(WorkTableError::NotFound), }; } @@ -549,6 +581,15 @@ impl InMemoryGenerator { Err(WorkTableError::AlreadyExists(at.to_string_value())) } + IndexError::ColumnSlotIdExhausted { + bits, + inserted_already, + } => { + self.0.indexes + .delete_from_indexes(row_new.merge(row_old.clone()), link, inserted_already)?; + + Err(WorkTableError::ColumnSlotIdExhausted(bits)) + } IndexError::NotFound => Err(WorkTableError::NotFound), }; } @@ -605,7 +646,7 @@ impl InMemoryGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -619,6 +660,8 @@ impl InMemoryGenerator { let custom_lock = self.gen_custom_lock_for_update(lock_ident); let data_write = self.gen_data_write_with_unwind(&row_updates, idx_idents); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); + let finish_update = if archived_swap_is_safe { quote! { #diff_process_insert @@ -627,6 +670,7 @@ impl InMemoryGenerator { #data_write #diff_process_remove + #columnar_dirty #persist_call @@ -651,8 +695,8 @@ impl InMemoryGenerator { .map(Into::into) .ok_or(WorkTableError::NotFound)?; - let mut bytes = rkyv::to_bytes::(&row).map_err(|_| WorkTableError::SerializeError)?; - let mut archived_row = unsafe { rkyv::access_unchecked_mut::<<#query_ident as rkyv::Archive>::Archived>(&mut bytes[..]).unseal_unchecked() }; + let mut bytes = worktable::prelude::rkyv::to_bytes::(&row).map_err(|_| WorkTableError::SerializeError)?; + let mut archived_row = unsafe { worktable::prelude::rkyv::access_unchecked_mut::<<#query_ident as worktable::prelude::rkyv::Archive>::Archived>(&mut bytes[..]).unseal_unchecked() }; #size_check #finish_update @@ -680,7 +724,7 @@ impl InMemoryGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -780,6 +824,7 @@ impl InMemoryGenerator { }; let full_row_lock = self.gen_full_lock_for_update(); let data_write = self.gen_data_write_with_unwind(&row_updates, idx_idents); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); let loop_tail = if has_unsized { quote! {} @@ -830,7 +875,7 @@ impl InMemoryGenerator { pks.sort_unstable(); pks.dedup(); - let mut guards: std::collections::HashMap<_, _> = std::collections::HashMap::new(); + let mut guards: worktable::prelude::HashMap<_, _> = worktable::prelude::HashMap::new(); // Full-row locks, not per-column custom locks: each row's // unsized reinsert path mutates the whole row under these // guards, and one uniform lock kind keeps every concurrent @@ -857,17 +902,18 @@ impl InMemoryGenerator { continue; } let _mutation_guard = self.0.lock_manager.mutation_guard(&pk); - let mut bytes = rkyv::to_bytes::(&row) + let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; let mut archived_row = unsafe { - rkyv::access_unchecked_mut::<<#query_ident as rkyv::Archive>::Archived>(&mut bytes[..]) + worktable::prelude::rkyv::access_unchecked_mut::<<#query_ident as worktable::prelude::rkyv::Archive>::Archived>(&mut bytes[..]) .unseal_unchecked() }; #size_check #loop_tail } + #columnar_dirty core::result::Result::Ok(()) } } @@ -902,7 +948,7 @@ impl InMemoryGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -945,6 +991,8 @@ impl InMemoryGenerator { }; let custom_lock = self.gen_custom_lock_for_update(lock_ident); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); + let finish_update = if archived_swap_is_safe { quote! { #diff_process_insert @@ -953,6 +1001,7 @@ impl InMemoryGenerator { #data_write #diff_process_remove + #columnar_dirty #persist_call @@ -964,21 +1013,43 @@ impl InMemoryGenerator { quote! { pub async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { - let mut bytes = rkyv::to_bytes::(&row) + let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; let mut archived_row = unsafe { - rkyv::access_unchecked_mut::<<#query_ident as rkyv::Archive>::Archived>(&mut bytes[..]) + worktable::prelude::rkyv::access_unchecked_mut::<<#query_ident as worktable::prelude::rkyv::Archive>::Archived>(&mut bytes[..]) .unseal_unchecked() }; - let mut link: Link = self.0.indexes - .#index - .get_value(#by) - .map(Into::into) - .ok_or(WorkTableError::NotFound)?; - - let pk = self.0.data.select_non_ghosted(link)?.get_primary_key().clone(); + let pk = { + let mut retries = 0u32; + loop { + // Pin before reading the index so a relocated slot + // cannot be reclaimed and reused while resolving its PK. + // Drop the pin before yielding or awaiting the row lock. + let resolved = { + let _read_guard = self.0.data.read_guard(); + let link: Link = self.0.indexes.#index.get_value(#by) + .map(Into::into) + .ok_or(WorkTableError::NotFound)?; + self.0.data.select_non_ghosted(link) + }; + match resolved { + core::result::Result::Ok(found) => break found.get_primary_key(), + core::result::Result::Err(error) if error.is_row_absent() => { + // Reinsert publishes a replacement before retiring + // the old slot. Resolve the index again, rather than + // reporting a deleted row from that stale slot. + if retries >= 64 { + return Err(WorkTableError::NotFound); + } + retries += 1; + worktable::prelude::yield_now().await; + } + core::result::Result::Err(error) => return Err(error.into()), + } + } + }; let pending_lock = { #custom_lock }; let _guard = pending_lock.into_guard_with_mutation(); @@ -1005,7 +1076,7 @@ impl InMemoryGenerator { return Err(WorkTableError::NotFound); } vacuum_retries += 1; - tokio::task::yield_now().await; + worktable::prelude::yield_now().await; } core::result::Result::Err(e) => return Err(e.into()), } @@ -1074,6 +1145,7 @@ mod tests { updates, deletes: IndexMap::new(), in_place: IndexMap::new(), + ..Default::default() }); generator.gen_primary_key_def().unwrap(); diff --git a/codegen/src/generators/in_memory/row.rs b/codegen/src/generators/in_memory/row.rs index 089adc33..7bd7aa37 100644 --- a/codegen/src/generators/in_memory/row.rs +++ b/codegen/src/generators/in_memory/row.rs @@ -81,7 +81,8 @@ impl InMemoryGenerator { }; quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, Clone, rkyv::Serialize, PartialEq, MemStat)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, Clone, worktable::prelude::rkyv::Serialize, PartialEq, MemStat)] + #[rkyv(crate = worktable::prelude::rkyv)] #custom_derives #[rkyv(derive(Debug))] #[repr(C)] @@ -122,7 +123,8 @@ impl InMemoryGenerator { .collect(); quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, Clone, rkyv::Serialize, PartialEq)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, Clone, worktable::prelude::rkyv::Serialize, PartialEq)] + #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(Debug))] #[repr(C)] pub enum #enum_name { diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index 198e390a..54778189 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -109,7 +109,7 @@ impl InMemoryGenerator { #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds + 'a, + R: core::ops::RangeBounds + 'a, Pk: Clone + 'a, { let converted_range = ( @@ -118,7 +118,7 @@ impl InMemoryGenerator { ); // Delay the grace-period guard until the returned iterator is // consumed so an idle query builder cannot pin reclamation. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .range_links(converted_range) @@ -293,7 +293,7 @@ impl InMemoryGenerator { } if backoff_spins < 8 { backoff_spins = backoff_spins.saturating_add(1); - tokio::task::yield_now().await; + worktable::prelude::yield_now().await; } else { // Cap the exponent BEFORE shifting: `1u64 << 64` panics // (overflow) in debug/test builds. Clamp the shift to a @@ -302,7 +302,7 @@ impl InMemoryGenerator { let exponent = core::cmp::min(backoff_spins - 8, 8); let micros = core::cmp::min(1u64 << exponent, 256); backoff_spins = backoff_spins.saturating_add(1); - tokio::time::sleep(std::time::Duration::from_micros(micros)).await; + worktable::prelude::sleep(core::time::Duration::from_micros(micros)).await; } } } @@ -343,7 +343,7 @@ impl InMemoryGenerator { /// assigned contiguous keys before `insert_many`. Interleaved /// `get_next_pk` calls keep working and never overlap a /// reservation. - pub fn reserve_pks(&self, count: usize) -> std::ops::Range<#pk_inner_type> { + pub fn reserve_pks(&self, count: usize) -> core::ops::Range<#pk_inner_type> { self.0.reserve_pks(count) } } @@ -375,7 +375,7 @@ impl InMemoryGenerator { quote! { pub async fn iter_with_async< F: Fn(#row_type) -> Fut, - Fut: std::future::Future> + Fut: core::future::Future> >(&self, f: F) -> core::result::Result<(), WorkTableError> { #inner } @@ -435,9 +435,21 @@ impl InMemoryGenerator { let table_name = name_generator.get_work_table_literal_name(); let lock_type = name_generator.get_lock_type_ident(); + // Only when `worktable` itself has `std`. `EmptyDataVacuum` is not + // empty despite the name and holds the data pages, lock manager and + // persistence sink, so it cannot exist without one. A plain + // `#[cfg(feature = "std")]` emitted here would test the *consumer's* + // feature of that name, which is a different flag or none at all. quote! { - pub fn vacuum(&self) -> std::sync::Arc { - std::sync::Arc::new(EmptyDataVacuum::< + worktable::__wt_if_std! { + pub fn vacuum(&self) -> worktable::prelude::Arc { + self.vacuum_with_pacing(worktable::prelude::VacuumPacing::default()) + } + + /// Creates a sweep with the selected pacing policy. Zero batch pages + /// disables pacing; positive values yield between source-page batches. + pub fn vacuum_with_pacing(&self, pacing: worktable::prelude::VacuumPacing) -> worktable::prelude::Arc { + worktable::prelude::Arc::new(EmptyDataVacuum::< _, _, _, @@ -448,11 +460,12 @@ impl InMemoryGenerator { _ >::new( #table_name, - std::sync::Arc::clone(&self.0.data), - std::sync::Arc::clone(&self.0.lock_manager), - std::sync::Arc::clone(&self.0.primary_index), - std::sync::Arc::clone(&self.0.indexes), - )) + worktable::prelude::Arc::clone(&self.0.data), + worktable::prelude::Arc::clone(&self.0.lock_manager), + worktable::prelude::Arc::clone(&self.0.primary_index), + worktable::prelude::Arc::clone(&self.0.indexes), + ).with_pacing(pacing)) + } } } } diff --git a/codegen/src/generators/in_memory/table/index_fns.rs b/codegen/src/generators/in_memory/table/index_fns.rs index 35d51744..1317e50c 100644 --- a/codegen/src/generators/in_memory/table/index_fns.rs +++ b/codegen/src/generators/in_memory/table/index_fns.rs @@ -99,7 +99,7 @@ impl InMemoryGenerator { if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None }; @@ -140,7 +140,7 @@ impl InMemoryGenerator { #column_range_type, #row_fields_ident> { - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.indexes.#field_ident .get(#by) @@ -173,7 +173,7 @@ impl InMemoryGenerator { let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { ( predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), @@ -183,7 +183,7 @@ impl InMemoryGenerator { ) } else { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { predicate_range.clone() }, ) }; @@ -216,7 +216,7 @@ impl InMemoryGenerator { }; let predicate_filter = quote! { .filter(move |row| { - std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + core::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) }) }; @@ -231,7 +231,7 @@ impl InMemoryGenerator { #predicate_setup // Query construction is not an active read. Pin the grace // period on the first row lookup instead. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); #index_range .filter_map(#select_row) diff --git a/codegen/src/generators/in_memory/table/mod.rs b/codegen/src/generators/in_memory/table/mod.rs index d5bd9e39..4b63c514 100644 --- a/codegen/src/generators/in_memory/table/mod.rs +++ b/codegen/src/generators/in_memory/table/mod.rs @@ -18,6 +18,8 @@ impl InMemoryGenerator { let index_fns = self.gen_table_index_fns()?; let select_query_executor_impl = self.gen_table_select_query_executor_impl(); let column_range_type = self.gen_table_column_range_type(); + let columnar_methods = crate::generators::columnar::table_methods(&self.name, &self.columns); + let table_ident = WorktableNameGenerator::from_table_name(self.name.to_string()).get_work_table_ident(); Ok(quote! { #page_size_consts @@ -27,6 +29,9 @@ impl InMemoryGenerator { #index_fns #select_query_executor_impl #column_range_type + impl #table_ident { + #columnar_methods + } }) } @@ -126,7 +131,7 @@ impl InMemoryGenerator { #index_type, #lock_ident, <#primary_key_type as TablePrimaryKey>::Generator, - { INNER_PAGE_SIZE }, + { #inner_const_name }, #node_type > ); diff --git a/codegen/src/generators/in_memory/table/select_executor.rs b/codegen/src/generators/in_memory/table/select_executor.rs index 0dd2560b..92649ccb 100644 --- a/codegen/src/generators/in_memory/table/select_executor.rs +++ b/codegen/src/generators/in_memory/table/select_executor.rs @@ -44,7 +44,7 @@ impl InMemoryGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - #variant_ident(std::ops::#range_ident<#ty_ident>), + #variant_ident(core::ops::#range_ident<#ty_ident>), } }) .collect(); @@ -65,8 +65,8 @@ impl InMemoryGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - impl From> for #column_range_type { - fn from(range: std::ops::#range_ident<#ty_ident>) -> Self { + impl From> for #column_range_type { + fn from(range: core::ops::#range_ident<#ty_ident>) -> Self { Self::#variant_ident(range) } } @@ -92,6 +92,11 @@ impl InMemoryGenerator { pub fn gen_table_select_query_executor_impl(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let row_type = name_generator.get_row_type_ident(); + let default_dispatch = if cfg!(feature = "std") { + quote! { Some(worktable::runtime::dispatcher::<<#row_type as worktable::runtime::TableRuntime>::Backend> as worktable::runtime::Dispatch) } + } else { + quote! { None } + }; let column_range_type = name_generator.get_column_range_type_ident(); let row_fields_ident = name_generator.get_row_fields_enum_ident(); @@ -100,8 +105,8 @@ impl InMemoryGenerator { let col_ident = Ident::new(&column.to_string(), Span::call_site()); quote! { #row_fields_ident::#column_variant => { - let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(std::cmp::Ordering::Equal); - if cmp != std::cmp::Ordering::Equal { + let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(core::cmp::Ordering::Equal); + if cmp != core::cmp::Ordering::Equal { return match order { Order::Asc => cmp, Order::Desc => cmp.reverse(), @@ -165,12 +170,30 @@ impl InMemoryGenerator { _ => continue, } } - std::cmp::Ordering::Equal + core::cmp::Ordering::Equal }); iter = Box::new(items.into_iter()); }; quote! { + impl worktable::prelude::SelectQueryAsyncExecutor<#row_type> + for SelectQueryBuilder<#row_type, I, #column_range_type, #row_fields_ident> + where I: DoubleEndedIterator + Sized, + { + fn execute_async(self) -> worktable::prelude::SelectQueryFuture<#row_type> { + let mut params = self.params; + let dispatch = params.dispatch.take().or(#default_dispatch); + // Release all borrowed iterators and caller predicates before + // creating a task. No lifetime is extended across the pool. + let rows: Vec<#row_type> = self.iter.collect(); + Box::pin(async move { + let plan = SelectQueryBuilder { params, iter: rows.into_iter() }; + if let Some(dispatch) = dispatch { + worktable::runtime::run_owned(dispatch, move || plan.execute()).await? + } else { plan.execute() } + }) + } + } impl SelectQueryExecutor<#row_type, I, #column_range_type, #row_fields_ident> for SelectQueryBuilder<#row_type, I, #column_range_type, #row_fields_ident> where @@ -191,6 +214,7 @@ impl InMemoryGenerator { } fn execute(self) -> Result, WorkTableError> { + if self.params.dispatch.is_some() { return Err(WorkTableError::RuntimeRequiresAsync); } let mut iter: Box> = Box::new(self.iter); #range diff --git a/codegen/src/generators/in_memory/wrapper.rs b/codegen/src/generators/in_memory/wrapper.rs index 48c31da7..84563e42 100644 --- a/codegen/src/generators/in_memory/wrapper.rs +++ b/codegen/src/generators/in_memory/wrapper.rs @@ -24,7 +24,8 @@ impl InMemoryGenerator { let wrapper_ident = name_generator.get_wrapper_type_ident(); quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, rkyv::Serialize)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, worktable::prelude::rkyv::Serialize)] + #[rkyv(crate = worktable::prelude::rkyv)] #[repr(C)] pub struct #wrapper_ident { inner: #row_ident, diff --git a/codegen/src/generators/index_backend.rs b/codegen/src/generators/index_backend.rs index b56f6f26..99a54c9f 100644 --- a/codegen/src/generators/index_backend.rs +++ b/codegen/src/generators/index_backend.rs @@ -26,6 +26,10 @@ pub(crate) fn unique_index_type( Ok(quote! { UpstreamIndexMap<#key, #value> }) } } + IndexBackend::FxHash => Err(syn::Error::new_spanned( + key, + "`using fxhash` cannot back a paged table: it has no ordered scan and no persisted page form. Use `vec: true`, or an ordered backend.", + )), IndexBackend::Congee => Ok(quote! { CongeeIndex<#key, #value> }), IndexBackend::Arctic => Ok(quote! { ArcticIndex<#key, #value> }), } @@ -64,6 +68,9 @@ pub(crate) fn primary_key_backend_impl( fields: &[&TokenStream], ) -> syn::Result<(TokenStream, TokenStream)> { match backend { + IndexBackend::FxHash => { + unreachable!("`using fxhash` on a paged table is refused in `worktable/mod.rs` before any generator runs") + } IndexBackend::WorktablesIndex | IndexBackend::Indexset => Ok((quote! {}, quote! {})), IndexBackend::Congee => { let field = single_supported_field(backend, fields, supported_types(backend))?; @@ -92,6 +99,9 @@ pub(crate) fn primary_key_backend_impl( } } + // Std only, for the same reason as the generated `vacuum` + // method: the trait lives behind the persistence module. + worktable::__wt_if_std! { impl ArtPersistenceKey for #primary_key { const WIDTH: u8 = <#field as ArtPersistenceKey>::WIDTH; @@ -99,10 +109,11 @@ pub(crate) fn primary_key_backend_impl( self.0.encode_art_key(output) } - fn decode_art_key(bytes: &[u8]) -> eyre::Result { + fn decode_art_key(bytes: &[u8]) -> worktable::prelude::eyre::Result { Ok(Self(<#field as ArtPersistenceKey>::decode_art_key(bytes)?)) } } + } }, )) } @@ -125,6 +136,9 @@ pub(crate) fn primary_key_backend_impl( } } + // Std only, for the same reason as the generated `vacuum` + // method: the trait lives behind the persistence module. + worktable::__wt_if_std! { impl ArtPersistenceKey for #primary_key { const WIDTH: u8 = <#field as ArtPersistenceKey>::WIDTH; @@ -132,10 +146,11 @@ pub(crate) fn primary_key_backend_impl( self.0.encode_art_key(output) } - fn decode_art_key(bytes: &[u8]) -> eyre::Result { + fn decode_art_key(bytes: &[u8]) -> worktable::prelude::eyre::Result { Ok(Self(<#field as ArtPersistenceKey>::decode_art_key(bytes)?)) } } + } }, )) } @@ -177,7 +192,7 @@ fn single_supported_field<'a>( Ok(field) } -fn primitive_name(field: &TokenStream) -> Option { +pub(crate) fn primitive_name(field: &TokenStream) -> Option { let syn::Type::Path(type_path) = syn::parse2::(field.clone()).ok()? else { return None; }; diff --git a/codegen/src/generators/mod.rs b/codegen/src/generators/mod.rs index 643fe56d..1b0fe791 100644 --- a/codegen/src/generators/mod.rs +++ b/codegen/src/generators/mod.rs @@ -1,6 +1,12 @@ +pub(crate) mod columnar; +pub(crate) mod dense_table; pub mod in_memory; pub(crate) mod index_backend; pub mod partitions; pub mod persist; pub(crate) mod primary_key; pub mod read_only; +pub(crate) mod runtime_backend; +pub mod vec_table; + +pub(crate) mod profile_dispatch; diff --git a/codegen/src/generators/partitions.rs b/codegen/src/generators/partitions.rs index 5cf07732..805e0de4 100644 --- a/codegen/src/generators/partitions.rs +++ b/codegen/src/generators/partitions.rs @@ -1,7 +1,8 @@ use proc_macro2::{Ident, TokenStream}; use quote::{format_ident, quote}; -use crate::common::model::{PartitionKey, Persistence}; +use crate::common::model::{Columns, PartitionKey, Persistence}; +use crate::generators::dense_table; /// Generate the router for a partitioned table. /// @@ -9,8 +10,51 @@ use crate::common::model::{PartitionKey, Persistence}; /// `worktable::partition::PartitionSet`, so the code emitted per partitioned /// table stays small: one `worktable!` already expands to roughly 1,940 lines, /// and a router that grew with it would be paid for by every table. -pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> TokenStream { - let table = format_ident!("{}WorkTable", name); +/// +/// # Which table a partition holds +/// +/// `partition_max_size` decides it, and `columns` is here so this can ask. A +/// narrow width (`bool`, `u8`, `u16`) means the rows fit a position-addressed +/// table with no index at all, and that is generated beside the router and used +/// as the payload. A wide one (`u32`, `u64`) keeps the full generated table, +/// which is what every partitioned table had before the width was declarable. +/// +/// The router itself does not change between the two. It names its payload +/// once and calls `Default::default`, `used_bytes` and `row_count` on it, and +/// both shapes have all three. +pub fn expand( + name: &Ident, + key: &PartitionKey, + persistence: Persistence, + columns: &Columns, + queries: &dense_table::DenseQueries, +) -> syn::Result { + // A dense partition has no persistence engine, no pages and no CDC, so a + // persisted declaration asking for one would be told yes and given a table + // that never writes anything. Refused rather than silently downgraded. + if key.max_size.is_dense() && persistence.is_persisted() { + return Err(syn::Error::new( + name.span(), + format!( + "`partition_max_size: {}` generates a partition with no pages, no index and no \ + persistence engine, so `persist: true` cannot be honoured for it. Use \ + `partition_max_size: u64`, which keeps the full table and persists, or drop \ + `persist`.", + key.max_size.type_name() + ), + )); + } + + let dense = if key.max_size.is_dense() { + Some(dense_table::expand(name, columns, key.max_size, queries)?) + } else { + None + }; + let table = if key.max_size.is_dense() { + dense_table::type_ident(name) + } else { + format_ident!("{}WorkTable", name) + }; let partitions = format_ident!("{}Partitions", name); let pinned = format_ident!("{}Pinned", name); let key_name = &key.name; @@ -38,7 +82,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok pub fn partition_or_create( &self, #key_name: #key_ty, - ) -> Result, worktable::partition::PartitionError> { + ) -> Result, worktable::partition::PartitionError> { self.inner.get_or_create(#key_name as u64, <#table as Default>::default) } } @@ -49,7 +93,9 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok See `{partitions}::pinned`." ); - quote! { + Ok(quote! { + #dense + #[doc = #pinned_doc] pub struct #pinned<'a> { inner: worktable::partition::Pinned<'a, #table>, @@ -82,7 +128,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok /// The partition routed to by `#key_name`, if it exists. #[inline] - pub fn partition(&self, #key_name: #key_ty) -> Option> { + pub fn partition(&self, #key_name: #key_ty) -> Option> { self.inner.partition(#key_name as u64) } @@ -94,7 +140,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok &self, #key_name: #key_ty, make: F, - ) -> Result, worktable::partition::PartitionError> + ) -> Result, worktable::partition::PartitionError> where F: FnOnce() -> #table, { @@ -146,7 +192,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok /// grace period). Removal and creation reclaim opportunistically, /// so a router shared behind an `Arc` does not accumulate removed /// partitions; `collect` is available for removal-only phases. - pub fn remove(&self, #key_name: #key_ty) -> Option> { + pub fn remove(&self, #key_name: #key_ty) -> Option> { self.inner.remove(#key_name as u64) } @@ -156,7 +202,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok } /// Every live partition with its key. - pub fn iter(&self) -> Vec<(#key_ty, std::sync::Arc<#table>)> { + pub fn iter(&self) -> Vec<(#key_ty, worktable::prelude::Arc<#table>)> { self.inner .iter() .into_iter() @@ -232,5 +278,5 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok out } } - } + }) } diff --git a/codegen/src/generators/persist/index/cdc.rs b/codegen/src/generators/persist/index/cdc.rs index 8aee8d6a..e9c037b8 100644 --- a/codegen/src/generators/persist/index/cdc.rs +++ b/codegen/src/generators/persist/index/cdc.rs @@ -76,6 +76,7 @@ impl PersistGenerator { }) .collect::>(); let idents = self.columns.indexes.values().map(|idx| &idx.name).collect::>(); + let columnar_save = crate::generators::columnar::save_row_cdc(&self.columns); quote! { fn save_row_cdc(&self, row: #row_type_ident, link: Link) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { @@ -83,6 +84,7 @@ impl PersistGenerator { let mut partial_events = #events_ident::default(); #(#save_rows)* + #columnar_save (#events_ident { #(#idents,)* }, Ok(())) @@ -170,6 +172,7 @@ impl PersistGenerator { }) .unzip(); let idents = self.columns.indexes.values().map(|idx| &idx.name).collect::>(); + let columnar_reinsert = crate::generators::columnar::reinsert_row_cdc(&self.columns); quote! { fn reinsert_row_cdc( @@ -184,6 +187,7 @@ impl PersistGenerator { #(#insert_rows)* #(#remove_rows)* + #columnar_reinsert (#events_ident { #(#idents,)* }, Ok(())) @@ -208,6 +212,11 @@ impl PersistGenerator { } else { quote! { row.#i } }; + let key = if self.columns.columnar_fields.is_empty() { + key + } else { + quote! { #key.clone() } + }; quote! { let (_, events) = TableIndexCdc::remove_cdc(&self.#index_field_name, #key, link); let #index_field_name = events.into_iter().map(|ev| ev.into()).collect(); @@ -215,10 +224,12 @@ impl PersistGenerator { }) .collect::>(); let idents = self.columns.indexes.values().map(|idx| &idx.name).collect::>(); + let columnar_delete = crate::generators::columnar::delete_row(&self.columns); quote! { fn delete_row_cdc(&self, row: #row_type_ident, link: Link) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { #(#delete_rows)* + #columnar_delete (#events_ident { #(#idents,)* }, Ok(())) @@ -333,14 +344,16 @@ impl PersistGenerator { } }); let idents = self.columns.indexes.values().map(|idx| &idx.name).collect::>(); + let columnar_dirty = crate::generators::columnar::mark_dirty(&self.columns); quote! { fn process_difference_remove_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { #(#process_difference_rows)* + #columnar_dirty (#events_ident { #(#idents,)* }, Ok(())) @@ -403,17 +416,19 @@ impl PersistGenerator { } }); let idents = self.columns.indexes.values().map(|idx| &idx.name).collect::>(); + let columnar_dirty = crate::generators::columnar::mark_dirty(&self.columns); quote! { fn process_difference_insert_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; let mut partial_events = #events_ident::default(); #(#process_difference_insert_rows)* + #columnar_dirty (#events_ident { #(#idents,)* }, Ok(())) diff --git a/codegen/src/generators/persist/index/info.rs b/codegen/src/generators/persist/index/info.rs index bfe4b970..8ea46a84 100644 --- a/codegen/src/generators/persist/index/info.rs +++ b/codegen/src/generators/persist/index/info.rs @@ -27,6 +27,9 @@ impl PersistGenerator { if idx.is_unique { let (capacity, node_count) = match idx.backend { + crate::common::model::IndexBackend::FxHash => unreachable!( + "`using fxhash` on a paged table is refused in `worktable/mod.rs` before any generator runs" + ), crate::common::model::IndexBackend::WorktablesIndex | crate::common::model::IndexBackend::Indexset => ( quote! { self.#index_field_name.capacity() }, diff --git a/codegen/src/generators/persist/index/mod.rs b/codegen/src/generators/persist/index/mod.rs index c32b314e..442904d1 100644 --- a/codegen/src/generators/persist/index/mod.rs +++ b/codegen/src/generators/persist/index/mod.rs @@ -11,6 +11,7 @@ use quote::quote; impl PersistGenerator { pub fn gen_index_def(&mut self) -> syn::Result { + let columnar_def = crate::generators::columnar::definitions(&self.name, &self.columns); let type_def = self.gen_type_def()?; let impl_def = self.gen_secondary_index_impl_def(); let info_def = self.gen_secondary_index_info_impl_def(); @@ -19,6 +20,7 @@ impl PersistGenerator { let available_indexes = self.gen_available_indexes(); Ok(quote! { + #columnar_def #type_def #impl_def #info_def @@ -80,11 +82,13 @@ impl PersistGenerator { let derive = quote! { #[derive(Debug, MemStat, PersistIndex)] }; + let columnar_field = crate::generators::columnar::index_struct_field(&self.name, &self.columns, true); Ok(quote! { #derive pub struct #ident { - #(#index_rows),* + #(#index_rows,)* + #columnar_field } }) } @@ -92,7 +96,7 @@ impl PersistGenerator { fn gen_index_default_impl(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let index_type_ident = name_generator.get_index_type_ident(); - let const_name = name_generator.get_page_inner_size_const_ident(); + let const_name = name_generator.get_disk_page_capacity(); let index_rows = self .columns @@ -115,6 +119,7 @@ impl PersistGenerator { #[allow(clippy::collapsible_else_if)] let res = if idx.is_unique { match idx.backend { + crate::common::model::IndexBackend::FxHash => unreachable!("`using fxhash` on a paged table is refused in `worktable/mod.rs` before any generator runs"), crate::common::model::IndexBackend::WorktablesIndex => { let map = if cfg!(feature = "logical-index-persistence") { quote! { PersistentWtiIndex } @@ -155,12 +160,14 @@ impl PersistGenerator { Ok::<_, syn::Error>(res) }) .collect::, syn::Error>>()?; + let columnar_field = crate::generators::columnar::index_default_field(&self.columns); Ok(quote! { impl Default for #index_type_ident { fn default() -> Self { Self { #(#index_rows)* + #columnar_field } } } @@ -185,11 +192,22 @@ impl PersistGenerator { } } else { quote! { - #[derive(Debug, Clone, Copy, MoreDisplay, PartialEq, PartialOrd, Ord, Hash, Eq)] + #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Ord, Hash, Eq)] pub enum #avt_type_ident { #(#indexes)* } + // Delegated to `Debug` rather than derived. Every variant here + // is fieldless, so `Debug` prints exactly the variant name, + // which is what `derive_more::Display` produced. Deriving it + // put `::derive_more::` paths in the expansion and so put that + // crate into this macro's contract. + impl core::fmt::Display for #avt_type_ident { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Debug::fmt(self, f) + } + } + impl AvailableIndex for #avt_type_ident { fn to_string_value(&self) -> String { ToString::to_string(&self) diff --git a/codegen/src/generators/persist/index/usual.rs b/codegen/src/generators/persist/index/usual.rs index e8629cc8..00e1472c 100644 --- a/codegen/src/generators/persist/index/usual.rs +++ b/codegen/src/generators/persist/index/usual.rs @@ -14,6 +14,7 @@ impl PersistGenerator { let avt_index_ident = name_generator.get_available_indexes_ident(); let save_row_fn = self.gen_save_row_index_fn(); + let publication_guard = crate::generators::columnar::publication_guard(&self.columns); let reinsert_row_fn = self.gen_reinsert_row_index_fn(); let delete_row_fn = self.gen_delete_row_index_fn(); let process_difference_insert_fn = self.gen_process_difference_insert_index_fn(); @@ -23,6 +24,7 @@ impl PersistGenerator { quote! { impl TableSecondaryIndex<#row_type_ident, #avt_type_ident, #avt_index_ident> for #index_type_ident { #save_row_fn + #publication_guard #reinsert_row_fn #delete_row_fn #process_difference_insert_fn @@ -69,11 +71,13 @@ impl PersistGenerator { } }) .collect::>(); + let columnar_save = crate::generators::columnar::save_row(&self.columns); quote! { fn save_row(&self, row: #row_type_ident, link: Link) -> core::result::Result<(), IndexError<#available_index_ident>> { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; #(#save_rows)* + #columnar_save core::result::Result::Ok(()) } } @@ -147,6 +151,7 @@ impl PersistGenerator { (insert, remove) }) .unzip(); + let columnar_reinsert = crate::generators::columnar::reinsert_row(&self.columns); quote! { fn reinsert_row(&self, @@ -159,6 +164,7 @@ impl PersistGenerator { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; #(#insert_rows)* #(#remove_rows)* + #columnar_reinsert core::result::Result::Ok(()) } } @@ -189,10 +195,12 @@ impl PersistGenerator { } }) .collect::>(); + let columnar_delete = crate::generators::columnar::delete_row(&self.columns); quote! { fn delete_row(&self, row: #row_type_ident, link: Link) -> core::result::Result<(), IndexError<#available_index_ident>> { #(#delete_rows)* + #columnar_delete core::result::Result::Ok(()) } } @@ -231,14 +239,16 @@ impl PersistGenerator { quote! {} } }); + let columnar_dirty = crate::generators::columnar::mark_dirty(&self.columns); quote! { fn process_difference_remove( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { #(#process_difference_remove_rows)* + #columnar_dirty core::result::Result::Ok(()) } } @@ -288,15 +298,17 @@ impl PersistGenerator { quote! {} } }); + let columnar_dirty = crate::generators::columnar::mark_dirty(&self.columns); quote! { fn process_difference_insert( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { let mut inserted_indexes: Vec<#avt_index_ident> = vec![]; #(#process_difference_insert_rows)* + #columnar_dirty core::result::Result::Ok(()) } } diff --git a/codegen/src/generators/persist/locks.rs b/codegen/src/generators/persist/locks.rs index 92a88a0c..86b55b60 100644 --- a/codegen/src/generators/persist/locks.rs +++ b/codegen/src/generators/persist/locks.rs @@ -24,7 +24,7 @@ impl PersistGenerator { .keys() .map(|i| { let name = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); - quote! { #name: Option>, } + quote! { #name: Option>, } }) .collect(); @@ -124,8 +124,8 @@ impl PersistGenerator { .collect(); quote! { - fn with_lock(id: u16) -> (Self, std::sync::Arc) { - let lock = std::sync::Arc::new(Lock::new(id)); + fn with_lock(id: u16) -> (Self, worktable::prelude::Arc) { + let lock = worktable::prelude::Arc::new(Lock::new(id)); ( Self { #(#rows),* @@ -154,9 +154,9 @@ impl PersistGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let lock = std::sync::Arc::new(Lock::new(id)); + fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* (set, lock) @@ -186,8 +186,8 @@ impl PersistGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> std::collections::HashSet> { - let mut set = std::collections::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { + let mut set = worktable::prelude::HashSet::new(); #(#rows)* set } diff --git a/codegen/src/generators/persist/primary_key.rs b/codegen/src/generators/persist/primary_key.rs index 59b002b9..5d1b4098 100644 --- a/codegen/src/generators/persist/primary_key.rs +++ b/codegen/src/generators/persist/primary_key.rs @@ -66,19 +66,68 @@ impl PersistGenerator { primary_key_backend_impl(self.columns.primary_index_backend, &ident, types)?; let borrowed_impl = gen_borrowed_primary_key_impl(&ident, types); + // `From` written out rather than derived. `derive_more::From` puts + // `::derive_more::` paths in its expansion, which made that crate part + // of this macro's contract: a consumer who never wrote `derive_more` + // had to declare it anyway. A newtype conversion is three lines. + // + // A composite key converts from the tuple, which is the shape + // `derive_more` produced for a multi-field tuple struct. + let from_impl = if types.len() == 1 { + let ty = &types[0]; + quote! { + impl From<#ty> for #ident { + fn from(value: #ty) -> Self { + Self(value) + } + } + } + } else { + let binding: Vec<_> = (0..types.len()) + .map(|i| syn::Ident::new(&format!("field{i}"), proc_macro2::Span::mixed_site())) + .collect(); + quote! { + impl From<(#(#types),*)> for #ident { + fn from((#(#binding),*): (#(#types),*)) -> Self { + Self(#(#binding),*) + } + } + } + }; + + // And the reverse direction, which was `derive_more::Into`. Same + // reasoning: it is one impl, and deriving it dragged the crate in. + let into_impl = if types.len() == 1 { + let ty = &types[0]; + quote! { + impl From<#ident> for #ty { + fn from(value: #ident) -> Self { + value.0 + } + } + } + } else { + let index: Vec<_> = (0..types.len()).map(syn::Index::from).collect(); + quote! { + impl From<#ident> for (#(#types),*) { + fn from(value: #ident) -> Self { + (#(value.#index),*) + } + } + } + }; + Ok(quote! { #[derive( Clone, #backend_derive - rkyv::Archive, + worktable::prelude::rkyv::Archive, Debug, Default, - rkyv::Deserialize, + worktable::prelude::rkyv::Deserialize, Hash, - rkyv::Serialize, - From, + worktable::prelude::rkyv::Serialize, Eq, - Into, PartialEq, PartialOrd, Ord, @@ -86,9 +135,13 @@ impl PersistGenerator { MemStat, #unsized_derive )] + #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(PartialEq, Eq, PartialOrd, Ord, Debug))] pub struct #ident(#(#types),*); + #from_impl + #into_impl + #borrowed_impl #backend_impl }) @@ -133,14 +186,14 @@ impl PersistGenerator { fn get_generator_from_type(type_: &TokenStream, i: &Ident) -> syn::Result { Ok(match type_.to_string().as_str() { - "u8" => quote! { std::sync::atomic::AtomicU8 }, - "u16" => quote! { std::sync::atomic::AtomicU16 }, - "u32" => quote! { std::sync::atomic::AtomicU32 }, - "u64" => quote! { std::sync::atomic::AtomicU64 }, - "i8" => quote! { std::sync::atomic::AtomicI8 }, - "i16" => quote! { std::sync::atomic::AtomicI16 }, - "i32" => quote! { std::sync::atomic::AtomicI32 }, - "i64" => quote! { std::sync::atomic::AtomicI64 }, + "u8" => quote! { core::sync::atomic::AtomicU8 }, + "u16" => quote! { core::sync::atomic::AtomicU16 }, + "u32" => quote! { core::sync::atomic::AtomicU32 }, + "u64" => quote! { core::sync::atomic::AtomicU64 }, + "i8" => quote! { core::sync::atomic::AtomicI8 }, + "i16" => quote! { core::sync::atomic::AtomicI16 }, + "i32" => quote! { core::sync::atomic::AtomicI32 }, + "i64" => quote! { core::sync::atomic::AtomicI64 }, // The accepted set is `worktable_dsl::AUTOINCREMENT_TYPES`, and the // arms above must stay equal to it. `check` uses that list to // answer "would the macro accept this", so a second copy drifting diff --git a/codegen/src/generators/persist/queries/delete.rs b/codegen/src/generators/persist/queries/delete.rs index c2efa53c..e57342f9 100644 --- a/codegen/src/generators/persist/queries/delete.rs +++ b/codegen/src/generators/persist/queries/delete.rs @@ -14,7 +14,13 @@ impl PersistGenerator { let table_ident = name_generator.get_work_table_ident(); let custom_deletes = if let Some(q) = &self.queries { + let profile = q.delete_runtime.clone(); let custom_deletes = self.gen_custom_deletes(q.deletes.clone()); + let custom_deletes = crate::generators::profile_dispatch::wrap( + custom_deletes, + profile.as_ref(), + &name_generator.get_row_type_ident(), + )?; quote! { #custom_deletes } @@ -38,6 +44,7 @@ impl PersistGenerator { let pk_ident = name_generator.get_primary_key_type_ident(); let delete_logic = self.gen_delete_logic(true); let full_row_lock = self.gen_full_lock_for_update(); + let publication = crate::generators::columnar::table_publication_guard(&self.columns); quote! { pub async fn delete(&self, pk: Pk) -> core::result::Result<(), WorkTableError> @@ -46,6 +53,7 @@ impl PersistGenerator { let pk: #pk_ident = pk.into(); let pending_lock = { #full_row_lock }; let _guard = pending_lock.into_guard_with_mutation(); + #publication #delete_logic @@ -58,6 +66,7 @@ impl PersistGenerator { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let pk_ident = name_generator.get_primary_key_type_ident(); let delete_logic = self.gen_delete_logic(false); + let publication = crate::generators::columnar::table_publication_guard(&self.columns); quote! { pub async fn delete_without_lock(&self, pk: Pk) -> core::result::Result<(), WorkTableError> @@ -65,6 +74,7 @@ impl PersistGenerator { { let pk: #pk_ident = pk.into(); let _mutation_guard = self.0.lock_manager.mutation_guard(&pk); + #publication #delete_logic core::result::Result::Ok(()) } @@ -98,7 +108,23 @@ impl PersistGenerator { row, link, ); - res?; + // `delete_row_cdc` produces events whether or not it succeeds, and + // the index has already assigned their ids. Propagating the error + // without queueing them leaves a hole the persistence stream can + // never fill, exactly as the restore path below is careful not to. + if let core::result::Result::Err(e) = res { + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events, + }); + self.1.apply_operation(ack_op)?; + return core::result::Result::Err(e.into()); + } let (_, primary_key_events) = self.0.primary_index.remove_cdc(pk.clone(), link); if let core::result::Result::Err(e) = self.0.data.delete(link) { let mut secondary_keys_events = secondary_keys_events; @@ -133,7 +159,7 @@ impl PersistGenerator { #pk_ident, #secondary_events_ident > = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(uuid::Uuid::now_v7()), + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), primary_key_events, secondary_keys_events, }); @@ -145,7 +171,7 @@ impl PersistGenerator { #pk_ident, #secondary_events_ident > = Operation::Delete(DeleteOperation { - id: uuid::Uuid::now_v7().into(), + id: worktable::prelude::uuid::Uuid::now_v7().into(), secondary_keys_events, primary_key_events, link, @@ -219,7 +245,7 @@ impl PersistGenerator { quote! { pub async fn #name(&self, by: #type_) -> core::result::Result<(), WorkTableError> { let _bulk_mutation = self.0.lock_manager.bulk_mutation_guard(); - let pks = std::cell::RefCell::new(Vec::new()); + let pks = core::cell::RefCell::new(Vec::new()); self.iter_with(|row| { if row.#field == by { pks.borrow_mut().push(row.get_primary_key()); @@ -387,5 +413,21 @@ mod tests { emitted.contains("Operation :: Acknowledge"), "acknowledge op missing:\n{emitted}" ); + + // A failed secondary removal used to propagate through a bare `res?`, + // dropping the events `delete_row_cdc` had already produced. Their ids + // are assigned when the index produces them, so the persistence stream + // gapped permanently and the stall named a range rather than a cause. + let secondary = emitted.find("delete_row_cdc").expect("secondary removal emitted"); + let tail = &emitted[secondary..]; + let ack = tail + .find("Operation :: Acknowledge") + .expect("a failed secondary removal must acknowledge its events"); + assert!( + ack < tail + .find("remove_cdc (pk . clone () , link)") + .expect("primary removal emitted"), + "the secondary removal propagates before acknowledging, which gaps the stream:\n{emitted}" + ); } } diff --git a/codegen/src/generators/persist/queries/in_place.rs b/codegen/src/generators/persist/queries/in_place.rs index 895ef11b..4b6d6289 100644 --- a/codegen/src/generators/persist/queries/in_place.rs +++ b/codegen/src/generators/persist/queries/in_place.rs @@ -13,7 +13,13 @@ impl PersistGenerator { let table_ident = name_generator.get_work_table_ident(); let custom_in_place = if let Some(q) = &self.queries { + let profile = q.in_place_runtime.clone(); let custom_in_place = self.gen_in_place_queries(q.in_place.clone()); + let custom_in_place = crate::generators::profile_dispatch::wrap( + custom_in_place, + profile.as_ref(), + &name_generator.get_row_type_ident(), + )?; quote! { #custom_in_place } @@ -69,12 +75,12 @@ impl PersistGenerator { let column_types = if types.len() == 1 { let t = types[0]; quote! { - &mut <#t as rkyv::Archive>::Archived + &mut <#t as worktable::prelude::rkyv::Archive>::Archived } } else { let types = types.iter().map(|t| { quote! { - &mut <#t as rkyv::Archive>::Archived + &mut <#t as worktable::prelude::rkyv::Archive>::Archived } }); quote! { @@ -97,13 +103,14 @@ impl PersistGenerator { } }; let custom_lock = self.gen_custom_lock_for_update(lock_ident); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); quote! { pub async fn #method_ident( &self, mut f: F, by: Pk, - ) -> eyre::Result<()> + ) -> worktable::prelude::eyre::Result<()> where #pk_type: From { let pk: #pk_type = by.into(); @@ -128,13 +135,14 @@ impl PersistGenerator { // reverted on restart. In-place queries cannot touch indexed // columns (rejected at parse time), so the event vectors stay // empty. - let op_id = OperationId::Single(uuid::Uuid::now_v7()); + let op_id = OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()); let secondary_keys_events: #secondary_events_ident = core::default::Default::default(); let mut op: Operation< <<#pk_type as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, #pk_type, #secondary_events_ident > = Operation::Update(UpdateOperation { + retired_link: None, id: op_id, primary_key_events: vec![], secondary_keys_events, @@ -148,6 +156,8 @@ impl PersistGenerator { }; self.1.apply_operation(op)?; + #columnar_dirty + Ok(()) } } diff --git a/codegen/src/generators/persist/queries/locks.rs b/codegen/src/generators/persist/queries/locks.rs index c6f0c3ee..6739cd86 100644 --- a/codegen/src/generators/persist/queries/locks.rs +++ b/codegen/src/generators/persist/queries/locks.rs @@ -95,9 +95,9 @@ impl PersistGenerator { quote! { #[allow(clippy::mutable_key_type)] - pub fn #ident(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let new_lock = std::sync::Arc::new(Lock::new(id)); + pub fn #ident(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let new_lock = worktable::prelude::Arc::new(Lock::new(id)); #(#inner)* (set, new_lock) } @@ -124,7 +124,7 @@ impl PersistGenerator { // timeout, task abort) would otherwise leave the registered lock // held forever and hang every later operation on this key. let pending_lock = PendingLock::new(op_lock, self.0.lock_manager.clone(), pk.clone()); - futures::future::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; pending_lock } } @@ -153,7 +153,7 @@ impl PersistGenerator { // timeout, task abort) would otherwise leave the registered lock // held forever and hang every later operation on this key. let pending_lock = PendingLock::new(op_lock, self.0.lock_manager.clone(), pk.clone()); - futures::future::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; + worktable::prelude::join_all(locks.iter().map(|l| l.wait()).collect::>()).await; pending_lock } } diff --git a/codegen/src/generators/persist/queries/select.rs b/codegen/src/generators/persist/queries/select.rs index 7627a6fb..581c4fd6 100644 --- a/codegen/src/generators/persist/queries/select.rs +++ b/codegen/src/generators/persist/queries/select.rs @@ -32,7 +32,7 @@ impl PersistGenerator { // Acquire the grace-period guard only when iteration starts. // Merely constructing and retaining a query builder must not // stall retired-link reclamation. - let iter = std::iter::once_with(move || { + let iter = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .iter_values() @@ -55,7 +55,7 @@ impl PersistGenerator { return None; } current_link = replacement; - std::hint::spin_loop(); + core::hint::spin_loop(); } None }) diff --git a/codegen/src/generators/persist/queries/type.rs b/codegen/src/generators/persist/queries/type.rs index 0080bfaf..55987206 100644 --- a/codegen/src/generators/persist/queries/type.rs +++ b/codegen/src/generators/persist/queries/type.rs @@ -46,20 +46,38 @@ impl PersistGenerator { .expect("should be valid because parsed from declaration"); let type_upper = map_to_uppercase(s); let type_upper = Ident::new(type_upper.as_str(), Span::mixed_site()); - Some(quote! { - #[from] - #type_upper(#type_ident), - }) + Some(( + quote! { + #type_upper(#type_ident), + }, + // Written out rather than derived. `derive_more::From` + // generates `::derive_more::` paths inside its expansion, + // which makes that crate part of this macro's contract: + // a consumer who never wrote `derive_more` still had to + // declare it to compile a table. One newtype variant per + // type is a two-line impl, so the dependency bought + // nothing that could not be spelled here. + quote! { + impl From<#type_ident> for #avt_type_ident { + fn from(value: #type_ident) -> Self { + Self::#type_upper(value) + } + } + }, + )) }) .collect(); + let (rows, from_impls): (Vec<_>, Vec<_>) = rows.into_iter().flatten().unzip(); if !rows.is_empty() { Ok(quote! { - #[derive(Clone, Debug, From, PartialEq)] + #[derive(Clone, Debug, PartialEq)] #[non_exhaustive] pub enum #avt_type_ident { #(#rows)* } + + #(#from_impls)* }) } else { Ok(quote! { @@ -122,7 +140,8 @@ impl PersistGenerator { Ok::<_, syn::Error>(quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, Clone, rkyv::Serialize)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, Clone, worktable::prelude::rkyv::Serialize)] + #[rkyv(crate = worktable::prelude::rkyv)] #[repr(C)] pub struct #ident { #(#rows)* diff --git a/codegen/src/generators/persist/queries/update.rs b/codegen/src/generators/persist/queries/update.rs index ffd41658..ff379424 100644 --- a/codegen/src/generators/persist/queries/update.rs +++ b/codegen/src/generators/persist/queries/update.rs @@ -10,7 +10,13 @@ use quote::quote; impl PersistGenerator { pub fn gen_query_update_impl(&mut self) -> syn::Result { let custom_updates = if let Some(q) = &self.queries { + let profile = q.update_runtime.clone(); let custom_updates = self.gen_custom_updates(q.updates.clone()); + let custom_updates = crate::generators::profile_dispatch::wrap( + custom_updates, + profile.as_ref(), + &WorktableNameGenerator::from_table_name(self.name.to_string()).get_row_type_ident(), + )?; quote! { #custom_updates @@ -42,7 +48,7 @@ impl PersistGenerator { .keys() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -59,6 +65,7 @@ impl PersistGenerator { let persist_call = self.gen_persist_call(); let persist_op = self.gen_persist_op(); let full_row_lock = self.gen_full_lock_for_update(); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); let const_name = name_generator.get_page_inner_size_const_ident(); let secondary_events_ident = name_generator.get_space_secondary_index_events_ident(); // A full-row update rewrites every column, hence every secondary @@ -84,13 +91,15 @@ impl PersistGenerator { #pk_ident, #secondary_events_ident > = Operation::Update(UpdateOperation { - id: OperationId::Single(uuid::Uuid::now_v7()), + retired_link: None, + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), primary_key_events: vec![], secondary_keys_events, bytes: self.0.data.select_raw(link)?, link, }); self.1.apply_operation(op)?; + #columnar_dirty return core::result::Result::Ok(()); } } @@ -126,15 +135,16 @@ impl PersistGenerator { // diverging size_check block. let update_body = if self.columns.is_sized { quote! { - let mut bytes = rkyv::to_bytes::(&row).map_err(|_| WorkTableError::SerializeError)?; - let mut archived_row = unsafe { rkyv::access_unchecked_mut::<<#row_ident as rkyv::Archive>::Archived>(&mut bytes[..]).unseal_unchecked() }; + let mut bytes = worktable::prelude::rkyv::to_bytes::(&row).map_err(|_| WorkTableError::SerializeError)?; + let mut archived_row = unsafe { worktable::prelude::rkyv::access_unchecked_mut::<<#row_ident as worktable::prelude::rkyv::Archive>::Archived>(&mut bytes[..]).unseal_unchecked() }; - let op_id = OperationId::Single(uuid::Uuid::now_v7()); + let op_id = OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()); #diff_process_insert #data_write #persist_op #diff_process_remove + #columnar_dirty #persist_call @@ -311,8 +321,8 @@ impl PersistGenerator { // compensation above). let mut merged_events = secondary_keys_events; if row_holds_old_values { - let mut reversed_diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = - std::collections::HashMap::new(); + let mut reversed_diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = + worktable::prelude::HashMap::new(); for (key, diff) in diffs { reversed_diffs.insert(key, Difference { old: diff.new, new: diff.old }); } @@ -321,7 +331,7 @@ impl PersistGenerator { merged_events.extend(rollback_events); } let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(uuid::Uuid::now_v7()), + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), primary_key_events: vec![], secondary_keys_events: merged_events, }); @@ -433,7 +443,8 @@ impl PersistGenerator { #primary_key_ident, #secondary_events_ident > = Operation::Update(UpdateOperation { - id: OperationId::Single(uuid::Uuid::now_v7()), + retired_link: None, + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), primary_key_events: vec![], secondary_keys_events, bytes: self.0.data.select_raw(current_link)?, @@ -489,6 +500,7 @@ impl PersistGenerator { #primary_key_ident, #secondary_events_ident > = Operation::Update(UpdateOperation { + retired_link: None, id: op_id, primary_key_events: vec![], secondary_keys_events, @@ -501,13 +513,14 @@ impl PersistGenerator { fn gen_process_diffs_insert_on_index(&self, idents: &[Ident], idx_idents: Option<&Vec>) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let avt_type_ident = name_generator.get_available_type_ident(); + let pk_ident = name_generator.get_primary_key_type_ident(); // `updated_bytes` is bound by gen_data_write_and_fetch, which captures // the real row bytes right after the data write. let diff_container = if idx_idents.is_some() { quote! { let row_old = self.0.data.select_non_ghosted(link)?; let row_new = row.clone(); - let mut diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = std::collections::HashMap::new(); + let mut diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = worktable::prelude::HashMap::new(); } } else { quote! {} @@ -559,7 +572,7 @@ impl PersistGenerator { merged_events.extend(rollback_secondary_events); let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(uuid::Uuid::now_v7()), + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), primary_key_events: vec![], secondary_keys_events: merged_events, }); @@ -567,7 +580,53 @@ impl PersistGenerator { Err(WorkTableError::AlreadyExists(at.to_string_value())) } - IndexError::NotFound => Err(WorkTableError::NotFound), + IndexError::ColumnSlotIdExhausted { + bits, + inserted_already, + } => { + let (rollback_secondary_events, _): (#secondary_events_ident, _) = self.0.indexes.delete_from_indexes_cdc( + row_new.merge(row_old.clone()), + link, + inserted_already + ); + + let mut merged_events = secondary_events.clone(); + merged_events.extend(rollback_secondary_events); + + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events: merged_events, + }); + self.1.apply_operation(ack_op)?; + + Err(WorkTableError::ColumnSlotIdExhausted(bits)) + } + IndexError::NotFound => { + // The insert side produced events before it + // failed, and the index has already assigned + // their ids. Returning without queueing them + // leaves a hole the persistence stream can + // never fill, which is what the sibling arm + // above avoids and what this arm used to + // cause. + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events: secondary_events.clone(), + }); + self.1.apply_operation(ack_op)?; + + Err(WorkTableError::NotFound) + } }; } let mut secondary_keys_events = secondary_events; @@ -587,10 +646,29 @@ impl PersistGenerator { } fn gen_process_diffs_remove_on_index(&self, idx_idents: Option<&Vec>) -> TokenStream { + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let pk_ident = name_generator.get_primary_key_type_ident(); + let secondary_events_ident = name_generator.get_space_secondary_index_events_ident(); if idx_idents.is_some() { quote! { let (secondary_keys_events_remove, res) = self.0.indexes.process_difference_remove_cdc(link, diffs); - res?; + // The removal produced events whether or not it succeeded, and + // their ids are already assigned. Propagating the error without + // queueing them gaps the stream permanently, so acknowledge + // them first and then propagate unchanged. + if let core::result::Result::Err(e) = res { + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events: secondary_keys_events_remove, + }); + self.1.apply_operation(ack_op)?; + return core::result::Result::Err(e.into()); + } op.extend_secondary_key_events(secondary_keys_events_remove); } } else { @@ -615,7 +693,7 @@ impl PersistGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -632,14 +710,16 @@ impl PersistGenerator { let custom_lock = self.gen_custom_lock_for_update(lock_ident); let data_write = self.gen_data_write_and_fetch(&row_updates, idx_idents); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); let finish_update = if archived_swap_is_safe { quote! { - let op_id = OperationId::Single(uuid::Uuid::now_v7()); + let op_id = OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()); #diff_process_insert #data_write #persist_op #diff_process_remove + #columnar_dirty #persist_call @@ -664,8 +744,8 @@ impl PersistGenerator { .map(Into::into) .ok_or(WorkTableError::NotFound)?; - let mut bytes = rkyv::to_bytes::(&row).map_err(|_| WorkTableError::SerializeError)?; - let mut archived_row = unsafe { rkyv::access_unchecked_mut::<<#query_ident as rkyv::Archive>::Archived>(&mut bytes[..]).unseal_unchecked() }; + let mut bytes = worktable::prelude::rkyv::to_bytes::(&row).map_err(|_| WorkTableError::SerializeError)?; + let mut archived_row = unsafe { worktable::prelude::rkyv::access_unchecked_mut::<<#query_ident as worktable::prelude::rkyv::Archive>::Archived>(&mut bytes[..]).unseal_unchecked() }; #size_check #finish_update @@ -693,7 +773,7 @@ impl PersistGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -775,6 +855,7 @@ impl PersistGenerator { #primary_key_ident, #secondary_events_ident > = Operation::Update(UpdateOperation { + retired_link: None, id: op_id, primary_key_events: vec![], secondary_keys_events, @@ -811,6 +892,7 @@ impl PersistGenerator { }; let full_row_lock = self.gen_full_lock_for_update(); let data_write = self.gen_data_write_and_fetch(&row_updates, idx_idents); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); let loop_tail = if has_unsized { quote! {} @@ -860,7 +942,7 @@ impl PersistGenerator { pks.sort_unstable(); pks.dedup(); - let mut guards: std::collections::HashMap<_, _> = std::collections::HashMap::new(); + let mut guards: worktable::prelude::HashMap<_, _> = worktable::prelude::HashMap::new(); // Full-row locks, not per-column custom locks: each row's // unsized reinsert path mutates the whole row under these // guards, and one uniform lock kind keeps every concurrent @@ -871,7 +953,7 @@ impl PersistGenerator { guards.insert(pk.clone(), pending_lock.into_guard()); } - let op_id = OperationId::Multi(uuid::Uuid::now_v7()); + let op_id = OperationId::Multi(worktable::prelude::uuid::Uuid::now_v7()); for pk in pks.into_iter() { // Re-resolve and re-validate under the held lock. The // query's lock set includes the predicate column, so the @@ -888,17 +970,18 @@ impl PersistGenerator { continue; } let _mutation_guard = self.0.lock_manager.mutation_guard(&pk); - let mut bytes = rkyv::to_bytes::(&row) + let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; let mut archived_row = unsafe { - rkyv::access_unchecked_mut::<<#query_ident as rkyv::Archive>::Archived>(&mut bytes[..]) + worktable::prelude::rkyv::access_unchecked_mut::<<#query_ident as worktable::prelude::rkyv::Archive>::Archived>(&mut bytes[..]) .unseal_unchecked() }; #size_check #loop_tail } + #columnar_dirty core::result::Result::Ok(()) } } @@ -933,7 +1016,7 @@ impl PersistGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -974,14 +1057,16 @@ impl PersistGenerator { let custom_lock = self.gen_custom_lock_for_update(lock_ident); let data_write = self.gen_data_write_and_fetch(&row_updates, idx_idents); + let columnar_dirty = crate::generators::columnar::table_mark_dirty(&self.columns); let finish_update = if archived_swap_is_safe { quote! { - let op_id = OperationId::Single(uuid::Uuid::now_v7()); + let op_id = OperationId::Single(worktable::prelude::uuid::Uuid::now_v7()); #diff_process_insert #data_write #persist_op #diff_process_remove + #columnar_dirty #persist_call @@ -993,21 +1078,43 @@ impl PersistGenerator { quote! { pub async fn #method_ident(&self, row: #query_ident, by: #by_ident) -> core::result::Result<(), WorkTableError> { - let mut bytes = rkyv::to_bytes::(&row) + let mut bytes = worktable::prelude::rkyv::to_bytes::(&row) .map_err(|_| WorkTableError::SerializeError)?; let mut archived_row = unsafe { - rkyv::access_unchecked_mut::<<#query_ident as rkyv::Archive>::Archived>(&mut bytes[..]) + worktable::prelude::rkyv::access_unchecked_mut::<<#query_ident as worktable::prelude::rkyv::Archive>::Archived>(&mut bytes[..]) .unseal_unchecked() }; - let mut link: Link = self.0.indexes - .#index - .get_value(#by) - .map(Into::into) - .ok_or(WorkTableError::NotFound)?; - - let pk = self.0.data.select_non_ghosted(link)?.get_primary_key().clone(); + let pk = { + let mut retries = 0u32; + loop { + // Pin before reading the index so a relocated slot + // cannot be reclaimed and reused while resolving its PK. + // Drop the pin before yielding or awaiting the row lock. + let resolved = { + let _read_guard = self.0.data.read_guard(); + let link: Link = self.0.indexes.#index.get_value(#by) + .map(Into::into) + .ok_or(WorkTableError::NotFound)?; + self.0.data.select_non_ghosted(link) + }; + match resolved { + core::result::Result::Ok(found) => break found.get_primary_key(), + core::result::Result::Err(error) if error.is_row_absent() => { + // Reinsert publishes a replacement before retiring + // the old slot. Resolve the index again, rather than + // reporting a deleted row from that stale slot. + if retries >= 64 { + return Err(WorkTableError::NotFound); + } + retries += 1; + worktable::prelude::yield_now().await; + } + core::result::Result::Err(error) => return Err(error.into()), + } + } + }; let pending_lock = { #custom_lock }; let _guard = pending_lock.into_guard_with_mutation(); @@ -1034,7 +1141,7 @@ impl PersistGenerator { return Err(WorkTableError::NotFound); } vacuum_retries += 1; - tokio::task::yield_now().await; + worktable::prelude::yield_now().await; } core::result::Result::Err(e) => return Err(e.into()), } @@ -1097,6 +1204,7 @@ mod tests { updates, deletes: IndexMap::new(), in_place: IndexMap::new(), + ..Default::default() }); generator.gen_primary_key_def().unwrap(); @@ -1143,5 +1251,45 @@ mod tests { .find("Operation :: Update (UpdateOperation") .expect("update op emitted"); assert!(insert < write && write < op_build, "emission order broken:\n{emitted}"); + + // Every event the index produced must reach the persistence stream, + // including on the paths that fail. The index assigns an event id at + // the moment it produces the event, so a path that returns without + // queueing one leaves a hole `BatchOperation::validate` will refuse + // forever, and the stall it causes names a range rather than a cause. + // + // The `NotFound` arm used to be exactly that: its sibling + // `AlreadyExists` arm built an Acknowledge and it did not. + let not_found = emitted + .find("IndexError :: NotFound =>") + .expect("not-found arm emitted"); + let tail = &emitted[not_found..]; + let ack = tail + .find("Operation :: Acknowledge") + .expect("not-found arm must acknowledge its events"); + let returns = tail + .find("Err (WorkTableError :: NotFound)") + .expect("not-found arm returns"); + assert!( + ack < returns, + "the not-found arm returns before acknowledging, which gaps the stream:\n{emitted}" + ); + + // Same for the removal side, where the events were dropped by a bare + // `res?` before the extend that would have carried them. + let removal = emitted + .find("process_difference_remove_cdc (link , diffs)") + .expect("old-key removal emitted"); + let tail = &emitted[removal..]; + let ack = tail + .find("Operation :: Acknowledge") + .expect("a failed removal must acknowledge its events"); + let extend = tail + .find("op . extend_secondary_key_events") + .expect("successful removal extends the operation"); + assert!( + ack < extend, + "a failed removal propagates before acknowledging, which gaps the stream:\n{emitted}" + ); } } diff --git a/codegen/src/generators/persist/row.rs b/codegen/src/generators/persist/row.rs index 8839652c..ae56a56b 100644 --- a/codegen/src/generators/persist/row.rs +++ b/codegen/src/generators/persist/row.rs @@ -78,7 +78,8 @@ impl PersistGenerator { }; quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, Clone, rkyv::Serialize, PartialEq, MemStat)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, Clone, worktable::prelude::rkyv::Serialize, PartialEq, MemStat)] + #[rkyv(crate = worktable::prelude::rkyv)] #custom_derives #[rkyv(derive(Debug))] #[repr(C)] @@ -118,7 +119,8 @@ impl PersistGenerator { .collect(); quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, Clone, rkyv::Serialize, PartialEq)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, Clone, worktable::prelude::rkyv::Serialize, PartialEq)] + #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(Debug))] #[repr(C)] pub enum #enum_name { diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index 39e453b2..0e84ee3c 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -231,6 +231,7 @@ impl PersistGenerator { let space_ident = name_generator.get_space_file_ident(); let pk_type = name_generator.get_primary_key_type_ident(); let const_name = name_generator.get_page_inner_size_const_ident(); + let node_capacity = name_generator.get_disk_page_capacity(); let secondary_index_events = name_generator.get_space_secondary_index_events_ident(); let avt_index_ident = name_generator.get_available_indexes_ident(); @@ -254,33 +255,36 @@ impl PersistGenerator { }; let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( PersistentArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); } } else if pk_types_unsized { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( - #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name) + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( + #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#node_capacity) )); } } else { match self.columns.primary_index_backend { + crate::common::model::IndexBackend::FxHash => unreachable!( + "`using fxhash` on a paged table is refused in `worktable/mod.rs` before any generator runs" + ), crate::common::model::IndexBackend::WorktablesIndex => quote! { - let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + let size = get_index_page_size_from_data_length::<#pk_type>(#node_capacity); + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( #wti_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) )); }, crate::common::model::IndexBackend::Indexset => quote! { - let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + let size = get_index_page_size_from_data_length::<#pk_type>(#node_capacity); + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( UpstreamIndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) )); }, crate::common::model::IndexBackend::Arctic => unreachable!("handled before variable-size dispatch"), crate::common::model::IndexBackend::Congee => quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( PersistentCongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); }, @@ -300,7 +304,7 @@ impl PersistGenerator { + 'static, C: Clone + PersistenceConfig, { - async fn new(mut engine: E) -> eyre::Result { + async fn new(mut engine: E) -> worktable::prelude::eyre::Result { let schema = Self::space_info_default().inner; engine .ensure_schema( @@ -318,11 +322,11 @@ impl PersistGenerator { )) } - async fn load(engine: E) -> eyre::Result { + async fn load(engine: E) -> worktable::prelude::eyre::Result { Self::load_with(engine, LoadMode::Strict).await } - async fn load_with(mut engine: E, mode: LoadMode) -> eyre::Result { + async fn load_with(mut engine: E, mode: LoadMode) -> worktable::prelude::eyre::Result { let schema = Self::space_info_default().inner; engine .validate_schema( @@ -337,7 +341,7 @@ impl PersistGenerator { }; let table = load_persisted_state(&table_path, async { let space = #space_ident::parse_file(&table_path).await?; - Ok::<_, eyre::Report>(space.into_worktable_with_mode(engine, &table_path, mode).await?) + Ok::<_, worktable::prelude::eyre::Report>(space.into_worktable_with_mode(engine, &table_path, mode).await?) }).await?; Ok(table) } @@ -409,7 +413,7 @@ impl PersistGenerator { #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds + 'a, + R: core::ops::RangeBounds + 'a, Pk: Clone + 'a, { let converted_range = ( @@ -418,7 +422,7 @@ impl PersistGenerator { ); // Delay the grace-period guard until the returned iterator is // consumed so an idle query builder cannot pin reclamation. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .range_links(converted_range) @@ -639,7 +643,7 @@ impl PersistGenerator { } if backoff_spins < 8 { backoff_spins = backoff_spins.saturating_add(1); - tokio::task::yield_now().await; + worktable::prelude::yield_now().await; } else { // Cap the exponent BEFORE shifting: `1u64 << 64` panics // (overflow) in debug/test builds. Clamp the shift to a @@ -648,7 +652,7 @@ impl PersistGenerator { let exponent = core::cmp::min(backoff_spins - 8, 8); let micros = core::cmp::min(1u64 << exponent, 256); backoff_spins = backoff_spins.saturating_add(1); - tokio::time::sleep(std::time::Duration::from_micros(micros)).await; + worktable::prelude::sleep(core::time::Duration::from_micros(micros)).await; } } } @@ -689,7 +693,7 @@ impl PersistGenerator { /// assigned contiguous keys before `insert_many`. Interleaved /// `get_next_pk` calls keep working and never overlap a /// reservation. - pub fn reserve_pks(&self, count: usize) -> std::ops::Range<#pk_inner_type> { + pub fn reserve_pks(&self, count: usize) -> core::ops::Range<#pk_inner_type> { self.0.reserve_pks(count) } } @@ -732,7 +736,7 @@ impl PersistGenerator { quote! { pub async fn iter_with_async< F: Fn(#row_type) -> Fut, - Fut: std::future::Future> + Fut: core::future::Future> >(&self, f: F) -> core::result::Result<(), WorkTableError> { #inner } @@ -794,8 +798,14 @@ impl PersistGenerator { let lock_type = name_generator.get_lock_type_ident(); quote! { - pub fn vacuum(&self) -> std::sync::Arc { - std::sync::Arc::new(EmptyDataVacuum::< + pub fn vacuum(&self) -> worktable::prelude::Arc { + self.vacuum_with_pacing(worktable::prelude::VacuumPacing::default()) + } + + /// Creates a persisted sweep with the selected pacing policy. + /// Index moves retain the same persistence sink as the default sweep. + pub fn vacuum_with_pacing(&self, pacing: worktable::prelude::VacuumPacing) -> worktable::prelude::Arc { + worktable::prelude::Arc::new(EmptyDataVacuum::< _, _, _, @@ -807,11 +817,11 @@ impl PersistGenerator { #secondary_index_events >::new( #table_name, - std::sync::Arc::clone(&self.0.data), - std::sync::Arc::clone(&self.0.lock_manager), - std::sync::Arc::clone(&self.0.primary_index), - std::sync::Arc::clone(&self.0.indexes), - ).with_persistence(self.1.vacuum_sink())) + worktable::prelude::Arc::clone(&self.0.data), + worktable::prelude::Arc::clone(&self.0.lock_manager), + worktable::prelude::Arc::clone(&self.0.primary_index), + worktable::prelude::Arc::clone(&self.0.indexes), + ).with_pacing(pacing).with_persistence(self.1.vacuum_sink())) } } } diff --git a/codegen/src/generators/persist/table/index_fns.rs b/codegen/src/generators/persist/table/index_fns.rs index e6712995..20f6f5e2 100644 --- a/codegen/src/generators/persist/table/index_fns.rs +++ b/codegen/src/generators/persist/table/index_fns.rs @@ -99,7 +99,7 @@ impl PersistGenerator { if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None }; @@ -140,7 +140,7 @@ impl PersistGenerator { #column_range_type, #row_fields_ident> { - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.indexes.#field_ident .get(#by) @@ -173,7 +173,7 @@ impl PersistGenerator { let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { ( predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), @@ -183,7 +183,7 @@ impl PersistGenerator { ) } else { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { predicate_range.clone() }, ) }; @@ -216,7 +216,7 @@ impl PersistGenerator { }; let predicate_filter = quote! { .filter(move |row| { - std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + core::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) }) }; @@ -231,7 +231,7 @@ impl PersistGenerator { #predicate_setup // Query construction is not an active read. Pin the grace // period on the first row lookup instead. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); #index_range .filter_map(#select_row) diff --git a/codegen/src/generators/persist/table/mod.rs b/codegen/src/generators/persist/table/mod.rs index 745ca8d1..0a5500d6 100644 --- a/codegen/src/generators/persist/table/mod.rs +++ b/codegen/src/generators/persist/table/mod.rs @@ -19,6 +19,8 @@ impl PersistGenerator { let index_fns = self.gen_table_index_fns()?; let select_query_executor_impl = self.gen_table_select_query_executor_impl(); let column_range_type = self.gen_table_column_range_type(); + let columnar_methods = crate::generators::columnar::table_methods(&self.name, &self.columns); + let table_ident = WorktableNameGenerator::from_table_name(self.name.to_string()).get_work_table_ident(); Ok(quote! { #page_size_consts @@ -29,6 +31,9 @@ impl PersistGenerator { #index_fns #select_query_executor_impl #column_range_type + impl #table_ident { + #columnar_methods + } }) } @@ -51,17 +56,24 @@ impl PersistGenerator { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let page_const_name = name_generator.get_page_size_const_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let row_type = name_generator.get_row_type_ident(); if let Some(page_size) = &self.config.as_ref().and_then(|c| c.page_size) { let page_size = Literal::usize_unsuffixed(*page_size as usize); quote! { const #page_const_name: usize = #page_size; - const #inner_const_name: usize = #page_size - GENERAL_HEADER_SIZE; + const #inner_const_name: usize = worktable::prelude::data_page_row_capacity( + #page_size, + core::mem::size_of::<<<#row_type as worktable::prelude::StorableRow>::WrappedRow as worktable::prelude::rkyv::Archive>::Archived>(), + ); } } else { quote! { const #page_const_name: usize = PAGE_SIZE; - const #inner_const_name: usize = #page_const_name - GENERAL_HEADER_SIZE; + const #inner_const_name: usize = worktable::prelude::data_page_row_capacity( + #page_const_name, + core::mem::size_of::<<<#row_type as worktable::prelude::StorableRow>::WrappedRow as worktable::prelude::rkyv::Archive>::Archived>(), + ); } } } @@ -102,6 +114,9 @@ impl PersistGenerator { .collect::>(); let pk_types_unsized = is_unsized_vec(pk_types); let derive = match (pk_types_unsized, self.columns.primary_index_backend) { + (_, crate::common::model::IndexBackend::FxHash) => unreachable!( + "`using fxhash` on a paged table is refused in `worktable/mod.rs` before any generator runs" + ), (true, crate::common::model::IndexBackend::Indexset) => quote! { #[derive(Debug, PersistTable)] #[table(pk_unsized, pk_upstream)] @@ -202,52 +217,27 @@ impl PersistGenerator { } }); - Ok(if self.config.as_ref().and_then(|c| c.page_size).is_some() { - quote! { - #derive - #schema_attribute - #secondary_schema_attribute - pub struct #ident( - // Public because the crate's own internals reach the inner - // table directly: a synchronous internal path cannot call - // the generated wrapper once that wrapper is async. - pub WorkTable< - #row_type, - #primary_key_type, - #avt_type_ident, - #avt_index_ident, - #index_type, - #lock_ident, - <#primary_key_type as TablePrimaryKey>::Generator, - #inner_const_name, - #node_type - > - , #persistence_task - ); - } - } else { - quote! { - #derive - #schema_attribute - #secondary_schema_attribute - pub struct #ident( - // Public because the crate's own internals reach the inner - // table directly: a synchronous internal path cannot call - // the generated wrapper once that wrapper is async. - pub WorkTable< - #row_type, - #primary_key_type, - #avt_type_ident, - #avt_index_ident, - #index_type, - #lock_ident, - <#primary_key_type as TablePrimaryKey>::Generator, - { INNER_PAGE_SIZE }, - #node_type - > - , #persistence_task - ); - } + Ok(quote! { + #derive + #schema_attribute + #secondary_schema_attribute + pub struct #ident( + // Public because the crate's own internals reach the inner + // table directly: a synchronous internal path cannot call + // the generated wrapper once that wrapper is async. + pub WorkTable< + #row_type, + #primary_key_type, + #avt_type_ident, + #avt_index_ident, + #index_type, + #lock_ident, + <#primary_key_type as TablePrimaryKey>::Generator, + #inner_const_name, + #node_type + > + , #persistence_task + ); }) } } diff --git a/codegen/src/generators/persist/table/select_executor.rs b/codegen/src/generators/persist/table/select_executor.rs index 1499d250..281a3340 100644 --- a/codegen/src/generators/persist/table/select_executor.rs +++ b/codegen/src/generators/persist/table/select_executor.rs @@ -44,7 +44,7 @@ impl PersistGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - #variant_ident(std::ops::#range_ident<#ty_ident>), + #variant_ident(core::ops::#range_ident<#ty_ident>), } }) .collect(); @@ -65,8 +65,8 @@ impl PersistGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - impl From> for #column_range_type { - fn from(range: std::ops::#range_ident<#ty_ident>) -> Self { + impl From> for #column_range_type { + fn from(range: core::ops::#range_ident<#ty_ident>) -> Self { Self::#variant_ident(range) } } @@ -92,6 +92,11 @@ impl PersistGenerator { pub fn gen_table_select_query_executor_impl(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let row_type = name_generator.get_row_type_ident(); + let default_dispatch = if cfg!(feature = "std") { + quote! { Some(worktable::runtime::dispatcher::<<#row_type as worktable::runtime::TableRuntime>::Backend> as worktable::runtime::Dispatch) } + } else { + quote! { None } + }; let column_range_type = name_generator.get_column_range_type_ident(); let row_fields_ident = name_generator.get_row_fields_enum_ident(); @@ -100,8 +105,8 @@ impl PersistGenerator { let col_ident = Ident::new(&column.to_string(), Span::call_site()); quote! { #row_fields_ident::#column_variant => { - let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(std::cmp::Ordering::Equal); - if cmp != std::cmp::Ordering::Equal { + let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(core::cmp::Ordering::Equal); + if cmp != core::cmp::Ordering::Equal { return match order { Order::Asc => cmp, Order::Desc => cmp.reverse(), @@ -165,12 +170,30 @@ impl PersistGenerator { _ => continue, } } - std::cmp::Ordering::Equal + core::cmp::Ordering::Equal }); iter = Box::new(items.into_iter()); }; quote! { + impl worktable::prelude::SelectQueryAsyncExecutor<#row_type> + for SelectQueryBuilder<#row_type, I, #column_range_type, #row_fields_ident> + where I: DoubleEndedIterator + Sized, + { + fn execute_async(self) -> worktable::prelude::SelectQueryFuture<#row_type> { + let mut params = self.params; + let dispatch = params.dispatch.take().or(#default_dispatch); + // Release all borrowed iterators and caller predicates before + // creating a task. No lifetime is extended across the pool. + let rows: Vec<#row_type> = self.iter.collect(); + Box::pin(async move { + let plan = SelectQueryBuilder { params, iter: rows.into_iter() }; + if let Some(dispatch) = dispatch { + worktable::runtime::run_owned(dispatch, move || plan.execute()).await? + } else { plan.execute() } + }) + } + } impl SelectQueryExecutor<#row_type, I, #column_range_type, #row_fields_ident> for SelectQueryBuilder<#row_type, I, #column_range_type, #row_fields_ident> where @@ -191,6 +214,7 @@ impl PersistGenerator { } fn execute(self) -> Result, WorkTableError> { + if self.params.dispatch.is_some() { return Err(WorkTableError::RuntimeRequiresAsync); } let mut iter: Box> = Box::new(self.iter); #range diff --git a/codegen/src/generators/persist/wrapper.rs b/codegen/src/generators/persist/wrapper.rs index 8308495a..a28bd36f 100644 --- a/codegen/src/generators/persist/wrapper.rs +++ b/codegen/src/generators/persist/wrapper.rs @@ -24,7 +24,8 @@ impl PersistGenerator { let wrapper_ident = name_generator.get_wrapper_type_ident(); quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, rkyv::Serialize)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, worktable::prelude::rkyv::Serialize)] + #[rkyv(crate = worktable::prelude::rkyv)] #[repr(C)] pub struct #wrapper_ident { inner: #row_ident, diff --git a/codegen/src/generators/profile_dispatch.rs b/codegen/src/generators/profile_dispatch.rs new file mode 100644 index 00000000..34ea27c1 --- /dev/null +++ b/codegen/src/generators/profile_dispatch.rs @@ -0,0 +1,57 @@ +//! Wrap explicitly scheduled mutations in owned, cancellable tasks. +use proc_macro2::{Ident, TokenStream}; +use quote::{format_ident, quote}; +use syn::{FnArg, GenericParam, ImplItem, Pat, parse_quote}; + +pub(crate) fn wrap(methods: TokenStream, profile: Option<&Ident>, row: &Ident) -> syn::Result { + let Some(profile) = profile else { return Ok(methods) }; + if !cfg!(feature = "std") { + return Err(syn::Error::new( + profile.span(), + "query runtime profiles require WorkTable's std feature", + )); + } + let parsed: syn::ItemImpl = syn::parse2(quote! { impl Placeholder { #methods } })?; + let mut out = TokenStream::new(); + for item in parsed.items { + let ImplItem::Fn(mut inline) = item else { + out.extend(quote! { #item }); + continue; + }; + let mut scheduled = inline.clone(); + let inline_name = format_ident!("__wt_inline_{}", inline.sig.ident); + inline.sig.ident = inline_name.clone(); + inline.vis = syn::Visibility::Inherited; + inline.attrs.push(parse_quote!(#[allow(dead_code)])); + let mut arguments = Vec::new(); + for argument in &mut scheduled.sig.inputs { + match argument { + FnArg::Receiver(receiver) => *receiver = parse_quote!(self: &worktable::prelude::Arc), + FnArg::Typed(argument) => { + let Pat::Ident(pattern) = &mut *argument.pat else { + return Err(syn::Error::new_spanned(argument, "query argument must be named")); + }; + pattern.mutability = None; + arguments.push(pattern.ident.clone()); + } + } + } + for parameter in &mut scheduled.sig.generics.params { + if let GenericParam::Type(parameter) = parameter { + parameter.bounds.push(parse_quote!(Send)); + parameter.bounds.push(parse_quote!('static)); + } + } + scheduled.block = parse_quote!({ + fn check_profile() + where P::Backend: worktable::runtime::RuntimeCompatibleWith<<#row as worktable::runtime::TableRuntime>::Backend> {} + check_profile::<#profile>(); + let table = worktable::prelude::Arc::clone(self); + worktable::runtime::run_profile::<#profile, _>(async move { + table.#inline_name(#(#arguments),*).await + }).await? + }); + out.extend(quote! { #inline #scheduled }); + } + Ok(out) +} diff --git a/codegen/src/generators/read_only/index/mod.rs b/codegen/src/generators/read_only/index/mod.rs index 048c679e..9a7980d7 100644 --- a/codegen/src/generators/read_only/index/mod.rs +++ b/codegen/src/generators/read_only/index/mod.rs @@ -10,6 +10,7 @@ use quote::quote; impl ReadOnlyGenerator { pub fn gen_index_def(&mut self) -> syn::Result { + let columnar_def = crate::generators::columnar::definitions(&self.name, &self.columns); let type_def = self.gen_type_def()?; let impl_def = self.gen_secondary_index_impl_def(); let info_def = self.gen_secondary_index_info_impl_def(); @@ -17,6 +18,7 @@ impl ReadOnlyGenerator { let available_indexes = self.gen_available_indexes(); Ok(quote! { + #columnar_def #type_def #impl_def #info_def @@ -71,11 +73,13 @@ impl ReadOnlyGenerator { #[derive(Debug, MemStat, PersistIndex)] #[index(read_only)] }; + let columnar_field = crate::generators::columnar::index_struct_field(&self.name, &self.columns, true); Ok(quote! { #derive pub struct #ident { - #(#index_rows),* + #(#index_rows,)* + #columnar_field } }) } @@ -106,6 +110,7 @@ impl ReadOnlyGenerator { #[allow(clippy::collapsible_else_if)] let res = if idx.is_unique { match idx.backend { + crate::common::model::IndexBackend::FxHash => unreachable!("`using fxhash` on a paged table is refused in `worktable/mod.rs` before any generator runs"), crate::common::model::IndexBackend::WorktablesIndex => { if is_unsized(&t.to_string()) { quote! { #i: IndexMap::with_maximum_node_size(#const_name), } @@ -138,12 +143,14 @@ impl ReadOnlyGenerator { Ok::<_, syn::Error>(res) }) .collect::, syn::Error>>()?; + let columnar_field = crate::generators::columnar::index_default_field(&self.columns); Ok(quote! { impl Default for #index_type_ident { fn default() -> Self { Self { #(#index_rows)* + #columnar_field } } } @@ -168,11 +175,22 @@ impl ReadOnlyGenerator { } } else { quote! { - #[derive(Debug, Clone, Copy, MoreDisplay, PartialEq, PartialOrd, Ord, Hash, Eq)] + #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Ord, Hash, Eq)] pub enum #avt_type_ident { #(#indexes)* } + // Delegated to `Debug` rather than derived. Every variant here + // is fieldless, so `Debug` prints exactly the variant name, + // which is what `derive_more::Display` produced. Deriving it + // put `::derive_more::` paths in the expansion and so put that + // crate into this macro's contract. + impl core::fmt::Display for #avt_type_ident { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Debug::fmt(self, f) + } + } + impl AvailableIndex for #avt_type_ident { fn to_string_value(&self) -> String { ToString::to_string(&self) diff --git a/codegen/src/generators/read_only/index/usual.rs b/codegen/src/generators/read_only/index/usual.rs index 0af30608..e61b7617 100644 --- a/codegen/src/generators/read_only/index/usual.rs +++ b/codegen/src/generators/read_only/index/usual.rs @@ -19,9 +19,11 @@ impl ReadOnlyGenerator { let process_difference_insert_fn = self.gen_process_difference_insert_index_fn(); let process_difference_remove_fn = self.gen_process_difference_remove_index_fn(); let delete_from_indexes = self.gen_index_delete_from_indexes_fn(); + let publication_guard = crate::generators::columnar::publication_guard(&self.columns); quote! { impl TableSecondaryIndex<#row_type_ident, #avt_type_ident, #avt_index_ident> for #index_type_ident { + #publication_guard #save_row_fn #reinsert_row_fn #delete_row_fn @@ -69,11 +71,13 @@ impl ReadOnlyGenerator { } }) .collect::>(); + let columnar_save = crate::generators::columnar::save_row(&self.columns); quote! { fn save_row(&self, row: #row_type_ident, link: Link) -> core::result::Result<(), IndexError<#available_index_ident>> { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; #(#save_rows)* + #columnar_save core::result::Result::Ok(()) } } @@ -147,6 +151,7 @@ impl ReadOnlyGenerator { (insert, remove) }) .unzip(); + let columnar_reinsert = crate::generators::columnar::reinsert_row(&self.columns); quote! { fn reinsert_row(&self, @@ -159,6 +164,7 @@ impl ReadOnlyGenerator { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; #(#insert_rows)* #(#remove_rows)* + #columnar_reinsert core::result::Result::Ok(()) } } @@ -189,10 +195,12 @@ impl ReadOnlyGenerator { } }) .collect::>(); + let columnar_delete = crate::generators::columnar::delete_row(&self.columns); quote! { fn delete_row(&self, row: #row_type_ident, link: Link) -> core::result::Result<(), IndexError<#available_index_ident>> { #(#delete_rows)* + #columnar_delete core::result::Result::Ok(()) } } @@ -236,7 +244,7 @@ impl ReadOnlyGenerator { fn process_difference_remove( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { #(#process_difference_remove_rows)* core::result::Result::Ok(()) @@ -293,7 +301,7 @@ impl ReadOnlyGenerator { fn process_difference_insert( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { let mut inserted_indexes: Vec<#avt_index_ident> = vec![]; #(#process_difference_insert_rows)* diff --git a/codegen/src/generators/read_only/locks.rs b/codegen/src/generators/read_only/locks.rs index 280afd28..ffa50040 100644 --- a/codegen/src/generators/read_only/locks.rs +++ b/codegen/src/generators/read_only/locks.rs @@ -24,7 +24,7 @@ impl ReadOnlyGenerator { .keys() .map(|i| { let name = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); - quote! { #name: Option>, } + quote! { #name: Option>, } }) .collect(); @@ -124,8 +124,8 @@ impl ReadOnlyGenerator { .collect(); quote! { - fn with_lock(id: u16) -> (Self, std::sync::Arc) { - let lock = std::sync::Arc::new(Lock::new(id)); + fn with_lock(id: u16) -> (Self, worktable::prelude::Arc) { + let lock = worktable::prelude::Arc::new(Lock::new(id)); ( Self { #(#rows),* @@ -154,9 +154,9 @@ impl ReadOnlyGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let lock = std::sync::Arc::new(Lock::new(id)); + fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* (set, lock) @@ -186,8 +186,8 @@ impl ReadOnlyGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> std::collections::HashSet> { - let mut set = std::collections::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { + let mut set = worktable::prelude::HashSet::new(); #(#rows)* set } diff --git a/codegen/src/generators/read_only/primary_key.rs b/codegen/src/generators/read_only/primary_key.rs index 192a2e44..e7ea6bff 100644 --- a/codegen/src/generators/read_only/primary_key.rs +++ b/codegen/src/generators/read_only/primary_key.rs @@ -66,19 +66,68 @@ impl ReadOnlyGenerator { primary_key_backend_impl(self.columns.primary_index_backend, &ident, types)?; let borrowed_impl = gen_borrowed_primary_key_impl(&ident, types); + // `From` written out rather than derived. `derive_more::From` puts + // `::derive_more::` paths in its expansion, which made that crate part + // of this macro's contract: a consumer who never wrote `derive_more` + // had to declare it anyway. A newtype conversion is three lines. + // + // A composite key converts from the tuple, which is the shape + // `derive_more` produced for a multi-field tuple struct. + let from_impl = if types.len() == 1 { + let ty = &types[0]; + quote! { + impl From<#ty> for #ident { + fn from(value: #ty) -> Self { + Self(value) + } + } + } + } else { + let binding: Vec<_> = (0..types.len()) + .map(|i| syn::Ident::new(&format!("field{i}"), proc_macro2::Span::mixed_site())) + .collect(); + quote! { + impl From<(#(#types),*)> for #ident { + fn from((#(#binding),*): (#(#types),*)) -> Self { + Self(#(#binding),*) + } + } + } + }; + + // And the reverse direction, which was `derive_more::Into`. Same + // reasoning: it is one impl, and deriving it dragged the crate in. + let into_impl = if types.len() == 1 { + let ty = &types[0]; + quote! { + impl From<#ident> for #ty { + fn from(value: #ident) -> Self { + value.0 + } + } + } + } else { + let index: Vec<_> = (0..types.len()).map(syn::Index::from).collect(); + quote! { + impl From<#ident> for (#(#types),*) { + fn from(value: #ident) -> Self { + (#(value.#index),*) + } + } + } + }; + Ok(quote! { #[derive( Clone, #backend_derive - rkyv::Archive, + worktable::prelude::rkyv::Archive, Debug, Default, - rkyv::Deserialize, + worktable::prelude::rkyv::Deserialize, Hash, - rkyv::Serialize, - From, + worktable::prelude::rkyv::Serialize, Eq, - Into, PartialEq, PartialOrd, Ord, @@ -86,9 +135,13 @@ impl ReadOnlyGenerator { MemStat, #unsized_derive )] + #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(PartialEq, Eq, PartialOrd, Ord, Debug))] pub struct #ident(#(#types),*); + #from_impl + #into_impl + #borrowed_impl #backend_impl }) @@ -133,14 +186,14 @@ impl ReadOnlyGenerator { fn get_generator_from_type(type_: &TokenStream, i: &Ident) -> syn::Result { Ok(match type_.to_string().as_str() { - "u8" => quote! { std::sync::atomic::AtomicU8 }, - "u16" => quote! { std::sync::atomic::AtomicU16 }, - "u32" => quote! { std::sync::atomic::AtomicU32 }, - "u64" => quote! { std::sync::atomic::AtomicU64 }, - "i8" => quote! { std::sync::atomic::AtomicI8 }, - "i16" => quote! { std::sync::atomic::AtomicI16 }, - "i32" => quote! { std::sync::atomic::AtomicI32 }, - "i64" => quote! { std::sync::atomic::AtomicI64 }, + "u8" => quote! { core::sync::atomic::AtomicU8 }, + "u16" => quote! { core::sync::atomic::AtomicU16 }, + "u32" => quote! { core::sync::atomic::AtomicU32 }, + "u64" => quote! { core::sync::atomic::AtomicU64 }, + "i8" => quote! { core::sync::atomic::AtomicI8 }, + "i16" => quote! { core::sync::atomic::AtomicI16 }, + "i32" => quote! { core::sync::atomic::AtomicI32 }, + "i64" => quote! { core::sync::atomic::AtomicI64 }, // The accepted set is `worktable_dsl::AUTOINCREMENT_TYPES`, and the // arms above must stay equal to it. `check` uses that list to // answer "would the macro accept this", so a second copy drifting diff --git a/codegen/src/generators/read_only/queries/select.rs b/codegen/src/generators/read_only/queries/select.rs index 0adcbe70..8c46e611 100644 --- a/codegen/src/generators/read_only/queries/select.rs +++ b/codegen/src/generators/read_only/queries/select.rs @@ -32,7 +32,7 @@ impl ReadOnlyGenerator { // Acquire the grace-period guard only when iteration starts. // Merely constructing and retaining a query builder must not // stall retired-link reclamation. - let iter = std::iter::once_with(move || { + let iter = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .iter_values() @@ -55,7 +55,7 @@ impl ReadOnlyGenerator { return None; } current_link = replacement; - std::hint::spin_loop(); + core::hint::spin_loop(); } None }) diff --git a/codegen/src/generators/read_only/queries/type.rs b/codegen/src/generators/read_only/queries/type.rs index 9503bdd0..17231d46 100644 --- a/codegen/src/generators/read_only/queries/type.rs +++ b/codegen/src/generators/read_only/queries/type.rs @@ -46,20 +46,38 @@ impl ReadOnlyGenerator { .expect("should be valid because parsed from declaration"); let type_upper = map_to_uppercase(s); let type_upper = Ident::new(type_upper.as_str(), Span::mixed_site()); - Some(quote! { - #[from] - #type_upper(#type_ident), - }) + Some(( + quote! { + #type_upper(#type_ident), + }, + // Written out rather than derived. `derive_more::From` + // generates `::derive_more::` paths inside its expansion, + // which makes that crate part of this macro's contract: + // a consumer who never wrote `derive_more` still had to + // declare it to compile a table. One newtype variant per + // type is a two-line impl, so the dependency bought + // nothing that could not be spelled here. + quote! { + impl From<#type_ident> for #avt_type_ident { + fn from(value: #type_ident) -> Self { + Self::#type_upper(value) + } + } + }, + )) }) .collect(); + let (rows, from_impls): (Vec<_>, Vec<_>) = rows.into_iter().flatten().unzip(); if !rows.is_empty() { Ok(quote! { - #[derive(Clone, Debug, From, PartialEq)] + #[derive(Clone, Debug, PartialEq)] #[non_exhaustive] pub enum #avt_type_ident { #(#rows)* } + + #(#from_impls)* }) } else { Ok(quote! { diff --git a/codegen/src/generators/read_only/row.rs b/codegen/src/generators/read_only/row.rs index 3565a04a..aa311c8b 100644 --- a/codegen/src/generators/read_only/row.rs +++ b/codegen/src/generators/read_only/row.rs @@ -70,7 +70,8 @@ impl ReadOnlyGenerator { } quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, Clone, rkyv::Serialize, PartialEq, MemStat)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, Clone, worktable::prelude::rkyv::Serialize, PartialEq, MemStat)] + #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(Debug))] #[repr(C)] pub struct #ident { @@ -109,7 +110,8 @@ impl ReadOnlyGenerator { .collect(); quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, Clone, rkyv::Serialize, PartialEq)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, Clone, worktable::prelude::rkyv::Serialize, PartialEq)] + #[rkyv(crate = worktable::prelude::rkyv)] #[rkyv(derive(Debug))] #[repr(C)] pub enum #enum_name { diff --git a/codegen/src/generators/read_only/table/impls.rs b/codegen/src/generators/read_only/table/impls.rs index 6e4577b6..033d328e 100644 --- a/codegen/src/generators/read_only/table/impls.rs +++ b/codegen/src/generators/read_only/table/impls.rs @@ -237,19 +237,19 @@ impl ReadOnlyGenerator { let pk_types_unsized = is_unsized_vec(pk_types); let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( ArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); } } else if self.columns.primary_index_backend == crate::common::model::IndexBackend::Congee { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( CongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); } } else if pk_types_unsized { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name) )); } @@ -260,7 +260,7 @@ impl ReadOnlyGenerator { }; quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) )); } @@ -279,25 +279,25 @@ impl ReadOnlyGenerator { + 'static, C: Clone + PersistenceConfig, { - async fn new(engine: E) -> eyre::Result { + async fn new(engine: E) -> worktable::prelude::eyre::Result { let mut inner = WorkTable::default(); inner.table_name = #table_name; #index_setup core::result::Result::Ok(Self(inner)) } - async fn load(engine: E) -> eyre::Result { + async fn load(engine: E) -> worktable::prelude::eyre::Result { Self::load_with(engine, LoadMode::Strict).await } - async fn load_with(engine: E, mode: LoadMode) -> eyre::Result { + async fn load_with(engine: E, mode: LoadMode) -> worktable::prelude::eyre::Result { let table_path = engine.config().table_path().to_owned(); if !std::path::Path::new(&table_path).exists() { return Self::new(engine).await; }; let table = load_persisted_state(&table_path, async { let space = #space_ident::parse_file(&table_path).await?; - Ok::<_, eyre::Report>(space.into_worktable_with_mode(&table_path, mode)?) + Ok::<_, worktable::prelude::eyre::Report>(space.into_worktable_with_mode(&table_path, mode)?) }).await?; Ok(table) } @@ -369,7 +369,7 @@ impl ReadOnlyGenerator { #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds + 'a, + R: core::ops::RangeBounds + 'a, Pk: Clone + 'a, { let converted_range = ( @@ -378,7 +378,7 @@ impl ReadOnlyGenerator { ); // Delay the grace-period guard until the returned iterator is // consumed so an idle query builder cannot pin reclamation. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .range_links(converted_range) @@ -460,7 +460,7 @@ impl ReadOnlyGenerator { quote! { pub async fn iter_with_async< F: Fn(#row_type) -> Fut, - Fut: std::future::Future> + Fut: core::future::Future> >(&self, f: F) -> core::result::Result<(), WorkTableError> { #inner } diff --git a/codegen/src/generators/read_only/table/index_fns.rs b/codegen/src/generators/read_only/table/index_fns.rs index 98a1d0f6..13ebb9c8 100644 --- a/codegen/src/generators/read_only/table/index_fns.rs +++ b/codegen/src/generators/read_only/table/index_fns.rs @@ -99,7 +99,7 @@ impl ReadOnlyGenerator { if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None }; @@ -140,7 +140,7 @@ impl ReadOnlyGenerator { #column_range_type, #row_fields_ident> { - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.indexes.#field_ident .get(#by) @@ -173,7 +173,7 @@ impl ReadOnlyGenerator { let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { ( predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), @@ -183,7 +183,7 @@ impl ReadOnlyGenerator { ) } else { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { predicate_range.clone() }, ) }; @@ -216,7 +216,7 @@ impl ReadOnlyGenerator { }; let predicate_filter = quote! { .filter(move |row| { - std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + core::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) }) }; @@ -231,7 +231,7 @@ impl ReadOnlyGenerator { #predicate_setup // Query construction is not an active read. Pin the grace // period on the first row lookup instead. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); #index_range .filter_map(#select_row) diff --git a/codegen/src/generators/read_only/table/mod.rs b/codegen/src/generators/read_only/table/mod.rs index 4079e981..16b2bef2 100644 --- a/codegen/src/generators/read_only/table/mod.rs +++ b/codegen/src/generators/read_only/table/mod.rs @@ -19,6 +19,8 @@ impl ReadOnlyGenerator { let index_fns = self.gen_table_index_fns()?; let select_query_executor_impl = self.gen_table_select_query_executor_impl(); let column_range_type = self.gen_table_column_range_type(); + let columnar_methods = crate::generators::columnar::table_methods(&self.name, &self.columns); + let table_ident = WorktableNameGenerator::from_table_name(self.name.to_string()).get_work_table_ident(); Ok(quote! { #page_size_consts @@ -29,6 +31,9 @@ impl ReadOnlyGenerator { #index_fns #select_query_executor_impl #column_range_type + impl #table_ident { + #columnar_methods + } }) } diff --git a/codegen/src/generators/read_only/table/select_executor.rs b/codegen/src/generators/read_only/table/select_executor.rs index bd8b3f7c..c0d805da 100644 --- a/codegen/src/generators/read_only/table/select_executor.rs +++ b/codegen/src/generators/read_only/table/select_executor.rs @@ -44,7 +44,7 @@ impl ReadOnlyGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - #variant_ident(std::ops::#range_ident<#ty_ident>), + #variant_ident(core::ops::#range_ident<#ty_ident>), } }) .collect(); @@ -65,8 +65,8 @@ impl ReadOnlyGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - impl From> for #column_range_type { - fn from(range: std::ops::#range_ident<#ty_ident>) -> Self { + impl From> for #column_range_type { + fn from(range: core::ops::#range_ident<#ty_ident>) -> Self { Self::#variant_ident(range) } } @@ -92,6 +92,11 @@ impl ReadOnlyGenerator { pub fn gen_table_select_query_executor_impl(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let row_type = name_generator.get_row_type_ident(); + let default_dispatch = if cfg!(feature = "std") { + quote! { Some(worktable::runtime::dispatcher::<<#row_type as worktable::runtime::TableRuntime>::Backend> as worktable::runtime::Dispatch) } + } else { + quote! { None } + }; let column_range_type = name_generator.get_column_range_type_ident(); let row_fields_ident = name_generator.get_row_fields_enum_ident(); @@ -100,8 +105,8 @@ impl ReadOnlyGenerator { let col_ident = Ident::new(&column.to_string(), Span::call_site()); quote! { #row_fields_ident::#column_variant => { - let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(std::cmp::Ordering::Equal); - if cmp != std::cmp::Ordering::Equal { + let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(core::cmp::Ordering::Equal); + if cmp != core::cmp::Ordering::Equal { return match order { Order::Asc => cmp, Order::Desc => cmp.reverse(), @@ -165,12 +170,30 @@ impl ReadOnlyGenerator { _ => continue, } } - std::cmp::Ordering::Equal + core::cmp::Ordering::Equal }); iter = Box::new(items.into_iter()); }; quote! { + impl worktable::prelude::SelectQueryAsyncExecutor<#row_type> + for SelectQueryBuilder<#row_type, I, #column_range_type, #row_fields_ident> + where I: DoubleEndedIterator + Sized, + { + fn execute_async(self) -> worktable::prelude::SelectQueryFuture<#row_type> { + let mut params = self.params; + let dispatch = params.dispatch.take().or(#default_dispatch); + // Release all borrowed iterators and caller predicates before + // creating a task. No lifetime is extended across the pool. + let rows: Vec<#row_type> = self.iter.collect(); + Box::pin(async move { + let plan = SelectQueryBuilder { params, iter: rows.into_iter() }; + if let Some(dispatch) = dispatch { + worktable::runtime::run_owned(dispatch, move || plan.execute()).await? + } else { plan.execute() } + }) + } + } impl SelectQueryExecutor<#row_type, I, #column_range_type, #row_fields_ident> for SelectQueryBuilder<#row_type, I, #column_range_type, #row_fields_ident> where @@ -191,6 +214,7 @@ impl ReadOnlyGenerator { } fn execute(self) -> Result, WorkTableError> { + if self.params.dispatch.is_some() { return Err(WorkTableError::RuntimeRequiresAsync); } let mut iter: Box> = Box::new(self.iter); #range diff --git a/codegen/src/generators/read_only/wrapper.rs b/codegen/src/generators/read_only/wrapper.rs index 9f69d5b4..3a8ead5f 100644 --- a/codegen/src/generators/read_only/wrapper.rs +++ b/codegen/src/generators/read_only/wrapper.rs @@ -24,7 +24,8 @@ impl ReadOnlyGenerator { let wrapper_ident = name_generator.get_wrapper_type_ident(); quote! { - #[derive(rkyv::Archive, Debug, rkyv::Deserialize, rkyv::Serialize)] + #[derive(worktable::prelude::rkyv::Archive, Debug, worktable::prelude::rkyv::Deserialize, worktable::prelude::rkyv::Serialize)] + #[rkyv(crate = worktable::prelude::rkyv)] #[repr(C)] pub struct #wrapper_ident { inner: #row_ident, diff --git a/codegen/src/generators/runtime_backend.rs b/codegen/src/generators/runtime_backend.rs new file mode 100644 index 00000000..d0642d03 --- /dev/null +++ b/codegen/src/generators/runtime_backend.rs @@ -0,0 +1,133 @@ +use proc_macro2::TokenStream; +use quote::quote; + +use crate::common::model::RuntimeBackend; + +/// Generates the concrete runtime type selected by the DSL. +/// +/// The twin of `index_backend::unique_index_type`, and deliberately shaped like +/// it: the DSL carries an enum, this turns the enum into a type the generated +/// code names, and nothing between the parser and the expansion has to know +/// which backend was chosen. The flavor is a type parameter rather than a +/// separate token because `NagoyaRt` is generic over it, so a table that picks +/// a tuning picks it at the type level and pays nothing at run time. +/// +/// Every marker type, plus `NagoyaRt` and `TokioRt`, is re-exported from +/// `worktable::prelude`, so the expansion needs no import of its own. +/// +/// The marker's spelling comes from [`Flavor::type_name`] rather than from a +/// match written here: a flavor added to the registry and missed here would +/// emit `NagoyaRt` for a table that asked for something else, which +/// compiles and then silently measures the wrong pool. +pub(crate) fn runtime_type(backend: RuntimeBackend) -> TokenStream { + match backend { + RuntimeBackend::Nagoya(flavor) => { + let marker = proc_macro2::Ident::new(flavor.type_name(), proc_macro2::Span::call_site()); + quote! { NagoyaRt<#marker> } + } + RuntimeBackend::Tokio => quote! { TokioRt }, + } +} + +/// Resolves which backend applies at one site. +/// +/// The chain is: a section's own annotation, then the table's `runtime:`, then +/// the built-in default. The middle step is the one worth stating: a table that +/// declares `runtime: tokio` and has an unannotated `update` section must give +/// that section tokio, not the built-in nagoya, or the table would silently run +/// two runtimes. +/// +/// `None` for both arguments must produce exactly what `runtime: nagoya` +/// produces, because every declaration written before this existed omits the +/// key and none of them may change. +pub(crate) fn resolve_runtime(section: Option, table: Option) -> RuntimeBackend { + section.or(table).unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use crate::common::model::Flavor; + + use super::*; + + fn rendered(backend: RuntimeBackend) -> String { + runtime_type(backend).to_string() + } + + #[test] + fn every_backend_maps_to_its_contract_type() { + assert_eq!( + rendered(RuntimeBackend::Nagoya(Flavor::Locality)), + "NagoyaRt < Locality >" + ); + assert_eq!(rendered(RuntimeBackend::Nagoya(Flavor::Spread)), "NagoyaRt < Spread >"); + assert_eq!( + rendered(RuntimeBackend::Nagoya(Flavor::Throughput)), + "NagoyaRt < Throughput >" + ); + assert_eq!(rendered(RuntimeBackend::Tokio), "TokioRt"); + assert_eq!( + rendered(RuntimeBackend::Nagoya(Flavor::LowLatency)), + "NagoyaRt < LowLatency >" + ); + assert_eq!( + rendered(RuntimeBackend::Nagoya(Flavor::WideInjector)), + "NagoyaRt < WideInjector >" + ); + } + + /// Every flavor has to emit a distinct type. A missing arm used to fall + /// through to `Locality`, which compiles and then measures the wrong pool + /// under the right name, so it is asserted rather than assumed. + #[test] + fn every_flavor_emits_its_own_marker() { + let mut rendered: Vec = Flavor::ALL + .into_iter() + .map(|flavor| super::runtime_type(RuntimeBackend::Nagoya(flavor)).to_string()) + .collect(); + let before = rendered.len(); + rendered.sort(); + rendered.dedup(); + assert_eq!(before, rendered.len(), "two flavors emit the same type: {rendered:?}"); + } + + /// An omitted `runtime:` and a bare `runtime: nagoya` are the same table, + /// whichever flavor is currently the default. Asserted against + /// `Flavor::default()` rather than a named flavor, so that moving the + /// default is one edit rather than a hunt through the tests. + #[test] + fn the_default_backend_is_nagoya_at_the_default_flavor() { + assert_eq!( + rendered(RuntimeBackend::default()), + rendered(RuntimeBackend::Nagoya(Flavor::default())) + ); + } + + #[test] + fn a_section_annotation_wins_over_the_table() { + assert_eq!( + resolve_runtime( + Some(RuntimeBackend::Nagoya(Flavor::Spread)), + Some(RuntimeBackend::Tokio) + ), + RuntimeBackend::Nagoya(Flavor::Spread) + ); + } + + #[test] + fn an_unannotated_section_falls_back_to_the_table_not_the_default() { + assert_eq!( + resolve_runtime(None, Some(RuntimeBackend::Tokio)), + RuntimeBackend::Tokio + ); + assert_eq!( + resolve_runtime(None, Some(RuntimeBackend::Nagoya(Flavor::Throughput))), + RuntimeBackend::Nagoya(Flavor::Throughput) + ); + } + + #[test] + fn neither_declared_resolves_to_the_built_in_default() { + assert_eq!(resolve_runtime(None, None), RuntimeBackend::Nagoya(Flavor::default())); + } +} diff --git a/codegen/src/generators/vec_table/mod.rs b/codegen/src/generators/vec_table/mod.rs new file mode 100644 index 00000000..44387faf --- /dev/null +++ b/codegen/src/generators/vec_table/mod.rs @@ -0,0 +1,1541 @@ +//! A `Vec`-backed table with the same shape as a `worktable!` and none of its machinery. +//! +//! # What this is for +//! +//! `worktable!` buys concurrency and durability with an archived row, paged +//! storage behind links, a row-level lock map and change-data-capture. A +//! single-threaded table that never persists pays all of that for nothing, and +//! what applications otherwise grow by hand is a `Vec` plus a map from key to +//! position. +//! +//! So this generates that, from the same declaration, so the two can sit side +//! by side and be compared on identical rows. +//! +//! # What it drops, deliberately +//! +//! Each of these is the reason a `worktable!` costs what it does, and dropping +//! them is the point rather than an omission: +//! +//! - **The archived row.** Rows are stored as themselves. No `rkyv`, no +//! serialize on write, no `Archived` type on read. +//! - **Paging and links.** One contiguous `Vec` and an index into it, so +//! no page ids, no offsets, no empty-link registry and nothing for vacuum to +//! do. +//! - **The lock map.** Mutation takes `&mut self`. That is what makes it +//! single-writer, and what makes it as fast as the `Vec` it is. +//! - **Change-data-capture.** CDC exists to feed persistence and vacuum. With +//! neither, it is pure cost. +//! - **The async surface.** `insert` and friends are synchronous, because +//! nothing here can queue. That also takes the executor off the hot path. +//! +//! # What it keeps, and what it deliberately does not +//! +//! It keeps the declaration and the method names. `insert`, `upsert`, +//! `select`, `select_all` and `delete` are the same words doing the same job, +//! so the two tables read alike and a reader carries one vocabulary. +//! +//! It does **not** keep the signatures, and that is the safety property here +//! rather than an omission. This comment used to claim a table "can be moved +//! between the two by changing which macro is called", which is false and was +//! advertising the one hazard worth avoiding: a swap that changes a table's +//! concurrency and durability guarantees while every call site still compiles. +//! +//! Every call site breaks instead: +//! +//! | | `worktable!` | `worktable!` with `vec: true` | +//! |---|---|---| +//! | `insert` | `async fn(&self, Row) -> Result` | `fn(&mut self, Row) -> Result<(), Row>` | +//! | `upsert` | `async fn(&self, Row) -> Result<(), WorkTableError>` | `fn(&mut self, Row)` | +//! | `delete` | `async fn(&self, Pk) -> Result<(), WorkTableError>` | `fn(&mut self, &Pk) -> Option` | +//! | `select` | `fn(&self, Pk) -> Option`, cloned out | `fn(&self, &Pk) -> Option<&Row>`, borrowed | +//! +//! A missing `.await`, `&self` against `&mut self`, an owned row against a +//! borrowed one: the compiler rejects the swap four different ways before it +//! can silently weaken anything. The guarantees differ, so the types differ. +//! That is what makes the difference safe to live with, not the fact that this +//! is a separate macro. A second macro, or a second crate, would relabel the +//! divergence without catching it. +//! +//! **And the index backend.** This is not a detail. The first version of this +//! generator hardcoded `BTreeMap` and accepted `using arctic` without +//! honouring it, which is the worst of both: a stated choice silently dropped, +//! and the slower structure chosen on the caller's behalf. `worktable-vec` +//! measures the same two arms over a five-field row and one million point +//! lookups and reports 32.34 ns/query for `Vec + BTreeMap` against 5.25 for +//! `Vec + Arctic`. Defaulting to `BTreeMap` gave away roughly six times the +//! lookup, for a macro whose entire claim is that it costs what a `Vec` costs. +//! +//! So the default here is Arctic, which is `worktable!`'s default, and `using` +//! selects as it does there: +//! +//! | clause | this macro emits | non-unique | +//! |---|---|---| +//! | absent, or `using arctic` | `ArcticIndex` | `ArcticMultiIndex` | +//! | `using worktables_index` | WTI's `IndexMap` | refused, no shared multimap trait | +//! | `using congee` | `CongeeIndex` | refused, congee has no multimap | +//! | `using indexset` | `BTreeMap`, the plain ordered map | `BTreeMap>` | +//! | `using fxhash` | `FxHashMap`, **no ranges** | `FxHashMap>` | +//! +//! `worktable!` additionally demands an explicit `persist` before it accepts +//! congee, because congee behaves differently persisted and the author has to +//! say which they meant. This macro has no persistence at all, so the question +//! is already answered and the rule does not carry over. Congee was refused +//! here for a while on the strength of that rule's name rather than its +//! reason. +//! +//! `using fxhash` is the only one of these that is not a tree, and it is the +//! only one this macro can offer: `UniqueIndex` requires `range_values` and +//! `range_links`, which a hash map cannot answer, and a paged table both ranges +//! and writes its indexes to disk as sorted pages. This generator asks its +//! index for point operations and one order-independent walk, so it is the one +//! place the trait is not in the way. A table using it gets no `range` and no +//! `range_by_`, by omission rather than by panic. +//! +//! `using indexset` is the way to ask for `BTreeMap` deliberately. The reason +//! that used to be given for it — that `delete` shifts every position above the +//! hole and a `BTreeMap` shifts in place where an ART reinserts — no longer +//! applies: a delete ghosts its slot and shifts nothing. What is left is +//! `compact`, which renumbers once, and there the same asymmetry holds at a +//! fraction of the frequency. + +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; +use syn::Ident; +use worktable_dsl::{Columns, IndexBackend}; + +use crate::generators::index_backend::primitive_name; + +// Paths are written through `worktable::prelude`, never as bare `alloc::` or +// `std::`. The macro expands in the consumer's crate, so anything it names has +// to resolve there: emitting `alloc::` requires the consumer to have declared +// `extern crate alloc`, and emitting a crate name makes that crate part of this +// macro's contract. The same mistake has been made here with `tokio::`, +// `futures::` and `worktable::prelude::rkyv::`. + +/// What a resolved backend actually stores. +/// +/// Arctic and WTI collapse into one arm for unique indexes because both +/// implement `UniqueIndex`, so the emitted calls are identical and only the +/// type name differs. They separate again for non-unique ones, where the two +/// multimaps do not share a trait. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Repr { + Arctic, + Wti, + Congee, + Ordered, + /// A hash map, reached through inherent methods like `Ordered`. + /// + /// It cannot implement `UniqueIndex`, because that trait requires + /// `range_values` and `range_links`. That is the whole reason this backend + /// exists only here: the `vec: true` generator is the one that asks its + /// index for point operations and an order-independent walk, and nothing + /// else. + Fx, +} + +impl Repr { + /// Does this store positions through the `UniqueIndex` trait rather than + /// through inherent `BTreeMap` methods? + fn is_trait_backed(self) -> bool { + matches!(self, Repr::Arctic | Repr::Wti | Repr::Congee) + } + + /// Can this backend answer an ordered scan? + /// + /// False for exactly one backend today. It decides whether `range` and + /// `range_by_` are emitted at all: a hash map cannot answer a range, and a + /// method that existed and returned the wrong thing, or panicked, would be + /// the silent no-op this crate refuses everywhere else. + fn is_ordered(self) -> bool { + self != Repr::Fx + } + + /// Has this backend a multimap for a non-unique index? + fn has_multimap(self) -> bool { + matches!(self, Repr::Arctic | Repr::Ordered | Repr::Fx) + } + + /// The `using` spelling, for error messages. + fn name(self) -> &'static str { + match self { + Repr::Arctic => "arctic", + Repr::Wti => "worktables_index", + Repr::Congee => "congee", + Repr::Ordered => "indexset", + Repr::Fx => "fxhash", + } + } +} + +/// Resolve a declared backend, refusing what this table cannot honour. +/// +/// `what` names the index in the error, because a table with four of them +/// otherwise reports a refusal with nothing to attach it to. +fn resolve(backend: IndexBackend, ty: &TokenStream, span: proc_macro2::Span, what: &str) -> syn::Result { + let repr = match backend { + IndexBackend::Arctic => Repr::Arctic, + IndexBackend::WorktablesIndex => Repr::Wti, + IndexBackend::Congee => Repr::Congee, + IndexBackend::Indexset => Repr::Ordered, + IndexBackend::FxHash => Repr::Fx, + }; + // `worktable!` additionally requires `persist` to be stated before it will + // accept congee, because congee behaves differently persisted and the + // author has to say which they meant. This macro has no persistence at + // all, so that question is already answered and the rule does not carry + // over. It was refused here for a while on the strength of the rule's + // name rather than its reason. + let Some(supported) = worktable_dsl::validate::supported_key_types(backend) else { + return Ok(repr); + }; + if primitive_name(ty) + .as_deref() + .is_some_and(|name| supported.contains(&name)) + { + return Ok(repr); + } + Err(syn::Error::new( + span, + format!( + "`using {}` indexes {what} on one of {}, and `{ty}` is not one of them. \ + Use `using worktables_index` to index it, or `using indexset` for a plain \ + ordered map. (Type aliases cannot be resolved by the macro.)", + repr.name(), + supported.join(", ") + ), + )) +} + +/// The stored type for a unique key-to-position map. +fn unique_type(repr: Repr, ty: &TokenStream) -> TokenStream { + match repr { + Repr::Arctic => quote! { worktable::prelude::ArcticIndex<#ty, u64> }, + Repr::Wti => quote! { worktable::prelude::IndexMap<#ty, u64> }, + Repr::Congee => quote! { worktable::prelude::CongeeIndex<#ty, u64> }, + Repr::Ordered => quote! { worktable::prelude::BTreeMap<#ty, usize> }, + Repr::Fx => quote! { worktable::prelude::FxHashMap<#ty, usize> }, + } +} + +/// The stored type for a non-unique key-to-positions map. +/// +/// Only two backends reach here; `has_multimap` refuses the others first. +fn multi_type(repr: Repr, ty: &TokenStream) -> TokenStream { + match repr { + Repr::Arctic => quote! { worktable::prelude::ArcticMultiIndex<#ty, u64> }, + Repr::Ordered => quote! { worktable::prelude::BTreeMap<#ty, worktable::prelude::Vec> }, + Repr::Fx => quote! { worktable::prelude::FxHashMap<#ty, worktable::prelude::Vec> }, + Repr::Wti | Repr::Congee => { + quote! { compile_error!("unreachable: this backend has no multimap and was refused during resolution") } + } + } +} + +/// Congee packs a key into one `usize`, so a `u64` key needs a 64-bit target. +/// +/// `impl CongeeKey for u64` is itself behind that cfg, so without this the +/// failure on a 32-bit target is an unsatisfied trait bound on a type the +/// author never wrote. `worktable!` emits the same guard for the same reason. +fn congee_width_guard(repr: Repr, ty: &TokenStream) -> TokenStream { + if repr == Repr::Congee && primitive_name(ty).as_deref() == Some("u64") { + quote! { + #[cfg(not(target_pointer_width = "64"))] + compile_error!("`using congee` with a `u64` key requires a 64-bit target"); + } + } else { + quote! {} + } +} + +// The five operations a unique map has to answer, emitted for whichever +// representation was resolved. `map` is the field access, already qualified. + +fn unique_contains(repr: Repr, map: &TokenStream, key: &TokenStream) -> TokenStream { + if repr.is_trait_backed() { + quote! { worktable::prelude::UniqueIndex::contains_key(&#map, #key) } + } else { + quote! { #map.contains_key(#key) } + } +} + +fn unique_get(repr: Repr, map: &TokenStream, key: &TokenStream) -> TokenStream { + if repr.is_trait_backed() { + quote! { worktable::prelude::UniqueIndex::get_value(&#map, #key).map(|at| at as usize) } + } else { + quote! { #map.get(#key).copied() } + } +} + +fn unique_insert(repr: Repr, map: &TokenStream, key: &TokenStream, at: &TokenStream) -> TokenStream { + if repr.is_trait_backed() { + quote! { let _ = worktable::prelude::UniqueIndex::insert_value(&#map, #key, #at as u64); } + } else { + quote! { #map.insert(#key, #at); } + } +} + +/// Insert unless the key is already there, in one traversal. True means it was. +/// +/// `insert` used to ask `contains_key` and then `insert_value`, which is two +/// full traversals of the index on every single insert, and it was the whole +/// of the macro's overhead: a hand-written `Vec` plus `ArcticIndex` ran 5.9 ms +/// over 200,000 rows, the same code with a `contains_key` guard added ran +/// 7.7 ms, and the generated table ran 7.7 ms. Both backends can answer the +/// question and do the work at once, so they do. +fn unique_insert_checked(repr: Repr, map: &TokenStream, key: &TokenStream, at: &TokenStream) -> TokenStream { + if repr.is_trait_backed() { + quote! { + worktable::prelude::UniqueIndex::insert_value_checked(&#map, #key, #at as u64).is_none() + } + } else if repr == Repr::Fx { + quote! { + match #map.entry(#key) { + worktable::prelude::HashMapEntry::Occupied(_) => true, + worktable::prelude::HashMapEntry::Vacant(slot) => { + slot.insert(#at); + false + } + } + } + } else { + quote! { + match #map.entry(#key) { + worktable::prelude::BTreeMapEntry::Occupied(_) => true, + worktable::prelude::BTreeMapEntry::Vacant(slot) => { + slot.insert(#at); + false + } + } + } + } +} + +fn unique_remove(repr: Repr, map: &TokenStream, key: &TokenStream) -> TokenStream { + if repr.is_trait_backed() { + quote! { worktable::prelude::UniqueIndex::remove_value(&#map, #key).map(|(_, at)| at as usize) } + } else { + quote! { #map.remove(#key) } + } +} + +/// Positions whose keys fall inside `bounds`, in key order. +/// +/// Every backend this macro can resolve is an ordered tree — the two ARTs, +/// WTI's B-tree and a plain `BTreeMap` — so this is not a capability some of +/// them have and others emulate. `UniqueIndex` already requires +/// `range_links`, which means the operation was always there and only the +/// generated table declined to expose it. +/// +/// What a range costs that a point lookup does not is the row fetch: the +/// positions come out in key order and the rows they name are scattered +/// through the vector, so a long range is a walk of random accesses rather +/// than a sequential read. +fn unique_range(repr: Repr, map: &TokenStream, bounds: &TokenStream) -> TokenStream { + if repr.is_trait_backed() { + quote! { + worktable::prelude::UniqueIndex::range_links(&#map, #bounds).map(|at| at as usize) + } + } else { + quote! { #map.range(#bounds).map(|(_, at)| *at) } + } +} + +/// Point every entry at where its row moved to, after a compaction. +/// +/// Compaction is the only thing that moves a row, and it never removes an +/// index entry: a ghosted row left its indexes at the moment it was deleted, +/// so every entry still here names a row that survives. That is why this is a +/// rewrite of values and not a rebuild, and why it can keep the maps +/// themselves — replacing them with `Default::default()` would silently +/// discard a `with_node_size` the caller asked for. +fn unique_renumber(repr: Repr, map: &TokenStream, moved: &TokenStream) -> TokenStream { + if repr.is_trait_backed() { + quote! { + let entries: worktable::prelude::Vec<_> = + worktable::prelude::UniqueIndex::iter_values(&#map).collect(); + for (key, position) in entries { + let to = #moved[position as usize]; + if to != position { + let _ = worktable::prelude::UniqueIndex::insert_value(&#map, key, to); + } + } + } + } else { + quote! { + for position in #map.values_mut() { + *position = #moved[*position] as usize; + } + } + } +} + +pub fn expand( + name: Ident, + columns: Columns, + queries: Option<&worktable_dsl::model::Queries>, +) -> syn::Result { + if columns.primary_keys.len() != 1 { + return Err(syn::Error::new( + name.span(), + "`vec: true` takes a single-column primary key. A composite key needs a tuple key \ + type, which is the machinery this storage exists to avoid.", + )); + } + if !columns.columnar_fields.is_empty() || !columns.columnar_indexes.is_empty() { + return Err(syn::Error::new( + name.span(), + "`vec: true` does not support columnar fields. Columnar storage is a paging \ + feature and this table has no pages.", + )); + } + + // `{Name}Row` and `{Name}WorkTable`, the same names the paged table gets. + // + // This was `{Name}VecRow` and `{Name}VecTable` while a second macro + // generated it, because two macros naming one table collided on the row. + // One macro and a `storage:` key removes the collision at the source, so + // there is no reason left to make a caller learn a parallel vocabulary: + // the storage is a property of the declaration, not of every identifier + // that comes out of it. + let row_ident = Ident::new(&format!("{name}Row"), name.span()); + let table_ident = Ident::new(&format!("{name}WorkTable"), name.span()); + + let pk = columns.primary_keys.first().expect("checked above").clone(); + let pk_type = columns + .columns_map + .get(&pk) + .expect("the primary key is a column") + .clone(); + + let pk_repr = resolve(columns.primary_index_backend, &pk_type, pk.span(), "the primary key")?; + let pk_map_type = unique_type(pk_repr, &pk_type); + let mut width_guards = vec![congee_width_guard(pk_repr, &pk_type)]; + + // WTI is the only backend with a node-size knob; arctic and congee have no + // node-size concept at all. A constructor that took one on a table with no + // WTI index would be a silent no-op, which this crate refuses everywhere + // else, so it is emitted only when there is something for it to set. + let pk_is_wti = matches!(pk_repr, Repr::Wti); + + let field_names: Vec<_> = columns.columns_map.keys().cloned().collect(); + let field_types: Vec<_> = columns.columns_map.values().cloned().collect(); + + // Secondary indexes, as positions. Unique ones keep the reject. + let mut index_fields = Vec::new(); + let mut index_map_types = Vec::new(); + let mut index_columns = Vec::new(); + let mut index_reprs = Vec::new(); + let mut index_unique = Vec::new(); + for (index_name, index) in &columns.indexes { + let column = &index.field; + let ty = columns + .columns_map + .get(column) + .ok_or_else(|| syn::Error::new(index_name.span(), format!("no column `{column}`")))?; + // `columns.indexes` is keyed by the indexed *column*; the name the + // author wrote is `index.name`. Errors quote that one, because it is + // the token they can go and edit. + let declared = &index.name; + let repr = resolve(index.backend, ty, index_name.span(), &format!("`{declared}`"))?; + if !index.is_unique && !repr.has_multimap() { + return Err(syn::Error::new( + index_name.span(), + format!( + "the non-unique index `{declared}` cannot use `{}`: congee has no multimap at \ + all, and WTI's does not share a trait with Arctic's, so this would be a second \ + code path with no measurement behind it. Use arctic (the default), \ + `using indexset`, or declare the index `unique`.", + repr.name() + ), + )); + } + width_guards.push(congee_width_guard(repr, ty)); + index_fields.push(Ident::new(&format!("{index_name}_map"), index_name.span())); + index_map_types.push(if index.is_unique { + unique_type(repr, ty) + } else { + multi_type(repr, ty) + }); + index_columns.push(column.clone()); + index_reprs.push(repr); + index_unique.push(index.is_unique); + } + + // Per-index statement fragments, so the method bodies below stay readable. + let mut index_reject_duplicate = Vec::new(); + let mut index_validate_replacement = Vec::new(); + let mut index_insert = Vec::new(); + let mut index_upsert_move = Vec::new(); + let mut index_delete_remove = Vec::new(); + let mut index_renumber = Vec::new(); + for ((field, (column, (repr, unique))), _) in index_fields + .iter() + .zip( + index_columns + .iter() + .zip(index_reprs.iter().copied().zip(index_unique.iter().copied())), + ) + .zip(0..) + { + let map = quote! { self.#field }; + let key = quote! { &row.#column }; + let owned = quote! { row.#column.clone() }; + let at = quote! { at }; + + index_reject_duplicate.push(if unique { + let contains = unique_contains(repr, &map, &key); + quote! { if #contains { return Err(row); } } + } else { + quote! {} + }); + if unique { + let owner = unique_get(repr, &map, &key); + index_validate_replacement.push(quote! { + assert!( + #owner.is_none_or(|owner| owner == at), + "mutation gave a row a unique secondary key another row already holds" + ); + }); + } + + index_insert.push(if unique { + unique_insert(repr, &map, &owned, &at) + } else { + match repr { + Repr::Arctic => quote! { #map.insert_pair(#owned, at as u64); }, + _ => quote! { #map.entry(#owned).or_default().push(at); }, + } + }); + + // On upsert the row keeps its position and only its key changes, so + // the old pair comes out and the new one goes in at the same `at`. + // + // The old key is bound to a local first. Reading it inline would + // borrow the whole table (`row_at` takes `&self`) while the map call + // it feeds wants `&mut` on a field, and the two-phase borrow that let + // `self.rows[at]` work here does not reach through a method. + let was = Ident::new(&format!("was_{field}_key"), field.span()); + index_upsert_move.push(if unique { + let remove = unique_remove(repr, &map, "e! { &#was }); + let insert = unique_insert(repr, &map, &owned, &at); + quote! { + let #was = self.row_at(at).#column.clone(); + let _ = #remove; + #insert + } + } else { + match repr { + Repr::Arctic => quote! { + let #was = self.row_at(at).#column.clone(); + let _ = #map.remove_pair(&#was, &(at as u64)); + #map.insert_pair(#owned, at as u64); + }, + _ => quote! { + let #was = self.row_at(at).#column.clone(); + if let Some(positions) = #map.get_mut(&#was) { + positions.retain(|p| *p != at); + if positions.is_empty() { + #map.remove(&#was); + } + } + #map.entry(#owned).or_default().push(at); + }, + } + }); + + index_delete_remove.push(if unique { + let key = quote! { &row.#column }; + let remove = unique_remove(repr, &map, &key); + quote! { let _ = #remove; } + } else { + match repr { + Repr::Arctic => quote! { let _ = #map.remove_pair(&row.#column, &(at as u64)); }, + _ => quote! { + if let Some(positions) = #map.get_mut(&row.#column) { + positions.retain(|p| *p != at); + if positions.is_empty() { + #map.remove(&row.#column); + } + } + }, + } + }); + + index_renumber.push(if unique { + unique_renumber(repr, &map, "e! { moved }) + } else { + match repr { + Repr::Arctic => quote! { + let pairs: worktable::prelude::Vec<_> = #map.iter().collect(); + for (key, position) in pairs { + let to = moved[position as usize]; + if to != position { + let _ = #map.remove_pair(&key, &position); + #map.insert_pair(key, to); + } + } + }, + _ => quote! { + for positions in #map.values_mut() { + for position in positions.iter_mut() { + *position = moved[*position] as usize; + } + } + }, + } + }); + } + + let select_by: Vec<_> = columns + .indexes + .iter() + .zip(index_reprs.iter().copied()) + .map(|((index_name, index), repr)| { + let column = &index.field; + let fn_name = Ident::new(&format!("select_by_{column}"), index_name.span()); + let field = Ident::new(&format!("{index_name}_map"), index_name.span()); + let map = quote! { self.#field }; + let ty = columns.columns_map.get(column).expect("checked above"); + if index.is_unique { + let get = unique_get(repr, &map, "e! { key }); + // Emitted only for an ordered backend. `using fxhash` gets the + // point lookup and no range, so a caller who needs one gets a + // missing method at the call site rather than a method that + // exists and cannot answer. + let range_by = if repr.is_ordered() { + let range_fn = Ident::new(&format!("range_by_{column}"), index_name.span()); + let range = unique_range(repr, &map, "e! { bounds }); + quote! { + /// Every row whose indexed value falls inside `bounds`, + /// in that value's order. + /// + /// Free for the same reason the primary-key range is: + /// this index is an ordered tree and was already + /// answering ranges, so the walk is the index's own and + /// the only added work is the row fetch each position + /// names. + pub fn #range_fn<'a, R>( + &'a self, + bounds: R, + ) -> impl DoubleEndedIterator + 'a + where + R: core::ops::RangeBounds<#ty> + 'a, + { + #range.map(|at| self.row_at(at)) + } + } + } else { + quote! {} + }; + quote! { + /// The row this key indexes, if any. + pub fn #fn_name(&self, key: &#ty) -> Option<&#row_ident> { + #get.map(|at| self.row_at(at)) + } + + #range_by + } + } else { + let positions = match repr { + Repr::Arctic => quote! { + let mut positions: worktable::prelude::Vec = + #map.get(key).map(|(_, at)| at as usize).collect(); + // Arctic orders pairs by value, which is position, which + // is insertion order. Sorting states that rather than + // relying on it. + positions.sort_unstable(); + }, + _ => quote! { + let positions: worktable::prelude::Vec = + #map.get(key).map(|found| found.clone()).unwrap_or_default(); + }, + }; + quote! { + /// Every row this key indexes, in insertion order. + pub fn #fn_name(&self, key: &#ty) -> Vec<&#row_ident> { + #positions + positions.into_iter().map(|at| self.row_at(at)).collect() + } + } + } + }) + .collect(); + + // Per-index fragments for `update`: what the key was before the edit, and + // the repair when it changed. A non-unique index moves one pair; a unique + // one re-keys a single entry. + let index_before: Vec = index_fields + .iter() + .map(|field| Ident::new(&format!("was_{field}"), field.span())) + .collect(); + let mut index_repair = Vec::new(); + for (((field, column), repr), unique) in index_fields + .iter() + .zip(index_columns.iter()) + .zip(index_reprs.iter().copied()) + .zip(index_unique.iter().copied()) + { + let map = quote! { self.#field }; + let before = Ident::new(&format!("was_{field}"), field.span()); + // Bound to a local for the same borrow reason `index_upsert_move` + // binds its old key: the map calls below take `&mut` on a field, and + // `row_at` borrows the whole table. + let now = quote! { now }; + let repair = if unique { + let remove = unique_remove(repr, &map, "e! { &#before }); + let insert = unique_insert(repr, &map, &now, "e! { at }); + quote! { let _ = #remove; #insert } + } else { + match repr { + Repr::Arctic => quote! { + let _ = #map.remove_pair(&#before, &(at as u64)); + #map.insert_pair(#now, at as u64); + }, + _ => quote! { + if let Some(positions) = #map.get_mut(&#before) { + positions.retain(|p| *p != at); + if positions.is_empty() { + #map.remove(&#before); + } + } + #map.entry(#now).or_default().push(at); + }, + } + }; + index_repair.push(quote! { + if self.row_at(at).#column != #before { + let now = self.row_at(at).#column.clone(); + #repair + } + }); + } + + let pk_map = quote! { self.by_pk }; + let at_expr = quote! { at }; + let pk_insert_checked = unique_insert_checked(pk_repr, &pk_map, "e! { row.#pk.clone() }, &at_expr); + let pk_get_for_select = unique_get(pk_repr, &pk_map, "e! { key }); + let pk_get_for_upsert = unique_get(pk_repr, &pk_map, "e! { &row.#pk }); + let pk_remove = unique_remove(pk_repr, &pk_map, "e! { key }); + let pk_get_for_moved_row = unique_get(pk_repr, &pk_map, "e! { &now_pk }); + let pk_remove_old = { + let remove = unique_remove(pk_repr, &pk_map, "e! { &was_pk }); + quote! { let _ = #remove; } + }; + let pk_reinsert_moved = unique_insert(pk_repr, &pk_map, "e! { now_pk }, "e! { at }); + let pk_renumber = unique_renumber(pk_repr, &pk_map, "e! { moved }); + // What `with_capacity` can actually reserve. + // + // For a tree this is nothing: `arctic-prealloc` measured the ceiling on + // pooling Arctic's node allocation at **0.92x**, below one, because free + // allocation changes where nodes land and sequential order is worse for a + // tree walked in key order. There is no reserve to offer and nothing would + // be gained by inventing one. + // + // A hash map is the opposite case and the only one: one growing buffer + // with a doubling sequence, which is exactly what `with_capacity` deletes. + // Measured hand-written on this shape, reserving is worth a further 3.1x to + // 5.1x on build beyond the hash map itself. So the reserve is emitted for + // `fxhash` and for nothing else, which is not a special case so much as the + // only backend that has an answer. + let pk_capacity = if pk_repr == Repr::Fx { + quote! { + by_pk: <#pk_map_type>::with_capacity_and_hasher( + capacity, + worktable::prelude::FxBuildHasher, + ), + } + } else { + quote! {} + }; + let index_capacity: Vec<_> = index_fields + .iter() + .zip(index_map_types.iter()) + .zip(index_reprs.iter().copied()) + .filter(|((_, _), repr)| *repr == Repr::Fx) + .map(|((field, ty), _)| { + quote! { + #field: <#ty>::with_capacity_and_hasher( + capacity, + worktable::prelude::FxBuildHasher, + ), + } + }) + .collect(); + + // `range` exists only when the primary index can answer one. On a table + // `using fxhash` the method is simply not there, so a caller who needs a + // range gets "no method named `range`" at their own call site instead of a + // method that compiles and cannot do the job. + let pk_range_fn = if pk_repr.is_ordered() { + let pk_range = unique_range(pk_repr, &pk_map, "e! { bounds }); + quote! { + /// Every live row whose primary key falls inside `bounds`, in key + /// order. + /// + /// This costs nothing to provide and was simply never exposed. The + /// ordered backends are trees, `UniqueIndex` already requires + /// `range_links`, and the index was answering ranges the whole + /// time. + /// + /// What it is not is a sorted vector. The keys come out in order + /// and the rows they name are wherever insertion put them, so a + /// long range is a sequence of random accesses into the row + /// vector. Ordered, correct, and not sequential. + /// + /// Not emitted for `using fxhash`, which has no order to walk. + /// + /// ```ignore + /// for row in table.range(10..20) { .. } + /// for row in table.range(..).rev() { .. } + /// ``` + pub fn range<'a, R>( + &'a self, + bounds: R, + ) -> impl DoubleEndedIterator + 'a + where + R: core::ops::RangeBounds<#pk_type> + 'a, + { + #pk_range.map(|at| self.row_at(at)) + } + } + } else { + quote! {} + }; + + // rkyv's derives only when the table can be written out. They are not free + // to a caller who never persists: an `Archived` type per row, a resolver + // per row, and the compile time to produce both. + // + // The crate path is `worktable::prelude::rkyv`, and `#[rkyv(crate = ..)]` + // redirects the derive's own generated paths to it. Emitting a bare `rkyv` + // would make the consumer's manifest part of this macro's contract, which + // is the leak `worktable!` still has. + // Always, not behind a flag. `persist` is refused on this table, so there + // is nothing left to gate them with, and the alternative is a third key. + // Measured at 20 tables of five columns: 305 ms without, 470 ms with, so + // about 8 ms a table. Real, and not worth a key. + // The node-size constructor. Emitted only when there is a WTI index to set + // it on, so it can never be a knob that does nothing. + let with_node_size = if pk_is_wti || index_reprs.iter().any(|r| matches!(r, Repr::Wti)) { + let pk_init = if pk_is_wti { + quote! { by_pk: <#pk_map_type>::with_maximum_node_size(node_size), } + } else { + quote! { by_pk: Default::default(), } + }; + let index_inits: Vec<_> = index_fields + .iter() + .zip(index_map_types.iter()) + .zip(index_reprs.iter()) + .map(|((field, ty), repr)| { + if matches!(repr, Repr::Wti) { + quote! { #field: <#ty>::with_maximum_node_size(node_size), } + } else { + quote! { #field: Default::default(), } + } + }) + .collect(); + quote! { + /// A table whose `worktables_index` indexes use `node_size` as + /// their leaf width, instead of the default 1,024. + /// + /// The width is a call-site decision rather than a declaration one, + /// because the right value depends on the workload and not on the + /// schema: the same table read-mostly in one process and written + /// hard in another wants different numbers, and a declaration can + /// only say one thing. + /// + /// Measured at a million shuffled keys + /// (`perf-benchmarks/benchmarks/wti-node-size.rs`): + /// + /// | width | insert | lookup | drop | + /// |---:|---:|---:|---:| + /// | 128 | 127.36 ns | 134.58 ns | 555.3 us | + /// | 256 | 128.80 ns | 129.17 ns | 290.5 us | + /// | 1,024 (default) | 200.34 ns | 124.58 ns | 85.7 us | + /// | 16,384 | 1,445.89 ns | 116.67 ns | 9.9 us | + /// + /// Narrow is much better for writing, slightly worse for reading, + /// and worse for teardown. 256 is the write-heavy pick; the default + /// stays 1,024 because a wrong guess is worse than no guess, and + /// only the call site knows which way this table leans. + /// + /// **Nothing is capped.** The width is the leaf size a node splits + /// at, not a limit on rows: the tree grows by adding nodes exactly + /// as it does at the default, so an undersized guess costs + /// performance and never correctness. + /// + /// Emitted only for tables that have at least one + /// `using worktables_index`, so it is never a knob with nothing to + /// turn. + #[must_use] + pub fn with_node_size(node_size: usize) -> Self { + Self { + rows: worktable::prelude::Vec::new(), + live: 0, + #pk_init + #(#index_inits)* + } + } + + /// Both knobs at once: rows sized for `capacity`, WTI leaves at + /// `node_size`. + #[must_use] + pub fn with_capacity_and_node_size(capacity: usize, node_size: usize) -> Self { + let mut table = Self::with_node_size(node_size); + table.rows = worktable::prelude::Vec::with_capacity(capacity); + table + } + } + } else { + quote! {} + }; + + let row_derives = { + quote! { + #[derive( + Clone, + Debug, + PartialEq, + worktable::prelude::rkyv::Archive, + worktable::prelude::rkyv::Serialize, + worktable::prelude::rkyv::Deserialize, + )] + #[rkyv(crate = worktable::prelude::rkyv)] + } + }; + + let hydrate = { + quote! { + /// Every row as pages, ready to be written somewhere. + /// + /// A page stands alone, so damage is local to one page and an + /// append does not rewrite the file. + /// + /// # Errors + /// + /// [`worktable::prelude::RowTooLarge`] when one row's archive does + /// not fit a page body. Nothing is produced in that case, rather + /// than a file that will not load. + /// + /// Ghosted slots are not written, so a file never carries a row + /// that was deleted. Collecting the live rows to do that costs one + /// clone each, which is real and is dwarfed by the archive write + /// that follows it. + pub fn unload(&self) -> Result, worktable::prelude::RowTooLarge> { + let live: worktable::prelude::Vec<#row_ident> = + self.rows.iter().flatten().cloned().collect(); + worktable::prelude::to_pages(&live) + } + + /// Live rows from `first` onward, numbered from `pages_before`. + /// + /// `first` counts live rows in insertion order, skipping ghosts. + /// Pass the existing byte length divided by the codec PAGE_SIZE + /// as `pages_before`. The existing terminal page is not rewritten. + /// Use this only for newly inserted rows: updates, deletes or + /// changes before the saved cursor require a full snapshot. + /// + /// # Errors + /// + /// Refuses oversized rows or page-number overflow. + pub fn unload_appending(&self, first: usize, pages_before: u32) + -> Result, worktable::vec_hydrate::UnloadError> + { + let live: worktable::prelude::Vec<#row_ident> = + self.rows.iter().flatten().skip(first).cloned().collect(); + worktable::vec_hydrate::to_pages_at(&live, pages_before) + } + + /// A table back from pages, with every index rebuilt. + /// + /// The indexes are not stored. They are positions into the row + /// vector, so they are cheaper to rebuild on load than to write, + /// validate and keep consistent with the rows on disk. + /// + /// # Errors + /// + /// [`worktable::prelude::LoadError`], naming the page that went + /// wrong. A different row type's file is refused by its + /// fingerprint rather than read as debris. + pub fn load(bytes: &[u8]) -> Result { + let rows: worktable::prelude::Vec<#row_ident> = worktable::prelude::from_pages(bytes)?; + let mut table = Self::with_capacity(rows.len()); + for row in rows { + // A duplicate key in a loaded file is a corrupt file, not a + // caller error, and `insert` is the only thing that builds + // every index. Refusing here would be better still, but + // `LoadError` describes bytes rather than rows and there is + // no variant that could honestly say this. + let _ = table.insert(row); + } + Ok(table) + } + } + }; + + let (query_structs, query_methods) = gen_queries(queries, &pk, &pk_type, &columns, &index_columns, &index_unique)?; + + Ok(quote! { + #(#width_guards)* + + #(#query_structs)* + + #row_derives + pub struct #row_ident { + #(pub #field_names: #field_types,)* + } + + /// A `Vec`-backed table with the same surface as the generated `WorkTable`. + /// + /// Single-writer by construction: every mutation takes `&mut self`. + /// + /// # Ghosts + /// + /// A slot is `None` once its row is deleted. That is the paged table's + /// model applied to a vector, and it is what makes `delete` O(1) + /// instead of O(rows + index): closing the hole would mean a memmove + /// of every row above it *and* a rewrite of every index entry above + /// it, which measured 21 milliseconds per delete at a million rows. + /// + /// The cost is that ghosts accumulate and nothing reclaims them until + /// [`Self::compact`] is called, exactly as a paged table accumulates + /// them until vacuum runs. [`Self::ghost_count`] and [`Self::slots`] + /// are there so a caller can decide when that is worth doing. + #[derive(Debug, Default)] + pub struct #table_ident { + /// Slots. `None` is a ghost: a row that was deleted and whose + /// position no index names any more. + rows: worktable::prelude::Vec>, + /// Live rows, so `len` does not walk the vector counting them. + live: usize, + /// Primary key to position. The lookup a bare `Vec` does linearly. + by_pk: #pk_map_type, + #(#index_fields: #index_map_types,)* + } + + impl #table_ident { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// A table whose row vector can hold `capacity` rows without + /// reallocating. + /// + /// Only the rows are sized. The indexes are trees and have no + /// equivalent knob, so an accurate capacity removes the row + /// vector's growth entirely and leaves theirs alone. + #[must_use] + pub fn with_capacity(capacity: usize) -> Self { + Self { + rows: worktable::prelude::Vec::with_capacity(capacity), + #pk_capacity + #(#index_capacity)* + ..Self::default() + } + } + + #with_node_size + + /// How many rows fit before the row vector grows again. + #[must_use] + pub fn capacity(&self) -> usize { + self.rows.capacity() + } + + /// Make room for `additional` more rows. + pub fn reserve(&mut self, additional: usize) { + self.rows.reserve(additional); + } + + /// Every live row, in insertion order. + /// + /// Ghosted slots are skipped, so this yields [`Self::len`] rows + /// and not [`Self::slots`] of them. + pub fn iter(&self) -> impl Iterator { + self.rows.iter().flatten() + } + + /// The live rows, leaving the indexes and the ghosts behind. + /// + /// For handing the data to something that does not want a table. + /// The indexes are positions into this vector and mean nothing + /// without it, so they are dropped rather than returned. + #[must_use] + pub fn into_rows(self) -> worktable::prelude::Vec<#row_ident> { + self.rows.into_iter().flatten().collect() + } + + /// The row at a position an index gave us. + /// + /// # Panics + /// + /// If the slot is a ghost. Every index entry is removed the moment + /// its row is deleted, so a position that came out of an index + /// always names a live row; reaching this panic means an index and + /// the vector disagree, which is a bug in this macro rather than + /// in a caller. + #[inline] + fn row_at(&self, at: usize) -> &#row_ident { + self.rows[at] + .as_ref() + .expect("an index position always names a live row") + } + + /// Row bytes plus index bytes. + /// + /// The same name and the same intent as the paged table's + /// `used_bytes`, so a partitioned router can total either payload + /// without knowing which it holds. + /// + /// Rows are counted as `len * size_of::()`: the inline row + /// only. A column that owns a heap allocation, a `String` most + /// obviously, has its buffer counted by neither this nor the paged + /// table's equivalent. The indexes are counted through `MemStat`, + /// which is where most of the cost is at small row counts: arctic + /// holds about 600 bytes per 24-byte row at 64 rows and does not + /// settle until a thousand. + /// + /// Slots are counted, not rows: a ghost still occupies its slot + /// until [`Self::compact`] runs, and an `Option` is what a + /// slot costs. For a row with a spare bit pattern that is the same + /// as the row; for one with none it is the row plus its alignment. + #[must_use] + pub fn used_bytes(&self) -> u64 { + let rows = self.rows.len() * core::mem::size_of::>(); + let indexes = worktable::prelude::MemStat::heap_size(&self.by_pk) + #(+ worktable::prelude::MemStat::heap_size(&self.#index_fields))*; + (rows + indexes) as u64 + } + + /// Live rows. + #[must_use] + pub fn len(&self) -> usize { + self.live + } + + /// Slots, live and ghosted together. + /// + /// The length of the underlying vector, which is what memory is + /// proportional to and what a range or a scan walks. + #[must_use] + pub fn slots(&self) -> usize { + self.rows.len() + } + + /// Deleted rows whose slots are still held. + /// + /// `slots() - len()`. A caller watching this decides when + /// [`Self::compact`] is worth its cost, the same judgement a + /// paged table makes about vacuum. + #[must_use] + pub fn ghost_count(&self) -> usize { + self.rows.len() - self.live + } + + /// Rows currently in the table. + /// + /// The same figure as [`Self::len`], under the name the paged + /// table uses, so a partitioned router reads either payload + /// through one call. Neither counts ghosts; [`Self::slots`] is the + /// figure that does. + #[must_use] + pub fn row_count(&self) -> usize { + self.live + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.live == 0 + } + + /// Insert, refusing a key that is already present. + /// + /// `Err` carries the row back rather than dropping it, so a caller + /// that wants `upsert` semantics on failure still has the value. + pub fn insert(&mut self, row: #row_ident) -> Result<(), #row_ident> { + // The unique secondaries are checked first and separately, + // because a rejection from one of them must not leave the + // primary key inserted. They are the only reads here that are + // not also writes. + #(#index_reject_duplicate)* + let at = self.rows.len(); + if #pk_insert_checked { + return Err(row); + } + #(#index_insert)* + self.rows.push(Some(row)); + self.live += 1; + Ok(()) + } + + /// Insert, or replace the row this key already names. + /// + /// # Panics + /// + /// Refuses a replacement whose unique secondary key belongs to + /// another row, before changing either the row or its indexes. + pub fn upsert(&mut self, row: #row_ident) { + if let Some(at) = #pk_get_for_upsert { + #(#index_validate_replacement)* + #(#index_upsert_move)* + self.rows[at] = Some(row); + return; + } + let _ = self.insert(row); + } + + /// The row this key names, if any. + #[must_use] + pub fn select(&self, key: &#pk_type) -> Option<&#row_ident> { + #pk_get_for_select.map(|at| self.row_at(at)) + } + + /// Every live row, in insertion order. + /// + /// An iterator rather than the `&[Row]` this returned before + /// ghosting: a deleted row leaves a hole, so the live rows are no + /// longer a contiguous slice and no slice could be handed back + /// without first paying the compaction this design exists to + /// defer. Call [`Self::compact`] and then [`Self::iter`] if a + /// caller genuinely needs one. + pub fn select_all(&self) -> impl Iterator { + self.rows.iter().flatten() + } + + #pk_range_fn + + #(#select_by)* + + #(#query_methods)* + + /// Edit a cloned candidate, validate unique keys, then replace + /// the row and repair its indexes. + /// + /// `worktable-vec` hands out `&mut (K, V)` for this, but only from + /// `LinearTable`, which has no indexes to invalidate. Doing that + /// here would let a caller change an indexed column and leave the + /// index pointing at a key the row no longer has, which is silent + /// and unfindable. A closure lets the table see what changed. + /// + /// Returns `false` when no row has that key, leaving the table + /// untouched. + /// + /// # Panics + /// + /// If the edit gives the row a primary or unique secondary key + /// that another row holds. The closure edits a cloned candidate; + /// a collision or a panic inside the closure leaves the stored + /// row and every index unchanged. + pub fn update(&mut self, key: &#pk_type, edit: impl FnOnce(&mut #row_ident)) -> bool { + let Some(at) = #pk_get_for_select else { + return false; + }; + // Validate a candidate before committing any row or index + // mutation. In particular, a panicking user closure must not + // leave a changed row behind stale indexes. + let mut row = self.row_at(at).clone(); + edit(&mut row); + let now_pk = row.#pk.clone(); + let taken = #pk_get_for_moved_row; + assert!( + taken.is_none_or(|other| other == at), + "update gave a row a primary key another row already holds" + ); + #(#index_validate_replacement)* + let was_pk = self.row_at(at).#pk.clone(); + #(let #index_before = self.row_at(at).#index_columns.clone();)* + self.rows[at] = Some(row); + if now_pk != was_pk { + #pk_remove_old + #pk_reinsert_moved + } + #(#index_repair)* + true + } + + #hydrate + + /// Remove the row this key names, returning it and leaving a ghost + /// where it was. + /// + /// Constant time. The row comes out of its slot, its index entries + /// come out of the indexes, and nothing else moves: no position + /// changes, so no other index entry needs touching. + /// + /// This used to close the hole with `Vec::remove`, which meant a + /// memmove of every row above it plus a rewrite of every index + /// entry above it. On a `BTreeMap` that rewrite is an in-place + /// walk; on an ART it is a read-and-reinsert of each affected + /// entry. Measured on a million-row table it cost **21 + /// milliseconds a delete**, so two hundred deletes took four + /// seconds. + /// + /// What it costs instead is a slot that stays allocated until + /// [`Self::compact`] runs, and the row order that `select_all` + /// walks getting sparser as ghosts accumulate. + pub fn delete(&mut self, key: &#pk_type) -> Option<#row_ident> { + let at = #pk_remove?; + let row = self.rows[at].take()?; + self.live -= 1; + #(#index_delete_remove)* + Some(row) + } + + /// Reclaim every ghosted slot, moving the live rows down to close + /// the holes and pointing the indexes at where they went. + /// + /// This is the vacuum a paged table runs, and it is the other half + /// of what makes `delete` constant time: the expensive work exists, + /// it is O(slots + index), and it happens once when a caller asks + /// for it rather than on every delete. + /// + /// Insertion order is preserved. Returns the number of slots + /// reclaimed, which is what [`Self::ghost_count`] read beforehand. + /// + /// The row vector keeps its capacity, so a table that churns does + /// not give memory back to the allocator and then ask for it + /// again. [`Self::shrink_to_fit`] is there for a caller that wants + /// the memory back rather than the reuse. + pub fn compact(&mut self) -> usize { + let reclaimed = self.rows.len() - self.live; + if reclaimed == 0 { + return 0; + } + + // Where each old position ends up. Ghosted slots get a value + // no index entry can name, because no index entry names them: + // a delete takes its entries out at the time it ghosts the row. + let mut moved = worktable::prelude::Vec::with_capacity(self.rows.len()); + let mut next = 0u64; + for slot in &self.rows { + moved.push(next); + if slot.is_some() { + next += 1; + } + } + + #pk_renumber + #(#index_renumber)* + + self.rows.retain(Option::is_some); + debug_assert_eq!(self.rows.len(), self.live); + reclaimed + } + + /// Give the row vector's spare capacity back to the allocator. + /// + /// Separate from [`Self::compact`] because they answer different + /// questions: compaction is about ghosts, this is about capacity, + /// and a table that compacts in order to keep inserting wants the + /// capacity it already has. + pub fn shrink_to_fit(&mut self) { + self.rows.shrink_to_fit(); + } + } + }) +} + +/// Declared `queries:` against a `vec: true` table. +/// +/// These are named wrappers, not a new execution path. A declared update is +/// `update(&pk, |row| ..)` with the columns filled in from a generated struct, +/// and `update` already repairs every index the edit moved a row under — so +/// delegating to it is both the shortest implementation and the only one that +/// cannot get index repair wrong in a second place. +/// +/// `by` may name the primary key, a unique secondary, or a non-unique +/// secondary. All three are *equality* lookups, which is the only shape a +/// declared query has, and every backend answers those — including `fxhash`. +/// Nothing here needs an ordered index, which is why these are emitted whatever +/// the `using` clause says while `range` and `range_by_` are not. +/// +/// A non-unique key names many rows, so those methods return how many they +/// touched rather than whether they touched one. +#[allow(clippy::too_many_arguments)] +fn gen_queries( + queries: Option<&worktable_dsl::model::Queries>, + pk: &Ident, + pk_type: &TokenStream, + columns: &Columns, + index_columns: &[Ident], + index_unique: &[bool], +) -> syn::Result<(Vec, Vec)> { + let Some(queries) = queries else { + return Ok((Vec::new(), Vec::new())); + }; + let mut structs = Vec::new(); + let mut methods = Vec::new(); + + // How a `by` column is reached, and whether it names one row or many. + let resolve_by = |by: &Ident| -> syn::Result<(TokenStream, bool)> { + let ty = columns + .columns_map + .get(by) + .ok_or_else(|| syn::Error::new(by.span(), format!("no column `{by}` to key a query by")))?; + if by == pk { + return Ok((quote! { #ty }, true)); + } + match index_columns.iter().position(|c| c == by) { + Some(at) => Ok((quote! { #ty }, index_unique[at])), + None => Err(syn::Error::new( + by.span(), + format!( + "a query keyed `by {by}` needs an index on `{by}`, and this table has none. \ + Add `{by}_idx: {by}` to `indexes:`, or key the query by the primary key. \ + Scanning instead would turn a keyed operation into a linear one silently." + ), + )), + } + }; + + // The pks a key selects. One for the primary key or a unique index, many + // for a non-unique one. Collected before mutating, because every mutation + // below takes `&mut self` and the lookup borrows `&self`. + let selected = |by: &Ident, unique: bool| -> TokenStream { + if by == pk { + quote! { let keys = worktable::prelude::vec![key.clone()]; } + } else { + let select = format_ident!("select_by_{by}"); + if unique { + quote! { + let keys: worktable::prelude::Vec<#pk_type> = + self.#select(key).map(|row| row.#pk.clone()).into_iter().collect(); + } + } else { + quote! { + let keys: worktable::prelude::Vec<#pk_type> = + self.#select(key).into_iter().map(|row| row.#pk.clone()).collect(); + } + } + } + }; + + for (name, op) in &queries.updates { + let (by_type, unique) = resolve_by(&op.by)?; + let query_ty = format_ident!("{}Query", name); + let fields = &op.columns; + let field_types: Vec<_> = fields + .iter() + .map(|f| { + columns + .columns_map + .get(f) + .cloned() + .ok_or_else(|| syn::Error::new(f.span(), format!("no column `{f}`"))) + }) + .collect::>()?; + structs.push(quote! { + #[derive(Clone, Debug, PartialEq)] + pub struct #query_ty { + #(pub #fields: #field_types,)* + } + }); + + // The declared name already carries the key — `AmountById` becomes + // `update_amount_by_id` — which is the paged table's convention and the + // whole point of generating these. + let method = format_ident!("update_{}", snake_of(name)); + let pick = selected(&op.by, unique); + let doc = format!( + "`update {name}` keyed by `{}`.\n\n\ + Sets {} and repairs every index the change moved a row under.\n\n\ + The paged table's method of this name is `async` and returns \ + `Result<(), WorkTableError>`. This one is neither, so a call cannot \ + move silently between the two shapes.", + op.by, + fields.iter().map(|f| format!("`{f}`")).collect::>().join(", ") + ); + methods.push(quote! { + #[doc = #doc] + pub fn #method(&mut self, query: #query_ty, key: &#by_type) -> usize { + #pick + let mut touched = 0usize; + for found in keys { + if self.update(&found, |row| { + #(row.#fields = query.#fields.clone();)* + }) { + touched += 1; + } + } + touched + } + }); + } + + for (name, op) in &queries.deletes { + let (by_type, unique) = resolve_by(&op.by)?; + let method = format_ident!("delete_{}", snake_of(name)); + let pick = selected(&op.by, unique); + let doc = format!( + "`delete {name}` keyed by `{}`.\n\n\ + Ghosts each row it names and returns how many. Nothing else moves: a \ + delete leaves its slot and takes only its own index entries, so the \ + expensive half is `compact`, when you ask for it.", + op.by + ); + methods.push(quote! { + #[doc = #doc] + pub fn #method(&mut self, key: &#by_type) -> usize { + #pick + let mut removed = 0usize; + for found in keys { + if self.delete(&found).is_some() { + removed += 1; + } + } + removed + } + }); + } + + for (name, op) in &queries.in_place { + let (by_type, unique) = resolve_by(&op.by)?; + if op.columns.len() != 1 { + return Err(syn::Error::new( + name.span(), + "an `in_place` query edits exactly one column through a closure. \ + For several columns at once use an `update` query, which takes a \ + struct of them.", + )); + } + let column = &op.columns[0]; + let column_type = columns + .columns_map + .get(column) + .ok_or_else(|| syn::Error::new(column.span(), format!("no column `{column}`")))?; + let method = format_ident!("update_{}_in_place", snake_of(name)); + let pick = selected(&op.by, unique); + let doc = format!( + "`in_place {name}` keyed by `{}`.\n\n\ + Hands a cloned candidate's `{column}` to the closure, then validates \ + unique keys before replacing the row. Returns how many rows it reached.", + op.by + ); + methods.push(quote! { + #[doc = #doc] + pub fn #method( + &mut self, + mut edit: impl FnMut(&mut #column_type), + key: &#by_type, + ) -> usize { + #pick + let mut touched = 0usize; + for found in keys { + if self.update(&found, |row| edit(&mut row.#column)) { + touched += 1; + } + } + touched + } + }); + } + + Ok((structs, methods)) +} + +fn snake_of(name: &Ident) -> String { + use convert_case::{Case, Casing as _}; + name.to_string().from_case(Case::Pascal).to_case(Case::Snake) +} diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index 79545d7a..2d18b426 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -7,11 +7,14 @@ // The 127 `crate::common::` paths across this crate are unchanged, so the diff // is a move rather than a sweep. mod common; +#[cfg(feature = "s3-support")] +mod database_s3_persistence; mod generators; mod mem_stat; mod migration_engine; mod persist_index; mod persist_table; +mod runtimes; #[cfg(feature = "s3-support")] mod s3_persistence; mod worktable; @@ -35,6 +38,35 @@ pub fn s3_sync_persistence(input: TokenStream) -> TokenStream { .into() } +#[cfg(feature = "s3-support")] +#[proc_macro] +pub fn database_s3_persistence(input: TokenStream) -> TokenStream { + database_s3_persistence::expand(input.into()) + .unwrap_or_else(|e| e.to_compile_error()) + .into() +} + +/// Declares the process's named runtime profiles. +/// +/// ```ignore +/// runtimes! { +/// tokio_max: tokio, +/// fast_local: nagoya(locality), +/// wide: nagoya(spread), +/// } +/// ``` +/// +/// One unit struct per entry, implementing `worktable::prelude::Profile`. Every +/// pool the process will ever create can be enumerated by reading one of these +/// blocks, which is the reason profiles are named rather than spelled out at +/// call sites. +#[proc_macro] +pub fn runtimes(input: TokenStream) -> TokenStream { + runtimes::expand(input.into()) + .unwrap_or_else(|e| e.to_compile_error()) + .into() +} + #[proc_macro_derive(PersistIndex, attributes(index))] pub fn persist_index(input: TokenStream) -> TokenStream { persist_index::expand(input.into()) diff --git a/codegen/src/mem_stat/mod.rs b/codegen/src/mem_stat/mod.rs index ea8d6b49..b674be14 100644 --- a/codegen/src/mem_stat/mod.rs +++ b/codegen/src/mem_stat/mod.rs @@ -3,11 +3,11 @@ use quote::quote; use syn::{Data, DeriveInput, Fields, Result, Type}; fn gen_heap_size_body(data: &Data) -> Result { - gen_mem_fn_body(data, quote! { heap_size() }, quote! { std::mem::size_of::() }) + gen_mem_fn_body(data, quote! { heap_size() }, quote! { core::mem::size_of::() }) } fn gen_used_size_body(data: &Data) -> Result { - gen_mem_fn_body(data, quote! { used_size() }, quote! { std::mem::size_of::() }) + gen_mem_fn_body(data, quote! { used_size() }, quote! { core::mem::size_of::() }) } fn gen_mem_fn_body(data: &Data, method: TokenStream, default_for_copy: TokenStream) -> Result { diff --git a/codegen/src/migration_engine/generator.rs b/codegen/src/migration_engine/generator.rs index 77e5e035..972f45fd 100644 --- a/codegen/src/migration_engine/generator.rs +++ b/codegen/src/migration_engine/generator.rs @@ -38,7 +38,7 @@ pub fn generate(input: MigrationEngineInput) -> TokenStream { source_path: &str, target: &mut #current_table, ctx: &#ctx_type, - ) -> eyre::Result<()> { + ) -> worktable::prelude::eyre::Result<()> { let config = DiskConfig::new_with_table_name(source_path, #table_name_lit, #version); let engine = ReadOnlyPersistenceEngine::create(config).await?; let source = #table_path::load(engine).await?; @@ -72,7 +72,7 @@ pub fn generate(input: MigrationEngineInput) -> TokenStream { source_path: &str, target_path: &str, ctx: &#ctx_type, - ) -> eyre::Result { + ) -> worktable::prelude::eyre::Result { let source_table_path = format!("{}/{}", source_path, #table_name_lit); let version = worktable::migration::detect_version::<<<#pk_type as worktable::prelude::TablePrimaryKey>::Generator as worktable::prelude::PrimaryKeyGeneratorState>::State>(&source_table_path).await?; @@ -82,7 +82,7 @@ pub fn generate(input: MigrationEngineInput) -> TokenStream { match version { #( #match_arms )* - v => return Err(eyre::eyre!("Unsupported version: {}", v)), + v => return Err(worktable::prelude::eyre::eyre!("Unsupported version: {}", v)), }; target.wait_for_ops().await?; diff --git a/codegen/src/persist_index/generator.rs b/codegen/src/persist_index/generator.rs index 49b78938..76b0e35a 100644 --- a/codegen/src/persist_index/generator.rs +++ b/codegen/src/persist_index/generator.rs @@ -17,6 +17,7 @@ pub struct Generator { pub struct_def: ItemStruct, pub field_types: HashMap, pub attributes: PersistIndexAttributes, + pub skipped_fields: Vec, } pub(super) struct IndexLayout { @@ -90,11 +91,29 @@ impl WorktableNameGenerator { } impl Generator { - pub fn with_attributes(struct_def: ItemStruct, attributes: PersistIndexAttributes) -> Self { + pub fn with_attributes(mut struct_def: ItemStruct, attributes: PersistIndexAttributes) -> Self { let mut fields = vec![]; let mut types = vec![]; + let mut skipped_fields = vec![]; for field in &struct_def.fields { + let skipped = field.attrs.iter().any(|attribute| { + if !attribute.path().is_ident("index") { + return false; + } + let mut skipped = false; + let _ = attribute.parse_nested_meta(|meta| { + if meta.path.is_ident("skip") { + skipped = true; + } + Ok(()) + }); + skipped + }); + if skipped { + skipped_fields.push(field.ident.clone().expect("index fields should always be named fields")); + continue; + } fields.push(field.ident.clone().expect("index fields should always be named fields")); let syn::Type::Path(type_path) = &field.ty else { @@ -122,12 +141,21 @@ impl Generator { types.push(ty.to_token_stream()); } + if let syn::Fields::Named(named) = &mut struct_def.fields { + named.named = named + .named + .iter() + .filter(|field| !skipped_fields.iter().any(|ident| field.ident.as_ref() == Some(ident))) + .cloned() + .collect(); + } let map = fields.into_iter().zip(types).collect::>(); Self { struct_def, field_types: map, attributes, + skipped_fields, } } @@ -150,7 +178,7 @@ impl Generator { let field_type = &field.ty; Ok(quote! { #i: #field_type, }) } else if is_unsized(&t.to_string()) { - let const_size = name_generator.get_page_inner_size_const_ident(); + let const_size = name_generator.get_disk_page_capacity(); Ok(quote! { #i: (Vec>>, Vec>>), }) @@ -193,7 +221,8 @@ impl Generator { fn gen_persist_fn(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let ident = name_generator.get_work_table_ident(); - let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); + let inner_const_name = name_generator.get_disk_page_capacity(); let version_const_name = name_generator.get_version_const_ident(); let index_extension = Literal::string(WT_INDEX_EXTENSION); @@ -216,15 +245,15 @@ impl Generator { }, _ => quote! { { - let mut file = tokio::fs::File::create(format!("{}/{}{}", path, #index_name_literal, #index_extension)).await?; + let mut file = worktable::prelude::fsx::create(format!("{}/{}{}", path, #index_name_literal, #index_extension)).await?; let mut info = #ident::space_info_default(); info.inner.page_count = self.#i.1.len() as u32 + self.#i.0.len() as u32; - persist_page(&mut info, &mut file).await?; + persist_page::<_, { #page_const_name as u32 }>(&mut info, &mut file).await?; for mut page in &mut self.#i.0 { - persist_page(&mut page, &mut file).await?; + persist_page::<_, { #page_const_name as u32 }>(&mut page, &mut file).await?; } for mut page in &mut self.#i.1 { - persist_page(&mut page, &mut file).await?; + persist_page::<_, { #page_const_name as u32 }>(&mut page, &mut file).await?; } } }, @@ -234,7 +263,7 @@ impl Generator { .expect("generated index layouts were validated"); quote! { - pub async fn persist(&mut self, path: &str) -> eyre::Result<()> + pub async fn persist(&mut self, path: &str) -> worktable::prelude::eyre::Result<()> { #(#persist_logic)* Ok(()) @@ -247,7 +276,7 @@ impl Generator { fn gen_parse_from_file_fn(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let page_const_name = name_generator.get_page_size_const_ident(); - let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let inner_const_name = name_generator.get_disk_page_capacity(); let version_const_name = name_generator.get_version_const_ident(); let index_extension = Literal::string(WT_INDEX_EXTENSION); @@ -295,18 +324,18 @@ impl Generator { _ => quote! { let #i: #parsed_type = { let mut #i = vec![]; - let mut file = tokio::fs::File::open(format!("{}/{}{}", path, #literal, #index_extension)).await?; - let info = parse_page::, { #page_const_name as u32 }>(&mut file, 0).await?; - let file_length = file.metadata().await?.len(); + let mut file = worktable::prelude::fsx::open_read_only(format!("{}/{}{}", path, #literal, #index_extension)).await?; + let info = parse_page::, { #inner_const_name as u32 }, { #page_const_name as u32 }>(&mut file, 0).await?; + let file_length = worktable::prelude::fsx::file_metadata(&mut file).await?; // Pages sit at a fixed #page_const_name stride // (header inside the slot): the next free page id // is ceil(len / stride). The previous divisor used // stride + header and an unconditional +1. let page_id = file_length.div_ceil(#page_const_name as u64); - let next_page_id = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(page_id as u32)); - let toc = IndexTableOfContents::<_, { #page_const_name as u32 }>::parse_from_file(&mut file, 0.into(), next_page_id.clone()).await?; + let next_page_id = worktable::prelude::Arc::new(core::sync::atomic::AtomicU32::new(page_id as u32)); + let toc = IndexTableOfContents::<_, { #inner_const_name as u32 }, { #page_const_name as u32 }>::parse_from_file(&mut file, 0.into(), next_page_id.clone()).await?; for page_id in toc.iter().map(|(_, page_id)| page_id) { - let index = parse_page::<_, { #page_const_name as u32 }>(&mut file, (*page_id).into()).await?; + let index = parse_page::<_, { #inner_const_name as u32 }, { #page_const_name as u32 }>(&mut file, (*page_id).into()).await?; #i.push(index); } (toc.pages, #i) @@ -326,7 +355,7 @@ impl Generator { .collect::>(); quote! { - pub async fn parse_from_file(path: &str) -> eyre::Result { + pub async fn parse_from_file(path: &str) -> worktable::prelude::eyre::Result { #(#field_names_literals)* Ok(Self { @@ -369,7 +398,8 @@ impl Generator { /// `TreeIndex` into `Vec` of `IndexPage`s using `IndexPage::from_nod` function. fn gen_get_persisted_index_fn(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); - let const_name = name_generator.get_page_inner_size_const_ident(); + let const_name = name_generator.get_disk_page_capacity(); + let page_const_name = name_generator.get_page_size_const_ident(); let idents = self .struct_def @@ -401,7 +431,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(UnsizedIndexPage::from_node(node.as_ref())); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend == Some(ArtBackend::ArcticMulti) { @@ -415,7 +445,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend == Some(ArtBackend::Arctic) && is_unsized(&ty.to_string()) { @@ -428,7 +458,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(UnsizedIndexPage::from_node(node.as_ref())); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend == Some(ArtBackend::Arctic) { @@ -442,7 +472,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend.is_some() { @@ -462,7 +492,7 @@ impl Generator { let page = UnsizedIndexPage::from_node(node.as_ref()); pages.push(page); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else { @@ -472,7 +502,7 @@ impl Generator { let page = UnsizedIndexPage::from_node(node.as_ref()); pages.push(page); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } @@ -491,7 +521,7 @@ impl Generator { .collect(); pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else { @@ -502,7 +532,7 @@ impl Generator { let page = IndexPage::from_node(&node, size); pages.push(page); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } @@ -523,7 +553,7 @@ impl Generator { /// persisted page back to `TreeIndex` fn gen_from_persisted_fn(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); - let const_name = name_generator.get_page_inner_size_const_ident(); + let const_name = name_generator.get_disk_page_capacity(); let idents = self .struct_def @@ -701,6 +731,7 @@ impl Generator { } }) .collect::>>()?; + let skipped_fields = &self.skipped_fields; Ok(quote! { fn from_persisted(persisted: Self::PersistedIndex) -> Self { @@ -708,6 +739,7 @@ impl Generator { Self { #(#idents,)* + #(#skipped_fields: Default::default(),)* } } }) diff --git a/codegen/src/persist_index/mod.rs b/codegen/src/persist_index/mod.rs index ef17c461..4eab99d6 100644 --- a/codegen/src/persist_index/mod.rs +++ b/codegen/src/persist_index/mod.rs @@ -46,7 +46,7 @@ mod tests { #[derive(Debug, Default, Clone)] pub struct TestIndex { test_idx: TreeIndex, - exchnage_idx: TreeIndex>> + exchnage_idx: TreeIndex>> } }; @@ -83,4 +83,26 @@ mod tests { "read_only index should have from_persisted method" ); } + + #[test] + fn skipped_derived_field_is_not_part_of_persisted_index_format() { + let input = quote! { + #[derive(Debug, Default)] + pub struct DerivedIndex { + durable: TreeIndex, + #[index(skip)] + columnar: ParkingRwLock, + } + }; + + let output = expand(input).unwrap().to_string(); + let persisted_type = output + .split("struct DerivedIndexPersisted") + .nth(1) + .expect("persisted index type"); + let persisted_fields = persisted_type.split('}').next().unwrap(); + assert!(persisted_fields.contains("durable")); + assert!(!persisted_fields.contains("columnar")); + assert!(output.contains("columnar : Default :: default")); + } } diff --git a/codegen/src/persist_index/parser.rs b/codegen/src/persist_index/parser.rs index f7023697..4cb43d36 100644 --- a/codegen/src/persist_index/parser.rs +++ b/codegen/src/persist_index/parser.rs @@ -46,7 +46,7 @@ mod tests { #[derive(Debug, Default, Clone)] pub struct TestIndex { test_idx: TreeIndex, - exchnage_idx: TreeIndex>> + exchnage_idx: TreeIndex>> } }; assert!(Parser::parse_struct(input).is_ok()) diff --git a/codegen/src/persist_index/space/events.rs b/codegen/src/persist_index/space/events.rs index 28b66819..720096ad 100644 --- a/codegen/src/persist_index/space/events.rs +++ b/codegen/src/persist_index/space/events.rs @@ -100,8 +100,8 @@ impl Generator { .collect(); quote! { - fn first_evs(&self) -> std::collections::HashMap<#avt_index_ident, Option> { - let mut map = std::collections::HashMap::new(); + fn first_evs(&self) -> worktable::prelude::HashMap<#avt_index_ident, Option> { + let mut map = worktable::prelude::HashMap::new(); #(#fields_first)* map } @@ -126,8 +126,8 @@ impl Generator { .collect(); quote! { - fn last_evs(&self) -> std::collections::HashMap<#avt_index_ident, Option> { - let mut map = std::collections::HashMap::new(); + fn last_evs(&self) -> worktable::prelude::HashMap<#avt_index_ident, Option> { + let mut map = worktable::prelude::HashMap::new(); #(#fields_last)* map } @@ -200,7 +200,7 @@ impl Generator { quote! { fn iter_event_ids(&self) -> impl Iterator { - > as Iterator>::flatten( + > as Iterator>::flatten( vec![ #(#fields_iter),* ] diff --git a/codegen/src/persist_index/space/index.rs b/codegen/src/persist_index/space/index.rs index 166b514e..67a76a97 100644 --- a/codegen/src/persist_index/space/index.rs +++ b/codegen/src/persist_index/space/index.rs @@ -8,7 +8,8 @@ impl Generator { pub fn gen_space_secondary_index_type(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let ident = name_generator.get_space_secondary_index_ident(); - let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let inner_const_name = name_generator.get_disk_page_capacity(); + let page_const_name = name_generator.get_page_size_const_ident(); let fields: Vec<_> = self .struct_def @@ -20,31 +21,31 @@ impl Generator { let t = self.field_types.get(i).expect("field type was collected"); Ok(match layout.art_backend { Some(ArtBackend::Arctic) if is_unsized(&t.to_string()) => quote! { - #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::Arctic) => quote! { - #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::ArcticMulti) if is_unsized(&t.to_string()) => quote! { - #i: SpaceLogicalMultiIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalMultiIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::ArcticMulti) => quote! { - #i: SpaceLogicalMultiIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalMultiIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::Congee) => quote! { #i: SpaceCongeeIndex<#t, { #inner_const_name as u32}>, }, None if layout.logical_wti && is_unsized(&t.to_string()) => quote! { - #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, None if layout.logical_wti => quote! { - #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, None if is_unsized(&t.to_string()) => quote! { - #i: SpaceIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, None => quote! { - #i: SpaceIndex<#t, { #inner_const_name as u32}>, + #i: SpaceIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, }) }) @@ -121,7 +122,7 @@ impl Generator { .expect("generated index layouts were validated"); quote! { - async fn from_table_files_path>(path: S, version: u32) -> eyre::Result { + async fn from_table_files_path>(path: S, version: u32) -> worktable::prelude::eyre::Result { let path = path.as_ref(); Ok(Self { #(#fields)* @@ -147,7 +148,7 @@ impl Generator { .collect(); quote! { - async fn process_change_events(&mut self, events: #events_ident) -> eyre::Result<()> { + async fn process_change_events(&mut self, events: #events_ident) -> worktable::prelude::eyre::Result<()> { #(#process)* core::result::Result::Ok(()) } @@ -169,7 +170,7 @@ impl Generator { .collect(); quote! { - async fn process_change_event_batch(&mut self, events: #events_ident) -> eyre::Result<()> { + async fn process_change_event_batch(&mut self, events: #events_ident) -> worktable::prelude::eyre::Result<()> { #(#process)* core::result::Result::Ok(()) } diff --git a/codegen/src/persist_table/generator/space.rs b/codegen/src/persist_table/generator/space.rs index 35a9c929..fb9ba1df 100644 --- a/codegen/src/persist_table/generator/space.rs +++ b/codegen/src/persist_table/generator/space.rs @@ -32,6 +32,8 @@ impl Generator { let ident = name_generator.get_persistence_engine_ident(); let primary_key_type = name_generator.get_primary_key_type_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let disk_capacity = name_generator.get_disk_page_capacity(); + let page_const_name = name_generator.get_page_size_const_ident(); let const_name = name_generator.get_page_size_const_ident(); let space_primary_index = name_generator.get_space_primary_index_ident(); let space_secondary_indexes = name_generator.get_space_secondary_index_ident(); @@ -40,23 +42,23 @@ impl Generator { let space_index_type = if self.attributes.pk_arctic_string || (self.attributes.pk_unsized && self.attributes.pk_wti_logical) { quote! { - SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }> + SpaceLogicalIndexUnsized<#primary_key_type, { #disk_capacity as u32 }, { #page_const_name as u32 }> } } else if self.attributes.pk_unsized { quote! { - SpaceIndexUnsized<#primary_key_type, { #inner_const_name as u32 }> + SpaceIndexUnsized<#primary_key_type, { #disk_capacity as u32 }, { #page_const_name as u32 }> } } else if self.attributes.pk_wti_logical || self.attributes.pk_arctic { quote! { - SpaceLogicalIndex<#primary_key_type, { #inner_const_name as u32 }> + SpaceLogicalIndex<#primary_key_type, { #disk_capacity as u32 }, { #page_const_name as u32 }> } } else if self.attributes.pk_congee { quote! { - SpaceCongeeIndex<#primary_key_type, { #inner_const_name as u32 }> + SpaceCongeeIndex<#primary_key_type, { #disk_capacity as u32 }> } } else { quote! { - SpaceIndex<#primary_key_type, { #inner_const_name as u32 }> + SpaceIndex<#primary_key_type, { #disk_capacity as u32 }, { #page_const_name as u32 }> } }; diff --git a/codegen/src/persist_table/generator/space_file/mod.rs b/codegen/src/persist_table/generator/space_file/mod.rs index 93626f25..cec3a1fa 100644 --- a/codegen/src/persist_table/generator/space_file/mod.rs +++ b/codegen/src/persist_table/generator/space_file/mod.rs @@ -26,11 +26,12 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let index_persisted_ident = name_generator.get_persisted_index_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let disk_capacity = name_generator.get_disk_page_capacity(); let pk_type = name_generator.get_primary_key_type_ident(); let space_file_ident = name_generator.get_space_file_ident(); let primary_index = if self.attributes.pk_unsized { quote! { - pub primary_index: (Vec>>, Vec>>), + pub primary_index: (Vec>>, Vec>>), } } else if self.attributes.pk_congee { quote! { @@ -75,7 +76,7 @@ impl Generator { }); quote! { - fn get_primary_index_info(&self) -> eyre::Result>> { + fn get_primary_index_info(&self) -> worktable::prelude::eyre::Result>> { let mut info = { let inner = SpaceInfoPage { id: 0.into(), @@ -131,6 +132,7 @@ impl Generator { let index_ident = name_generator.get_index_type_ident(); let task_ident = name_generator.get_persistence_task_ident(); let const_name = name_generator.get_page_inner_size_const_ident(); + let node_capacity = name_generator.get_disk_page_capacity(); let pk_type = name_generator.get_primary_key_type_ident(); let lock_type = name_generator.get_lock_type_ident(); let table_name = name_generator.get_work_table_literal_name(); @@ -162,7 +164,7 @@ impl Generator { quote! { IndexMap } }; quote! { - let pk_map = #map_type::<#pk_ident, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name); + let pk_map = #map_type::<#pk_ident, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#node_capacity); let nodes = self.primary_index.1.into_iter().map(|page| { let node = page .inner @@ -173,7 +175,7 @@ impl Generator { value: p.value.into(), }) .collect(); - UnsizedNode::from_inner(node, #const_name) + UnsizedNode::from_inner(node, #node_capacity) }); pk_map.attach_nodes(nodes); let primary_index = PrimaryIndex::from_map(pk_map); @@ -206,7 +208,7 @@ impl Generator { quote! { pk_map.attach_nodes(nodes); } }; quote! { - let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); + let size = get_index_page_size_from_data_length::<#pk_type>(#node_capacity); let pk_map = #map_type::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size); let nodes = self.primary_index.1.into_iter().map(|page| { page @@ -241,23 +243,24 @@ impl Generator { data.set_page_id(page_id.into()); page_id += 1; - std::sync::Arc::new(data) + worktable::prelude::Arc::new(data) }) .collect(); let data = DataPages::from_data(data) - .with_empty_links(self.data_info.inner.empty_links_list); + .with_empty_links(self.data_info.inner.empty_links_list) + .map_err(|error| PersistenceLoadError::corrupt(path, error))?; let indexes = #index_ident::from_persisted(self.indexes); #primary_index_init let table = WorkTable { - data: std::sync::Arc::new(data), - primary_index: std::sync::Arc::new(primary_index), - indexes: std::sync::Arc::new(indexes), + data: worktable::prelude::Arc::new(data), + primary_index: worktable::prelude::Arc::new(primary_index), + indexes: worktable::prelude::Arc::new(indexes), pk_gen: PrimaryKeyGeneratorState::from_state(self.data_info.inner.pk_gen_state), - lock_manager: std::sync::Arc::new(LockMap::<#lock_type, #pk_type>::default()), + lock_manager: worktable::prelude::Arc::new(LockMap::<#lock_type, #pk_type>::default()), table_name: #table_name, - pk_phantom: std::marker::PhantomData, + pk_phantom: core::marker::PhantomData, }; table.validate_persisted_state(path)?; @@ -310,23 +313,24 @@ impl Generator { data.set_page_id(page_id.into()); page_id += 1; - std::sync::Arc::new(data) + worktable::prelude::Arc::new(data) }) .collect(); let data = DataPages::from_data(data) - .with_empty_links(self.data_info.inner.empty_links_list); + .with_empty_links(self.data_info.inner.empty_links_list) + .map_err(|error| PersistenceLoadError::corrupt(path, error))?; let indexes = #index_ident::from_persisted(self.indexes); #primary_index_init let table = WorkTable { - data: std::sync::Arc::new(data), - primary_index: std::sync::Arc::new(primary_index), - indexes: std::sync::Arc::new(indexes), + data: worktable::prelude::Arc::new(data), + primary_index: worktable::prelude::Arc::new(primary_index), + indexes: worktable::prelude::Arc::new(indexes), pk_gen: PrimaryKeyGeneratorState::from_state(self.data_info.inner.pk_gen_state), - lock_manager: std::sync::Arc::new(LockMap::<#lock_type, #pk_type>::default()), + lock_manager: worktable::prelude::Arc::new(LockMap::<#lock_type, #pk_type>::default()), table_name: #table_name, - pk_phantom: std::marker::PhantomData, + pk_phantom: core::marker::PhantomData, }; table.validate_persisted_state(path)?; @@ -346,6 +350,7 @@ impl Generator { let pk_type = name_generator.get_primary_key_type_ident(); let page_const_name = name_generator.get_page_size_const_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let disk_capacity = name_generator.get_disk_page_capacity(); let persisted_index_name = name_generator.get_persisted_index_ident(); let version_const_name = name_generator.get_version_const_ident(); let index_extension = Literal::string(WT_INDEX_EXTENSION); @@ -353,17 +358,17 @@ impl Generator { let parse_pk_page = if self.attributes.pk_unsized { quote! { - let index = parse_page::, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; + let index = parse_page::, { #disk_capacity as u32 }, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; } } else { quote! { - let index = parse_page::, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; + let index = parse_page::, { #disk_capacity as u32 }, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; } }; let parse_primary = if self.attributes.pk_congee { quote! { - SpaceCongeeIndex::<#pk_type, { #inner_const_name as u32 }>::load_index::<#inner_const_name>( + SpaceCongeeIndex::<#pk_type, { #disk_capacity as u32 }>::load_index::<#inner_const_name>( format!("{}/primary{}", path, #index_extension), #version_const_name, ).await? @@ -372,17 +377,17 @@ impl Generator { quote! { { let mut primary_index = vec![]; - let mut primary_file = tokio::fs::File::open(format!("{}/primary{}", path, #index_extension)).await?; - let info = parse_page::, { #page_const_name as u32 }>(&mut primary_file, 0).await?; - let file_length = primary_file.metadata().await?.len(); + let mut primary_file = worktable::prelude::fsx::open_read_only(format!("{}/primary{}", path, #index_extension)).await?; + let info = parse_page::, { #disk_capacity as u32 }, { #page_const_name as u32 }>(&mut primary_file, 0).await?; + let file_length = worktable::prelude::fsx::file_metadata(&mut primary_file).await?; // Pages sit at a fixed #page_const_name stride with the // general header inside the slot, so the next free page id // is ceil(len / stride). The previous divisor added the // header on top of the full stride and lagged one page // behind roughly every 512 pages. let count = file_length.div_ceil(#page_const_name as u64); - let next_page_id = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(count as u32)); - let toc = IndexTableOfContents::<_, { #page_const_name as u32 }>::parse_from_file(&mut primary_file, 0.into(), next_page_id.clone()).await?; + let next_page_id = worktable::prelude::Arc::new(core::sync::atomic::AtomicU32::new(count as u32)); + let toc = IndexTableOfContents::<_, { #disk_capacity as u32 }, { #page_const_name as u32 }>::parse_from_file(&mut primary_file, 0.into(), next_page_id.clone()).await?; for page_id in toc.iter().map(|(_, page_id)| page_id) { #parse_pk_page primary_index.push(index); @@ -393,15 +398,15 @@ impl Generator { }; quote! { - pub async fn parse_file(path: &str) -> eyre::Result { + pub async fn parse_file(path: &str) -> worktable::prelude::eyre::Result { let primary_index = #parse_primary; let indexes = #persisted_index_name::parse_from_file(path).await?; let (data, data_info) = { let mut data = vec![]; - let mut data_file = tokio::fs::File::open(format!("{}/{}", path, #data_extension)).await?; - let info = parse_page::::Generator as PrimaryKeyGeneratorState>::State>, { #page_const_name as u32 }>(&mut data_file, 0).await?; - let file_length = data_file.metadata().await?.len(); + let mut data_file = worktable::prelude::fsx::open_read_only(format!("{}/{}", path, #data_extension)).await?; + let info = parse_page::::Generator as PrimaryKeyGeneratorState>::State>, { #disk_capacity as u32 }, { #page_const_name as u32 }>(&mut data_file, 0).await?; + let file_length = worktable::prelude::fsx::file_metadata(&mut data_file).await?; // ceil(len / stride) counts every occupied page slot, // including the info page at id 0, whether or not the last // page fills its slot. The previous floor + inclusive @@ -410,7 +415,7 @@ impl Generator { // exactly fills its slot), failing the whole load. let count = file_length.div_ceil(#page_const_name as u64); for page_id in 1..count { - let index = parse_data_page::<{ #page_const_name as u32}, { #inner_const_name as usize }>(&mut data_file, page_id as u32).await?; + let index = parse_data_page::<{ #page_const_name as u32}, { #inner_const_name as usize }, { #page_const_name as u32 }>(&mut data_file, page_id as u32).await?; data.push(index); } (data, info) diff --git a/codegen/src/persist_table/generator/space_file/worktable_impls.rs b/codegen/src/persist_table/generator/space_file/worktable_impls.rs index c3d19f7d..12c92ccb 100644 --- a/codegen/src/persist_table/generator/space_file/worktable_impls.rs +++ b/codegen/src/persist_table/generator/space_file/worktable_impls.rs @@ -33,13 +33,13 @@ impl Generator { /// Retires an Arc-owned table generation after the caller's /// quiesce barrier has stopped new leases and drained old ones. pub async fn unload_gracefully( - self: std::sync::Arc, - timeout: std::time::Duration, + self: worktable::prelude::Arc, + timeout: core::time::Duration, quiesce: F, ) -> Result> where F: FnOnce() -> Fut, - Fut: std::future::Future, + Fut: core::future::Future, { // Attribute the generation at the retirement request. The // quiesce callback can give background maintenance time to @@ -47,25 +47,25 @@ impl Generator { // measuring afterwards would make the report depend on how // long the reader barrier happened to take. let estimated_released_bytes = self.heap_size(); - if tokio::time::timeout(timeout, quiesce()).await.is_err() { + if worktable::prelude::timeout(timeout, quiesce()).await.is_err() { return Err(UnloadFailure::retained( self, - eyre::eyre!("timed out waiting for generation leases to quiesce"), + worktable::prelude::eyre::eyre!("timed out waiting for generation leases to quiesce"), )); } - let owned = match std::sync::Arc::try_unwrap(self) { + let owned = match worktable::prelude::Arc::try_unwrap(self) { Ok(owned) => owned, Err(arc) => { - let outstanding = std::sync::Arc::strong_count(&arc).saturating_sub(1); + let outstanding = worktable::prelude::Arc::strong_count(&arc).saturating_sub(1); return Err(UnloadFailure::retained( arc, - eyre::eyre!("cannot unload generation: {outstanding} Arc lease(s) remain"), + worktable::prelude::eyre::eyre!("cannot unload generation: {outstanding} Arc lease(s) remain"), )); } }; owned.close().await.map_err(|error| { - UnloadFailure::after_close(eyre::Report::new(error)) + UnloadFailure::after_close(worktable::prelude::eyre::Report::new(error)) })?; Ok(UnloadReport { estimated_released_bytes }) } @@ -80,7 +80,7 @@ impl Generator { /// Returns the physical size of this table's `.wt.data` file. /// Persisted vacuum makes freed pages reusable across reloads, /// but does not truncate this file. - pub async fn persisted_data_file_size_bytes(&self) -> std::io::Result { + pub async fn persisted_data_file_size_bytes(&self) -> Result { self.1.persisted_data_file_size_bytes().await } } @@ -184,13 +184,16 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let pk_type = name_generator.get_primary_key_type_ident(); let const_name = name_generator.get_page_inner_size_const_ident(); + let node_capacity = name_generator.get_disk_page_capacity(); + let disk_capacity = name_generator.get_disk_page_capacity(); + let page_const_name = name_generator.get_page_size_const_ident(); if self.attributes.pk_congee { // Congee durability is maintained by its native checkpoint/WAL. quote! {} } else if self.attributes.pk_arctic_string { quote! { - pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { - let shadow = IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name); + pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { + let shadow = IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#node_capacity); for (key, value) in self.0.primary_index.pk_map.iter_values() { shadow.insert(key, value); } @@ -198,14 +201,14 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(UnsizedIndexPage::from_node(node.as_ref())); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #disk_capacity as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } } else if self.attributes.pk_arctic { quote! { pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { - let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); + let size = get_index_page_size_from_data_length::<#pk_type>(#node_capacity); let shadow = IndexMap::<#pk_type, OffsetEqLink<#const_name>>::with_maximum_node_size(size); for (key, value) in self.0.primary_index.pk_map.iter_values() { shadow.insert(key, value); @@ -214,19 +217,19 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #disk_capacity as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } } else if self.attributes.pk_unsized { quote! { - pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { + pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { let mut pages = vec![]; for node in self.0.primary_index.pk_map.snapshot_nodes() { let page = UnsizedIndexPage::from_node(node.as_ref()); pages.push(page); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #disk_capacity as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } @@ -254,10 +257,10 @@ impl Generator { }; quote! { pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { - let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); + let size = get_index_page_size_from_data_length::<#pk_type>(#node_capacity); let mut pages = vec![]; #collect_pages - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #disk_capacity as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } diff --git a/codegen/src/runtimes/mod.rs b/codegen/src/runtimes/mod.rs new file mode 100644 index 00000000..0b861f4e --- /dev/null +++ b/codegen/src/runtimes/mod.rs @@ -0,0 +1,366 @@ +//! The `runtimes!` macro: the process's named runtime profiles, in one block. +//! +//! ```ignore +//! runtimes! { +//! tokio_max: tokio, +//! fast_local: nagoya(locality), +//! wide: nagoya(spread), +//! } +//! ``` +//! +//! Each entry becomes a unit struct implementing `worktable::prelude::Profile`, +//! named exactly as written. One identifier serves as both the type and the +//! value, which is what lets a call site write `.runtime(wide)` and a schema +//! section write `runtime wide:` without a case convention between them. +//! +//! The struct's `Backend` associated type is the load-bearing part. A call site +//! naming a profile from the wrong backend then fails as an equality that does +//! not hold, and the compiler prints both backend types; without it the same +//! mistake would surface much further downstream, as whatever the mismatched +//! `RwLock` or `JoinHandle` broke first. + +use indexmap::IndexMap; +use proc_macro2::{Ident, Span, TokenStream}; +use quote::quote; +use syn::parse::{Parse, ParseStream}; +use syn::{Error, Token, parenthesized}; + +/// Backends named in the DSL but not built. Recognised only so the message can +/// say what happened, per the rule that an inert declaration is an error rather +/// than a thing that silently does nothing. +const NOT_IMPLEMENTED: &[&str] = &["forte", "blocking", "bwos"]; + +/// What is built, in the order the message should list them. +const IMPLEMENTED: &[&str] = &["nagoya", "tokio"]; + +/// The nagoya flavors, in the order the message should list them. +/// +/// Taken from the DSL's mirror of the runtime registry rather than written +/// out, so a flavor cannot be added to the parser and left out of the +/// message that claims to be exhaustive. +fn flavors() -> Vec<&'static str> { + worktable_dsl::model::Flavor::ALL.iter().map(|f| f.name()).collect() +} + +/// One `name: backend(flavor)` entry, resolved. +struct ProfileEntry { + name: Ident, + /// `Some` for nagoya, `None` for tokio. Kept as the source ident rather + /// than an enum so the emitted marker type and the span both come from + /// what was written. + backend: Ident, + flavor: Option, +} + +struct Runtimes { + profiles: IndexMap, +} + +impl Parse for Runtimes { + fn parse(input: ParseStream) -> syn::Result { + let mut profiles: IndexMap = IndexMap::new(); + + while !input.is_empty() { + let name: Ident = input.parse()?; + input.parse::()?; + let backend: Ident = input.parse()?; + + let flavor = if input.peek(syn::token::Paren) { + let inner; + parenthesized!(inner in input); + let flavor: Ident = inner.parse()?; + if !inner.is_empty() { + return Err(Error::new( + inner.span(), + "a runtime takes one flavor and nothing else; worker counts and durations are not \ + call-site parameters", + )); + } + Some(flavor) + } else { + None + }; + + let entry = resolve(name, backend, flavor)?; + + if let Some(previous) = profiles.get(&entry.name.to_string()) { + let mut err = Error::new(entry.name.span(), format!("duplicate runtime profile `{}`", entry.name)); + err.combine(Error::new( + previous.name.span(), + format!("`{}` was already declared here", previous.name), + )); + return Err(err); + } + profiles.insert(entry.name.to_string(), entry); + + if input.is_empty() { + break; + } + input.parse::()?; + } + + Ok(Self { profiles }) + } +} + +/// Checks one entry against the backends and flavors that exist. +/// +/// Everything rejected here is rejected at expansion rather than accepted inert, +/// so a declaration that reads as if it selected something either did or failed +/// to build. +fn resolve(name: Ident, backend: Ident, flavor: Option) -> syn::Result { + let backend_name = backend.to_string(); + + if NOT_IMPLEMENTED.contains(&backend_name.as_str()) { + return Err(Error::new( + backend.span(), + format!( + "runtime backend `{backend_name}` is not implemented; the backends that are: {}", + IMPLEMENTED.join(", ") + ), + )); + } + + match backend_name.as_str() { + "nagoya" => { + let flavor = match flavor { + None => Ident::new(worktable_dsl::model::Flavor::default().name(), backend.span()), + Some(flavor) => { + let flavor_name = flavor.to_string(); + if !flavors().contains(&flavor_name.as_str()) { + return Err(Error::new( + flavor.span(), + format!( + "unknown nagoya flavor `{flavor_name}`; expected one of: {}", + flavors().join(", ") + ), + )); + } + flavor + } + }; + Ok(ProfileEntry { + name, + backend, + flavor: Some(flavor), + }) + } + "tokio" => { + if let Some(flavor) = flavor { + return Err(Error::new( + flavor.span(), + "tokio has no flavors; write `tokio`. Flavors belong to nagoya, whose pool they tune", + )); + } + Ok(ProfileEntry { + name, + backend, + flavor: None, + }) + } + _ => Err(Error::new( + backend.span(), + format!( + "unknown runtime backend `{backend_name}`; expected one of: {}", + IMPLEMENTED.join(", ") + ), + )), + } +} + +impl ProfileEntry { + /// The concrete backend type and the expression that yields its tuning. + /// + /// Mirrors the emitted type tokens in the contract's section 4, which is + /// also what `runtime_backend.rs` emits for the table-level declaration; + /// the two have to agree or a table and a profile that both say + /// `nagoya(spread)` would not compare equal. + fn backend_tokens(&self) -> (TokenStream, TokenStream) { + match &self.flavor { + Some(flavor) => { + let marker = Ident::new( + worktable_dsl::model::Flavor::from_name(&flavor.to_string()) + .unwrap_or_default() + .type_name(), + flavor.span(), + ); + ( + quote! { worktable::prelude::NagoyaRt }, + quote! { ::tuning() }, + ) + } + None => ( + quote! { worktable::prelude::TokioRt }, + // Tokio's pool is not this crate's to tune, so the profile + // reports the defaults rather than inventing numbers nothing + // reads. + quote! { worktable::prelude::Tuning::default() }, + ), + } + } + + fn expand(&self) -> TokenStream { + let name = &self.name; + let (backend_type, tuning) = self.backend_tokens(); + + let declaration = match &self.flavor { + Some(flavor) => format!("`{}({})`", self.backend, flavor), + None => format!("`{}`", self.backend), + }; + let doc = format!( + "Runtime profile `{name}`: {declaration}.\n\n\ + Generated by `runtimes!`. Named at a call site as `.runtime({name})` and at a schema \ + section as `runtime {name}:`.\n\n\ + A unit struct rather than an enum variant so that deferred parameters (a worker count, \ + a backoff) can arrive later as fields, which changes this type and `tuning` and moves no \ + call site.", + ); + + quote! { + #[doc = #doc] + // The profile's name is the surface syntax, at the call site and in + // the schema alike, so it is spelled as written rather than + // converted into a type convention nothing else here uses. + #[allow(non_camel_case_types)] + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] + pub struct #name; + + impl worktable::prelude::Profile for #name { + type Backend = #backend_type; + + fn tuning() -> worktable::prelude::Tuning { + #tuning + } + } + } + } +} + +pub fn expand(input: TokenStream) -> syn::Result { + let runtimes: Runtimes = syn::parse2(input)?; + + if runtimes.profiles.is_empty() { + return Err(Error::new( + Span::call_site(), + "`runtimes!` with no profiles declares nothing; remove it or name a profile", + )); + } + + let profiles = runtimes.profiles.values().map(ProfileEntry::expand); + Ok(quote! { #(#profiles)* }) +} + +#[cfg(test)] +mod tests { + use quote::quote; + + use super::expand; + + fn expanded(input: proc_macro2::TokenStream) -> String { + expand(input).unwrap().to_string() + } + + fn rejected(input: proc_macro2::TokenStream) -> String { + expand(input).unwrap_err().to_string() + } + + #[test] + fn profiles_resolve_to_their_backend_and_tuning() { + let out = expanded(quote! { + tokio_max: tokio, + fast_local: nagoya(locality), + wide: nagoya(spread), + batch: nagoya(throughput), + }); + + assert!(out.contains("pub struct tokio_max"), "{out}"); + assert!(out.contains("type Backend = worktable :: prelude :: TokioRt"), "{out}"); + assert!( + out.contains("type Backend = worktable :: prelude :: NagoyaRt < worktable :: prelude :: Locality >"), + "{out}" + ); + assert!( + out.contains("type Backend = worktable :: prelude :: NagoyaRt < worktable :: prelude :: Spread >"), + "{out}" + ); + assert!( + out.contains("type Backend = worktable :: prelude :: NagoyaRt < worktable :: prelude :: Throughput >"), + "{out}" + ); + assert!( + out.contains("< worktable :: prelude :: Spread as worktable :: prelude :: FlavorMarker > :: tuning ()"), + "{out}" + ); + assert!(out.contains("worktable :: prelude :: Tuning :: default ()"), "{out}"); + } + + #[test] + fn bare_nagoya_is_the_registry_default() { + let bare = expanded(quote! { p: nagoya }); + let flavor = proc_macro2::Ident::new( + worktable_dsl::model::Flavor::default().name(), + proc_macro2::Span::call_site(), + ); + let explicit = expanded(quote! { p: nagoya(#flavor) }); + assert_eq!(bare, explicit); + } + + #[test] + fn duplicate_profile_names_are_rejected() { + let err = rejected(quote! { + wide: nagoya(spread), + wide: tokio, + }); + assert!(err.contains("duplicate runtime profile `wide`"), "{err}"); + } + + #[test] + fn unknown_backend_is_rejected() { + let err = rejected(quote! { p: smol }); + assert!(err.contains("unknown runtime backend `smol`"), "{err}"); + assert!(err.contains("nagoya, tokio"), "{err}"); + } + + #[test] + fn not_implemented_backends_are_rejected_rather_than_accepted_inert() { + for backend in ["forte", "blocking", "bwos"] { + let input: proc_macro2::TokenStream = format!("p: {backend}").parse().unwrap(); + let err = rejected(input); + assert!(err.contains(backend), "{err}"); + assert!(err.contains("is not implemented"), "{err}"); + assert!(err.contains("nagoya, tokio"), "{err}"); + } + } + + #[test] + fn tokio_takes_no_flavor() { + let err = rejected(quote! { p: tokio(spread) }); + assert!(err.contains("tokio has no flavors"), "{err}"); + } + + #[test] + fn unknown_flavor_lists_every_flavor() { + let err = rejected(quote! { p: nagoya(banana) }); + assert!(err.contains("unknown nagoya flavor `banana`"), "{err}"); + for flavor in worktable_dsl::model::Flavor::ALL { + assert!(err.contains(flavor.name()), "{} missing from: {err}", flavor.name()); + } + } + + #[test] + fn a_flavor_takes_no_parameters() { + let err = rejected(quote! { p: nagoya(spread, 12) }); + assert!(err.contains("one flavor and nothing else"), "{err}"); + } + + #[test] + fn an_empty_block_is_rejected() { + let err = rejected(quote! {}); + assert!(err.contains("declares nothing"), "{err}"); + } + + #[test] + fn a_trailing_comma_is_optional() { + assert_eq!(expanded(quote! { p: tokio, }), expanded(quote! { p: tokio })); + } +} diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index b7820ef8..cd11700f 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -1,7 +1,10 @@ use proc_macro2::TokenStream; +use quote::quote; use crate::common::Parser; +use crate::common::model::RuntimeBackend; use crate::common::name_generator::WorktableNameGenerator; +use crate::generators::runtime_backend::{resolve_runtime, runtime_type}; pub fn expand(input: TokenStream) -> syn::Result { // Keep the tokens. The declaration is read a second time at the end, as @@ -16,10 +19,13 @@ pub fn expand(input: TokenStream) -> syn::Result { let mut columns = None; let mut queries = None; let mut indexes = None; + let mut columnar_indexes = None; let mut config = None; + let mut runtime = None; let name = parser.parse_name()?; let version = parser.parse_version()?.unwrap_or(1); + let storage = parser.parse_storage()?; let persistence = parser.parse_persist()?; let partition_by = parser.parse_partition_by()?; while let Some(ident) = parser.peek_next() { @@ -32,6 +38,10 @@ pub fn expand(input: TokenStream) -> syn::Result { let res = parser.parse_indexes()?; indexes = Some(res); } + "columnar_indexes" => { + let res = parser.parse_columnar_indexes()?; + columnar_indexes = Some(res); + } "queries" => { let res = parser.parse_queries()?; queries = Some(res) @@ -40,6 +50,17 @@ pub fn expand(input: TokenStream) -> syn::Result { let res = parser.parse_configs()?; config = Some(res) } + "runtime" => { + // Free-order, but not repeatable: two `runtime:` keys would + // silently keep one of them, and which one is a detail of this + // loop rather than anything the author could read off the + // declaration. + if runtime.is_some() { + return Err(syn::Error::new(ident.span(), "duplicate `runtime` section")); + } + let res = parser.parse_runtime()?; + runtime = Some(res) + } "version" => { return Err(syn::Error::new( ident.span(), @@ -49,16 +70,32 @@ pub fn expand(input: TokenStream) -> syn::Result { // Positional declarations that landed after the blocks began, or in // the wrong relative order, would otherwise die as a bare // "Unexpected identifier" and cost the next person a bisect. + "vec" => { + return Err(syn::Error::new( + ident.span(), + "`vec` is positional and must come before `persist`; the required order is: name, version, vec, persist, partition_by, partition_max_size, then columns/indexes/queries/config", + )); + } "persist" => { return Err(syn::Error::new( ident.span(), - "`persist` is positional and must come before `partition_by` and the blocks; the required order is: name, version, persist, partition_by, then columns/indexes/queries/config", + "`persist` is positional and must come after `vec` and before `partition_by` and the blocks; the required order is: name, version, vec, persist, partition_by, partition_max_size, then columns/indexes/queries/config", )); } "partition_by" => { return Err(syn::Error::new( ident.span(), - "`partition_by` is positional and must come after `persist` and before the blocks; the required order is: name, version, persist, partition_by, then columns/indexes/queries/config", + "`partition_by` is positional and must come after `persist` and before the blocks; the required order is: name, version, vec, persist, partition_by, partition_max_size, then columns/indexes/queries/config", + )); + } + // Reached only when `partition_by` was absent: with it present this + // key is consumed there, and a stray second one would have to get + // past that. So the useful thing to say is that it needs a + // `partition_by` to belong to, not that it is out of order. + "partition_max_size" => { + return Err(syn::Error::new( + ident.span(), + "`partition_max_size` describes how large one partition gets, so it means nothing without `partition_by:` before it. Add the routing key, or remove this", )); } "attributes" => { @@ -70,7 +107,9 @@ pub fn expand(input: TokenStream) -> syn::Result { other => { return Err(syn::Error::new( ident.span(), - format!("Unexpected token `{other}`; expected one of `columns`, `indexes`, `queries`, `config`"), + format!( + "Unexpected token `{other}`; expected one of `columns`, `indexes`, `columnar_indexes`, `queries`, `config`, `runtime`" + ), )); } } @@ -80,22 +119,195 @@ pub fn expand(input: TokenStream) -> syn::Result { if let Some(i) = indexes { columns.indexes = i } + if let Some(i) = columnar_indexes { + columns.columnar_indexes = i.indexes; + } + + // `storage: vec` generates a different table, so it leaves here rather + // than falling through the paging, columnar and runtime machinery below. + // + // The keys it refuses are refused with an error naming what to use + // instead. A silent no-op would be worse: `runtime: nagoya(locality)` on a + // synchronous table is a reasonable thing to write and a completely + // meaningless thing to have accepted. + if let Some(q) = &queries { + worktable_dsl::validate::validate_query_storage(&columns, q, storage)?; + } + if storage.is_vec() { + if !columns.columnar_indexes.is_empty() || !columns.columnar_fields.is_empty() { + return Err(syn::Error::new( + name.span(), + "`vec: true` has no pages, and columnar storage is a paging feature. Remove \ + the columnar declarations, or drop `vec: true` for a paged table.", + )); + } + if runtime.is_some() { + return Err(syn::Error::new( + name.span(), + "`vec: true` is synchronous and never reaches a runtime. Remove `runtime:`, or \ + drop `vec: true` for a paged table.", + )); + } + if config.is_some() { + return Err(syn::Error::new( + name.span(), + "`vec: true` has no page size and no columnar chunking to configure. Remove \ + `config:`, or drop `vec: true` for a paged table.", + )); + } + if persistence != worktable_dsl::Persistence::Omitted { + return Err(syn::Error::new( + name.span(), + "`vec: true` has no persistence engine, so `persist` says nothing here. The rows \ + go to bytes and back through `unload` and `load`, which you call when you want \ + them: there is no task, no flush, and nothing paid for durability that is not \ + asked for. Remove `persist:`, or drop `vec: true` for a paged table.", + )); + } + // The router needs the columns to pick its payload, and `vec_table` + // consumes them. Cloned only when there is a router to build. + let vec_columns = partition_by.as_ref().map(|_| columns.clone()); + let narrow_key_lint = if partition_by.is_none() { + gen_narrow_primary_key_lint(&columns) + } else { + quote! {} + }; + let mut generated = crate::generators::vec_table::expand(name.clone(), columns, queries.as_ref())?; + generated.extend(narrow_key_lint); + // The router is storage-agnostic: it needs `Default` and `used_bytes` + // from its payload and nothing else, and a `vec: true` table has both. + // Partitioning is what makes the `Vec` shape correct rather than + // something it has nothing to do with, so this composes instead of + // being refused. + if let Some(key) = partition_by { + let columns = vec_columns.expect("cloned whenever `partition_by` is present"); + generated.extend(crate::generators::partitions::expand( + &name, + &key, + worktable_dsl::Persistence::MemoryOnly, + &columns, + // Vec queries are methods on the mutable table. The shared + // partition directory does not generate mutable query wrappers. + &crate::generators::dense_table::DenseQueries::default(), + )?); + } + generated.extend(gen_schema_const(&worktable_dsl::Schema::from_tokens(declaration)?)); + return Ok(generated); + } + + // Past this point the table is paged, and `fxhash` cannot be. + // + // Two reasons, and neither is a matter of taste. A paged table answers + // ranges — `select_by__range` is generated for every secondary + // index, and the persistence worker reads its own queue by range — and a + // hash map cannot answer one at any price. And a persisted index's on-disk + // form *is* sorted pages: `from_persisted` rebuilds each index with + // `attach_nodes` from B-tree nodes read off the file, and a hash map has no + // node structure to attach. + // + // Refused here rather than left to fail somewhere inside the index + // generators, where the error would land on a type the author never wrote. + { + let mut offenders = Vec::new(); + if columns.primary_index_backend == worktable_dsl::IndexBackend::FxHash { + offenders.push(( + columns + .primary_keys + .first() + .map(|key| key.span()) + .unwrap_or_else(proc_macro2::Span::call_site), + "the primary key".to_string(), + )); + } + for index in columns.indexes.values() { + if index.backend == worktable_dsl::IndexBackend::FxHash { + offenders.push((index.name.span(), format!("`{}`", index.name))); + } + } + if let Some((span, what)) = offenders.into_iter().next() { + return Err(syn::Error::new( + span, + format!( + "`using fxhash` on {what}: a hash index has no ordered scan and no persisted \ + page form, so it cannot back a paged table. This table generates \ + `select_by__range` for its indexes and, if persisted, writes each \ + index as sorted pages. Use `vec: true`, which is single-writer and asks its \ + index only for point operations, or pick an ordered backend \ + (`arctic` is the default)." + ), + )); + } + } + + let columnar_chunk_rows = config + .as_ref() + .map(|config| config.columnar_chunk_rows) + .unwrap_or(crate::common::model::DEFAULT_COLUMNAR_CHUNK_ROWS); + columns.column_slot_id = config + .as_ref() + .map(|config| config.columnar_slot_id) + .unwrap_or_default(); + for field in columns.columnar_fields.values_mut() { + let chunk_rows = field.chunk_rows.unwrap_or(columnar_chunk_rows); + let (smaller, larger) = if chunk_rows <= columnar_chunk_rows { + (chunk_rows, columnar_chunk_rows) + } else { + (columnar_chunk_rows, chunk_rows) + }; + let nested = larger % smaller == 0 && (larger / smaller).is_power_of_two(); + if !nested { + return Err(syn::Error::new( + proc_macro2::Span::call_site(), + format!( + "columnar chunk_rows({chunk_rows}) must be a power-of-two multiple or divisor of config.columnar_chunk_rows ({columnar_chunk_rows})" + ), + )); + } + field.chunk_rows = Some(chunk_rows); + } worktable_dsl::validate::validate_index_backends(&columns, persistence)?; + worktable_dsl::validate::validate_columnar_indexes(&columns)?; worktable_dsl::validate::validate_page_size(config.as_ref(), persistence)?; worktable_dsl::validate::validate_arctic_page_size(&columns, config.as_ref())?; if let Some(q) = &queries { worktable_dsl::validate::validate_in_place_queries(&columns, q)?; } + // The router needs the columns to decide its payload: a narrow + // `partition_max_size` selects a position-addressed table whose shape + // depends on the primary key. Cloned rather than borrowed because the table + // generators below consume `columns`, and only a partitioned declaration + // pays for the clone. + let partition_columns = partition_by.as_ref().map(|_| columns.clone()); + // Lifted before the table generators consume `queries`. `Queries` is not + // `Clone`, and the dense payload is emitted after them. + let partition_queries = crate::generators::dense_table::DenseQueries::from_model(queries.as_ref()); + + let narrow_key_lint = if partition_by.is_none() { + gen_narrow_primary_key_lint(&columns) + } else { + quote! {} + }; + let mut generated = if persistence.is_persisted() { crate::generators::persist::expand(name.clone(), columns, queries, config, version)? } else { crate::generators::in_memory::expand_from_parsed(name.clone(), columns, queries, config)? }; + generated.extend(narrow_key_lint); + generated.extend(gen_runtime_type(&name, runtime)); + if let Some(key) = partition_by { - generated.extend(crate::generators::partitions::expand(&name, &key, persistence)); + let columns = partition_columns.expect("cloned whenever `partition_by` is present"); + generated.extend(crate::generators::partitions::expand( + &name, + &key, + persistence, + &columns, + &partition_queries, + )?); } generated.extend(gen_schema_const(&worktable_dsl::Schema::from_tokens(declaration)?)); @@ -103,6 +315,106 @@ pub fn expand(input: TokenStream) -> syn::Result { Ok(generated) } +/// Warn about a primary key too narrow to be a table's, when it is a table's. +/// +/// A `u8` primary key counts to 256 and a `bool` one to two. On a *partitioned* +/// table that is correct and is the whole point: the routing key does the +/// spreading and the inner key only separates the handful of rows inside one +/// partition, which is what `partition_max_size` exists to say. On an +/// unpartitioned table it is a table that can never hold more than 256 rows, +/// which is almost always a key that was meant to be wider. +/// +/// A lint and not a ban, deliberately. Narrow keys are what make the dense +/// partition possible, and a 256-row lookup table is a real thing to want. +/// +/// # Why a deprecation +/// +/// A proc macro cannot emit a warning on stable. A `#[deprecated]` item used +/// once in the expansion produces one, carries a message naming the column, and +/// can be silenced the ordinary way: `#[allow(deprecated)]` on the module +/// holding the declaration. Everything is emitted inside an anonymous `const` +/// so none of it is nameable and nothing leaks into the consumer's namespace. +fn gen_narrow_primary_key_lint(columns: &worktable_dsl::model::Columns) -> TokenStream { + if columns.primary_keys.len() != 1 { + return quote! {}; + } + let pk = columns.primary_keys.first().expect("checked above"); + let Some(ty) = columns.columns_map.get(pk) else { + return quote! {}; + }; + let ty = ty.to_string().replace(' ', ""); + let rows = match ty.as_str() { + "u8" => "256", + "bool" => "2", + _ => return quote! {}, + }; + + let note = format!( + "`{pk}: {ty}` is the primary key of an unpartitioned table, so this table can never hold \ + more than {rows} rows. That is correct beside `partition_by`, where the routing key does \ + the spreading and this key only separates the rows inside one partition; on its own it is \ + usually a key that was meant to be wider. Partition the table, widen the key, or put \ + `#[allow(deprecated)]` on the module if {rows} rows is what you meant." + ); + + quote! { + const _: () = { + #[deprecated(note = #note)] + const NARROW_PRIMARY_KEY: () = (); + #[allow(unused)] + fn narrow_primary_key() { + let _ = NARROW_PRIMARY_KEY; + } + }; + } +} + +/// Name the runtime the table resolved to, once, as a type. +/// +/// This is the runtime half of what `index_backend` does for indexes: the DSL +/// carries an enum, the enum becomes a concrete type, and the generated code +/// names the type rather than knowing which backend was picked. +/// +/// It is an alias rather than a generic argument on the emitted `WorkTable<..>` +/// because `Runtime` is not a parameter of that type yet. When it becomes one, +/// this alias is the argument to pass, and the six emitted `worktable::prelude` +/// call sites become `<#ident as Runtime>::sleep` and friends, so the seam is +/// already in the right place. +/// +/// `allow(dead_code)` for the same reason `gen_schema_const` needs it: a +/// `worktable!` inside a function body puts this alias in that body, where a +/// user building with `-D warnings` would otherwise fail over a name they never +/// wrote. +pub(crate) fn gen_runtime_type(name: &proc_macro2::Ident, runtime: Option) -> TokenStream { + // Emit nothing into a `no_std` build. Every backend needs threads, so the + // prelude exports no runtime type there and naming one would not resolve. + // A table that never spawns is still a table, which is why this is silent + // rather than an error. + // + // This intentionally evaluates the proc-macro crate's own feature, the same + // way `index_backend` does: `worktable`'s `std` forwards to + // `worktable_codegen/std` in Cargo.toml, so the runtime types and the + // emitted types are selected together. Emitting a `cfg` into the expansion + // would instead test the consuming package's unrelated feature namespace. + if !cfg!(feature = "std") { + return TokenStream::new(); + } + + let ident = WorktableNameGenerator::from_table_name(name.to_string()).get_runtime_type_ident(); + // An omitted `runtime:` resolves through the same chain as an unannotated + // section, so a declaration written before this key existed emits exactly + // what `runtime: nagoya` emits. + let ty = runtime_type(resolve_runtime(None, runtime)); + let row = WorktableNameGenerator::from_table_name(name.to_string()).get_row_type_ident(); + + quote::quote! { + #[allow(dead_code)] + pub type #ident = #ty; + impl worktable::runtime::TableRuntime for #row { type Backend = #ident; } + impl worktable::runtime::RuntimeUnpinned for #row {} + } +} + /// Bake the declaration into the generated code, as the text it was written in. /// /// The point is that a compiled binary should be able to say what schema it was @@ -157,6 +469,125 @@ mod tests { assert!(error.to_string().contains("keep `primary_key`")); } + #[test] + fn columnar_index_requires_columnar_fields() { + let error = expand(quote! { + name: InvalidColumnarIndex, + persist: false, + columns: { + id: u64 primary_key, + host_id: u64, + }, + columnar_indexes: { + host_lookup: { + cluster_by: [host_id], + }, + }, + }) + .unwrap_err(); + + assert!( + error + .to_string() + .contains("requires at least one field declaring `columnar`") + ); + } + + #[test] + fn columnar_field_and_index_generate_scan_projection_and_lookup_apis() { + let output = expand(quote! { + name: ColumnarCodegen, + persist: false, + columns: { + id: u64 primary_key, + host_id: u64 columnar(chunk_rows(1024), compression(none)), + timestamp: i64 columnar(chunk_rows(2048), compression(none)), + }, + columnar_indexes: { + host_time: { + cluster_by: [host_id, timestamp], + }, + }, + }) + .unwrap() + .to_string(); + + assert!(output.contains("columnar_scan_host_id")); + assert!(output.contains("columnar_project_timestamp")); + assert!(output.contains("columnar_select_host_time")); + assert!(output.contains("ColumnarColumn :: new (1024")); + } + + #[test] + fn columnar_config_is_table_scoped_and_row_derives_stops_at_new_keys() { + let output = expand(quote! { + name: ColumnarConfig, + persist: false, + columns: { + id: u64 primary_key, + value: u64 columnar, + }, + config: { + row_derives: Default, + columnar_slot_id: ColumnSlotId16, + columnar_chunk_rows: 1024, + }, + }) + .unwrap() + .to_string(); + + assert!(output.contains("ColumnSlotId16")); + assert!(output.contains("ColumnarColumn :: new (1024")); + } + + #[test] + fn columnar_chunk_override_must_nest_with_table_default() { + let error = expand(quote! { + name: InvalidColumnarChunk, + persist: false, + columns: { + id: u64 primary_key, + value: u64 columnar(chunk_rows(50_000)), + }, + }) + .unwrap_err(); + + assert!(error.to_string().contains("power-of-two multiple or divisor")); + } + + #[test] + fn primary_key_cannot_redeclare_columnar_identity() { + let error = expand(quote! { + name: InvalidColumnarPrimaryKey, + persist: false, + columns: { + id: u64 primary_key columnar, + }, + }) + .unwrap_err(); + + assert!(error.to_string().contains("must not declare `columnar`")); + } + + #[test] + fn duplicate_columnar_config_is_rejected() { + let error = expand(quote! { + name: DuplicateColumnarConfig, + persist: false, + columns: { + id: u64 primary_key, + value: u64 columnar, + }, + config: { + columnar_slot_id: ColumnSlotId16, + columnar_slot_id: ColumnSlotId32, + }, + }) + .unwrap_err(); + + assert!(error.to_string().contains("Duplicate `columnar_slot_id`")); + } + fn assert_composite_primary_key_field_order(output: proc_macro2::TokenStream) { let output = output.to_string(); let get_primary_key = output @@ -481,9 +912,14 @@ mod tests { ); } + /// This used to assert the opposite. A persisted table was refused any page + /// size but 16384, because the seeks computed every offset from a hardcoded + /// constant while the generated table threaded the configured one, so the + /// two disagreed and the file was silently corrupt. Both take the stride as + /// a parameter now. #[test] - fn persisted_tables_reject_non_default_page_size() { - let error = expand(quote! { + fn persisted_tables_accept_a_non_default_page_size() { + expand(quote! { name: PersistedSmallPages, persist: true, columns: { @@ -493,10 +929,27 @@ mod tests { page_size: 8192, } }) + .expect("a persisted table may choose its page size"); + } + + /// What is left of the rule: a page on disk carries a 28-byte header, so + /// one this small is mostly header. + #[test] + fn persisted_tables_reject_a_page_smaller_than_the_floor() { + let error = expand(quote! { + name: PersistedTinyPages, + persist: true, + columns: { + id: u64 primary_key, + }, + config: { + page_size: 64, + } + }) .unwrap_err(); assert!( - error.to_string().contains("cannot be combined with `persist: true`"), + error.to_string().contains("below the 512-byte"), "unexpected error: {error}" ); } @@ -566,6 +1019,7 @@ mod position_tests { name: SymbolPosting, persist: true, partition_by: generation: u32, + partition_max_size: u64, columns: { id: u64 primary_key autoincrement, posting_hash: u64, records_blob: String }, indexes: { posting_idx: posting_hash unique } }) @@ -586,6 +1040,7 @@ mod position_tests { let expanded = expand(quote! { name: Price, partition_by: symbol_id: u16, + partition_max_size: u64, columns: { exchange_id: u8 primary_key, bid: f64 } }) .expect("in-memory partitioned table must expand") @@ -593,18 +1048,285 @@ mod position_tests { assert!(expanded.contains("partition_or_create")); } + /// A wide width keeps the full table, which is what every partitioned + /// declaration had before the width was declarable. + #[test] + fn a_wide_partition_max_size_keeps_the_full_table() { + let expanded = expand(quote! { + name: Price, + partition_by: symbol_id: u16, + partition_max_size: u64, + columns: { exchange_id: u8 primary_key, bid: f64 } + }) + .expect("must expand") + .to_string(); + assert!( + expanded.contains("PartitionSet < PriceWorkTable >"), + "the payload must be the full table: {expanded}" + ); + assert!( + !expanded.contains("PriceDenseTable"), + "no dense payload should be emitted" + ); + } + + /// A narrow one swaps the payload, and only the payload. + #[test] + fn a_narrow_partition_max_size_swaps_the_payload() { + let expanded = expand(quote! { + name: Price, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { exchange_id: u8 primary_key, bid: f64 } + }) + .expect("must expand") + .to_string(); + assert!( + expanded.contains("PartitionSet < PriceDenseTable >"), + "the payload must be the dense table: {expanded}" + ); + // The full table is still generated. It is the type the declaration + // names, and a caller may want one outside the router. + assert!( + expanded.contains("struct PriceWorkTable"), + "the full table is still declared" + ); + } + + /// A key that is not a position is refused, naming the column. + #[test] + fn a_dense_partition_refuses_a_key_that_cannot_be_a_position() { + let error = expand(quote! { + name: Named, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { label: String primary_key, bid: f64 } + }) + .expect_err("a String key has no position to be") + .to_string(); + assert!(error.contains("label"), "must name the column: {error}"); + assert!(error.contains("String"), "must name the type it refused: {error}"); + assert!( + error.contains("partition_max_size: u64"), + "must name the way out: {error}" + ); + } + + /// A composite key is refused for the same reason, and points at the + /// width that takes one. + #[test] + fn a_dense_partition_refuses_a_composite_key() { + let error = expand(quote! { + name: Pair, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { left: u32 primary_key, right: u32 primary_key, bid: f64 } + }) + .expect_err("a composite key has no single position") + .to_string(); + assert!(error.contains("2 primary key columns"), "must say what it saw: {error}"); + assert!( + error.contains("partition_max_size: u64"), + "must name the way out: {error}" + ); + } + + /// A width wider than the key declares rows the key cannot reach. + /// + /// Not a soundness problem, always a mistake: `u16` beside a `u8` key + /// declares 65,536 rows into a partition that can hold 256. + #[test] + fn a_width_the_key_cannot_reach_is_refused() { + let error = expand(quote! { + name: Price, + partition_by: symbol_id: u16, + partition_max_size: u16, + columns: { exchange_id: u8 primary_key, bid: f64 } + }) + .expect_err("a u8 key cannot reach 65,536 rows") + .to_string(); + assert!(error.contains("exchange_id"), "must name the column: {error}"); + assert!(error.contains("65536"), "must say how many rows were declared: {error}"); + assert!(error.contains("partition_max_size: u8"), "must name the fix: {error}"); + } + + /// A narrow key on an unpartitioned table warns. + #[test] + fn a_narrow_primary_key_on_an_unpartitioned_table_is_linted() { + // `using worktables_index` on the `bool` arm: arctic, the default, + // refuses a `bool` key outright, so that arm is only reachable through + // a backend that takes one. It is still worth linting, because WTI does. + for (ty, rows, backend) in [ + ("u8", "256", quote! {}), + ("bool", "2", quote! { using worktables_index }), + ] { + let ty = syn::Ident::new(ty, proc_macro2::Span::call_site()); + let expanded = expand(quote! { + name: Flag, + columns: { id: #ty primary_key #backend, v: u64 } + }) + .expect("must expand") + .to_string(); + assert!( + expanded.contains("NARROW_PRIMARY_KEY"), + "`{ty}` should be linted: {expanded}" + ); + assert!( + expanded.contains(rows), + "the note should say how many rows `{ty}` reaches: {expanded}" + ); + } + } + + /// Beside `partition_by` the same key is correct, so it is silent. + /// + /// This is the half that matters: a narrow key is what makes a dense + /// partition possible, and a lint that fired on it would be telling people + /// to undo the optimisation. + #[test] + fn a_narrow_primary_key_on_a_partitioned_table_is_silent() { + let expanded = expand(quote! { + name: Price, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { exchange_id: u8 primary_key, bid: f64 } + }) + .expect("must expand") + .to_string(); + assert!( + !expanded.contains("NARROW_PRIMARY_KEY"), + "a partitioned narrow key is correct and must not warn: {expanded}" + ); + } + + /// A key wide enough to be a table's is not linted. + #[test] + fn a_wide_primary_key_is_not_linted() { + for ty in ["u16", "u32", "u64", "String"] { + let ty = syn::Ident::new(ty, proc_macro2::Span::call_site()); + let expanded = expand(quote! { + name: Wide, + columns: { id: #ty primary_key, v: u64 } + }) + .expect("must expand") + .to_string(); + assert!(!expanded.contains("NARROW_PRIMARY_KEY"), "`{ty}` must not be linted"); + } + } + + /// A dense partition takes update and delete queries keyed by position. + #[test] + fn a_dense_partition_generates_its_queries() { + let expanded = expand(quote! { + name: Price, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { exchange_id: u8 primary_key, bid: f64, ask: f64 }, + queries: { + update: { TopPrice(bid, ask) by exchange_id, }, + delete: { Stale() by exchange_id, } + } + }) + .expect("must expand") + .to_string(); + assert!( + expanded.contains("impl PriceDenseTable"), + "the dense payload must be emitted: {expanded}" + ); + assert!(expanded.contains("fn update_top_price"), "missing the update query"); + assert!(expanded.contains("fn delete_stale"), "missing the delete query"); + } + + /// Keyed by anything else, it refuses rather than quietly scanning. + #[test] + fn a_dense_query_keyed_by_a_column_it_cannot_index_is_refused() { + let error = expand(quote! { + name: Price, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { exchange_id: u8 primary_key, venue: u32, bid: f64 }, + indexes: { venue_idx: venue }, + queries: { + update: { ByVenue(bid) by venue, } + } + }) + .expect_err("a dense partition has no secondary index") + .to_string(); + assert!(error.contains("venue"), "must name the column: {error}"); + assert!(error.contains("exchange_id"), "must name the key it can use: {error}"); + assert!( + error.contains("partition_max_size: u64"), + "must name the way out: {error}" + ); + } + + /// `in_place` is a synonym here, so it says so rather than generating a + /// second name for one method. + #[test] + fn in_place_on_a_dense_partition_is_refused_as_a_synonym() { + let error = expand(quote! { + name: Price, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { exchange_id: u8 primary_key, bid: f64 }, + queries: { + in_place: { Bump(bid) by exchange_id, } + } + }) + .expect_err("in_place has no meaning on a dense partition") + .to_string(); + assert!(error.contains("already in place"), "must say why: {error}"); + assert!(error.contains("update Bump"), "must name the replacement: {error}"); + } + + /// A dense partition cannot persist, and says so rather than pretending. + #[test] + fn a_dense_partition_refuses_persistence() { + let error = expand(quote! { + name: Price, + persist: true, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { exchange_id: u8 primary_key, bid: f64 } + }) + .expect_err("a dense partition has no persistence engine") + .to_string(); + assert!(error.contains("persist"), "must name the key it cannot honour: {error}"); + assert!( + error.contains("partition_max_size: u64"), + "must name the width that does persist: {error}" + ); + } + + /// The dense payload is a partition payload and nothing else: an + /// unpartitioned declaration never sees one. + #[test] + fn an_unpartitioned_table_gets_no_dense_payload() { + let expanded = expand(quote! { + name: Price, + columns: { exchange_id: u8 primary_key, bid: f64 } + }) + .expect("must expand") + .to_string(); + assert!( + !expanded.contains("DenseTable"), + "nothing to be dense about: {expanded}" + ); + } + #[test] fn partition_by_before_persist_names_the_required_order() { let error = expand(quote! { name: Wrong, partition_by: generation: u32, + partition_max_size: u64, persist: true, columns: { id: u64 primary_key, v: u64 } }) .expect_err("wrong order must be an error") .to_string(); assert!( - error.contains("name, version, persist, partition_by"), + error.contains("name, version, vec, persist, partition_by"), "the error must name the required order, got: {error}" ); } @@ -615,11 +1337,12 @@ mod position_tests { name: Wrong, columns: { id: u64 primary_key, v: u64 }, partition_by: generation: u32, + partition_max_size: u64, }) .expect_err("late partition_by must be an error") .to_string(); assert!( - error.contains("name, version, persist, partition_by"), + error.contains("name, version, vec, persist, partition_by"), "the error must name the required order, got: {error}" ); } @@ -704,6 +1427,7 @@ mod emitted_declarations { survives_the_round_trip(quote! { name: Price, partition_by: symbol_id: u16, + partition_max_size: u64, columns: { exchange_id: u8 primary_key, bid: f64, @@ -859,6 +1583,7 @@ mod schema_const { let declaration = quote! { name: Price, partition_by: symbol_id: u16, + partition_max_size: u64, columns: { exchange_id: u8 primary_key, bid: f64 }, }; @@ -885,4 +1610,267 @@ mod schema_const { let reparsed: TokenStream = syn::parse_str(&baked).expect("tokenises"); expand(reparsed).expect("the baked declaration expands"); } + + /// A paged table cannot take a hash index, and says why. + /// + /// Both halves matter. The refusal has to fire, because the alternative is + /// failing somewhere inside the index generators on a type the author never + /// wrote; and it has to name `vec: true`, because the backend does work + /// there and a refusal that does not say where to go sends people to the + /// issue tracker. + #[test] + fn fxhash_is_refused_on_a_paged_table() { + let on_the_primary_key = expand(quote! { + name: HashedPaged, + columns: { + id: u64 primary_key using fxhash, + value: u64, + }, + }) + .unwrap_err() + .to_string(); + assert!( + on_the_primary_key.contains("vec: true"), + "the refusal must say where the backend does work: {on_the_primary_key}" + ); + assert!( + on_the_primary_key.contains("ordered scan"), + "the refusal must say why: {on_the_primary_key}" + ); + + // And on a secondary, which reaches the same check by the other branch. + let on_a_secondary = expand(quote! { + name: HashedSecondary, + columns: { + id: u64 primary_key, + value: u64, + }, + indexes: { + value_idx: value unique using fxhash, + }, + }) + .unwrap_err() + .to_string(); + assert!( + on_a_secondary.contains("value_idx"), + "the refusal must name the index the author wrote: {on_a_secondary}" + ); + } + + /// A `vec: true` table accepts it, which is what makes the refusal above a + /// redirection rather than a ban. + #[test] + fn fxhash_is_accepted_on_a_vec_table() { + let output = expand(quote! { + name: HashedVec, + vec: true, + columns: { + id: u64 primary_key using fxhash, + value: u64, + }, + }) + .expect("a vec: true table takes a hash index"); + let text = output.to_string(); + assert!(text.contains("FxHashMap"), "the table should hold a hash map"); + assert!( + !text.contains("pub fn range"), + "a hash-backed table must not get a range method" + ); + } +} + +/// What the `runtime:` key generates. +/// +/// The table's runtime is named once, as `#{Name}Runtime`, and these assert the +/// mapping from `codegen::generators::runtime_backend` reaches that alias +/// unchanged. The mapping itself is unit-tested next to the function; what is +/// checked here is that a declaration selects it. +/// +/// Gated on `std` because the alias is: a build with no runtime emits no +/// runtime type. Without the gate these tests pass under `cargo test +/// --workspace`, where feature unification turns `std` on for them, and fail +/// under `cargo test -p worktable_codegen`, where nothing does. A test whose +/// result depends on which crate you ran it from is a false green either way. +#[cfg(all(test, feature = "std"))] +mod runtime_tests { + use quote::quote; + + use super::expand; + + /// Everything up to the alias, so a comparison is not defeated by the + /// unrelated tokens either side of it. + fn runtime_alias(declaration: proc_macro2::TokenStream) -> String { + let output = expand(declaration).expect("expands").to_string(); + let alias = output + .split("pub type SelectRuntime = ") + .nth(1) + .expect("the generated runtime alias"); + alias.split(';').next().expect("the alias body").trim().to_string() + } + + fn declaration(runtime: proc_macro2::TokenStream) -> proc_macro2::TokenStream { + quote! { + name: Select, + persist: false, + columns: { + id: u64 primary_key, + value: u64, + }, + #runtime + } + } + + /// A bare `nagoya` is whatever flavor is currently the default, spelled + /// out of the registry rather than named here, so that moving the default + /// is one edit rather than a hunt through the tests. + #[test] + fn bare_nagoya_selects_the_default_tuning() { + let expected = format!("NagoyaRt < {} >", worktable_dsl::model::Flavor::default().type_name()); + assert_eq!(runtime_alias(declaration(quote! { runtime: nagoya, })), expected); + } + + #[test] + fn each_flavor_selects_its_marker() { + assert_eq!( + runtime_alias(declaration(quote! { runtime: nagoya(locality), })), + "NagoyaRt < Locality >" + ); + assert_eq!( + runtime_alias(declaration(quote! { runtime: nagoya(spread), })), + "NagoyaRt < Spread >" + ); + assert_eq!( + runtime_alias(declaration(quote! { runtime: nagoya(throughput), })), + "NagoyaRt < Throughput >" + ); + } + + #[test] + fn tokio_selects_the_tokio_runtime() { + assert_eq!(runtime_alias(declaration(quote! { runtime: tokio, })), "TokioRt"); + } + + /// The no-regression guarantee, and the reason it is stated on the whole + /// expansion rather than on the alias: every `worktable!` written before + /// this key existed omits it, and none of them may generate a different + /// byte than they would with `runtime: nagoya` written in. + #[test] + fn omitting_the_key_emits_exactly_what_bare_nagoya_emits() { + let omitted = expand(declaration(quote! {})).expect("expands").to_string(); + let declared = expand(declaration(quote! { runtime: nagoya, })) + .expect("expands") + .to_string(); + + assert_same_tokens(&omitted, &declared); + } + + /// Borrowed from `generator_determinism`: two expansions that differ by one + /// token differ by one byte in a string thousands of bytes long, and + /// `assert_eq!` prints both in full rather than saying where. + fn assert_same_tokens(first: &str, second: &str) { + if first == second { + return; + } + let at = first + .bytes() + .zip(second.bytes()) + .position(|(left, right)| left != right) + .unwrap_or_else(|| first.len().min(second.len())); + let start = at.saturating_sub(120); + let first_end = (at + 240).min(first.len()); + let second_end = (at + 240).min(second.len()); + panic!( + "expansions first differ at byte {at}\nfirst: {}\nsecond: {}", + &first[start..first_end], + &second[start..second_end], + ); + } + + /// The free-order position: `runtime` is an arm beside the blocks, so it + /// may be written before or after any of them. + #[test] + fn the_key_may_be_written_before_or_after_the_blocks() { + let before = expand(quote! { + name: Select, + persist: false, + runtime: nagoya(spread), + columns: { id: u64 primary_key, value: u64 }, + }) + .expect("expands") + .to_string(); + let after = expand(declaration(quote! { runtime: nagoya(spread), })) + .expect("expands") + .to_string(); + + assert_same_tokens(&before, &after); + } + + #[test] + fn a_second_runtime_key_is_a_duplicate_section() { + let error = expand(quote! { + name: Select, + persist: false, + columns: { id: u64 primary_key }, + runtime: nagoya, + runtime: tokio, + }) + .unwrap_err() + .to_string(); + + assert!(error.contains("duplicate `runtime` section"), "{error}"); + } + + #[test] + fn an_unimplemented_backend_is_refused_by_name() { + for name in ["forte", "blocking", "bwos"] { + let name: proc_macro2::TokenStream = name.parse().unwrap(); + let error = expand(declaration(quote! { runtime: #name, })).unwrap_err().to_string(); + + assert!(error.contains("recognised but not implemented"), "{error}"); + assert!(error.contains("`nagoya` and `tokio`"), "{error}"); + } + } + + #[test] + fn an_unknown_flavor_is_refused_with_every_flavor_that_exists() { + let error = expand(declaration(quote! { runtime: nagoya(banana), })) + .unwrap_err() + .to_string(); + + assert!(error.contains("unknown nagoya flavor `banana`"), "{error}"); + // Every flavor in the registry, rather than a sentence. Pinning the + // wording is how this test came to fail for adding a flavor, which is + // the one thing it should not object to. + for flavor in worktable_dsl::model::Flavor::ALL { + assert!(error.contains(flavor.name()), "{} missing from: {error}", flavor.name()); + } + } + + #[test] + fn tokio_is_refused_a_flavor() { + let error = expand(declaration(quote! { runtime: tokio(spread), })) + .unwrap_err() + .to_string(); + + assert!(error.contains("`tokio` has no flavors"), "{error}"); + } + + /// The middle step of the fallback chain, at the only site that can show it + /// today: a table that declares a runtime and annotates no section reaches + /// the declared backend, not the built-in default. + #[test] + fn an_unannotated_table_body_takes_the_tables_runtime() { + let alias = runtime_alias(quote! { + name: Select, + persist: false, + runtime: tokio, + columns: { id: u64 primary_key, value: u64 }, + queries: { + update: { Value(value) by id, }, + delete: { ById() by id, }, + } + }); + + assert_eq!(alias, "TokioRt"); + } } diff --git a/codegen/src/worktable_version/mod.rs b/codegen/src/worktable_version/mod.rs index 496d0e7a..72b6278c 100644 --- a/codegen/src/worktable_version/mod.rs +++ b/codegen/src/worktable_version/mod.rs @@ -16,6 +16,12 @@ pub fn expand(input: TokenStream) -> syn::Result { match ident.to_string().as_str() { "columns" => columns = Some(parser.parse_columns()?), "indexes" => indexes = Some(parser.parse_indexes()?), + "columnar_indexes" => { + return Err(Error::new( + ident.span(), + "worktable_version! does not support columnar_indexes", + )); + } "queries" => { return Err(Error::new(ident.span(), "worktable_version! does not support queries")); } @@ -37,7 +43,9 @@ pub fn expand(input: TokenStream) -> syn::Result { columns.indexes = i } - read_only::expand(name, columns, version) + let runtime = crate::worktable::gen_runtime_type(&name, None); + let table = read_only::expand(name, columns, version)?; + Ok(quote::quote! { #table #runtime }) } #[cfg(test)] @@ -120,6 +128,25 @@ mod tests { assert!(res.is_err(), "should reject config section"); } + #[test] + fn test_rejects_columnar_indexes_explicitly() { + let input = quote! { + name: UserV1, + columns: { + id: u64 primary_key, + value: u64, + }, + columnar_indexes: { + value_idx: { + cluster_by: [value], + }, + }, + }; + + let error = expand(input).unwrap_err(); + assert!(error.to_string().contains("does not support columnar_indexes")); + } + #[test] fn test_explicit_version() { let input = quote! { diff --git a/docs/TODO.md b/docs/TODO.md index 15824525..1ae18543 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -3,7 +3,9 @@ What is known to be unfinished, and enough context to act on it without the conversation it came from. Ordered by whether it blocks a release. -Last reviewed 2026-09-04, against `master` after beta.17 publication. +Last reviewed 2026-09-07. Sections below still describe the repository as of +beta.17; `master` is now at 1.0.0-beta.19 and this file has not been swept for +what those two releases closed. ## Closed, and how @@ -80,9 +82,10 @@ The complete evidence and beta.13/beta.15/beta.17 performance grids are in benchmark workspace also passes its all-target test-mode gate against the local WorkTable/WTI/DataBucket/ps-reclaim stack. -The placeholder ignored S3 probe still rejects its literal `test` endpoint -before I/O, but configured runtime coverage is now complete through the local- -source support.cafe consumer. Beta.17 downloaded the live Tigris dataset, +The S3 engine now has a stateful offline object-service test covering immutable +chunk upload, table-manifest restore, and an interrupted manifest commit. Configured +runtime coverage is also complete through the local-source support.cafe consumer. +Beta.17 downloaded the live Tigris dataset, recovered three legacy tables with missing secondary entries, rebuilt them into a rollback-safe prefix, strict-loaded all six tables, performed an S3-backed mutation and reloaded it after restart. ACME, HTTPS and WebSocket startup also @@ -111,46 +114,115 @@ yank beta.16 once beta.17 supersedes it. ## Not blocking, but wrong today -### `congee-wt` still pulls `crossbeam-epoch` - -beta.16 removes crossbeam from WorkTable's own reclamation, not from the build. -`congee-wt` depends on it directly and re-exports its `Guard`, which -`src/index/congee.rs:101` names in a signature; `crossbeam-skiplist` also -arrives under `WorkTablesIndex` and `indexset`. - -congee's use is shallow: 34 references, none of them `Atomic<>`, `Owned::` or -`Shared<>`, and most in tests. It only ever calls `pin()` and passes `Guard` -around as an opaque token, so porting it to `ps-reclaim` is mechanical. The -catch is that `Guard` is in congee-wt's public API, so it is a breaking change -there plus the call sites here. - -`arctic-wt` should **not** be ported. It reclaims through `seize`, and is right -to: a trie with short reads reaches quiescence constantly, which is the exact -property that makes `seize` wrong for this crate, where `select` holds a read -guard. - -### Persistence stalls on a primary index event gap, rarely - -One run of `cargo test --workspace --all-targets --all-features` failed with - - persistence stalled on primary index event gap: last applied Id(1439), - next available Id(1455) (attempt 9) - -in `tests/persistence/loaded_index_growth.rs`. Not a flaky timeout: the guard -at `src/persistence/operation/batch.rs:346` is deliberate, added in `c0c06ba`, -and its comment says a gap that persists past eight deferrals means an event id -was consumed without its event being queued, which only non-CDC index mutations -do. The gap is 16 ids wide. - -1 failure in 6 full runs on this branch, 0 in 3 on master, 0 in 15 -persistence-only runs, so it needs whole-suite load and is not a beta.16 -regression. Do not start with a repro hunt: instrument `IndexChangeEventId` -assignment against event queueing so the next occurrence names its own cause. -Evidence at `~/code/wt-event-gap-2026-09-01.txt`. - +### `congee-wt` no longer pulls `crossbeam-epoch` + +Corrected 2026-09-07. This section said the port was outstanding and mechanical. +Both halves were wrong. + +`congee-wt` dropped `crossbeam-epoch` in 0.4.4. `Cargo.toml:22` now reads +`ps-reclaim = { version = "0.1.4", default-features = false, features = +["libc", "spin"] }`, no `crossbeam_epoch` reference survives in its sources or +tests, and this repository's `Cargo.lock` already resolves `congee-wt 0.4.4`. +The two remaining `crossbeam` strings in that crate are attribution comments on +a seqlock and a backoff loop. + +The call site this section worried about needed no change. It has moved to +`src/index/congee.rs:120` and still reads +`fn retire_old(pointer: usize, guard: &congee::epoch::Guard) -> Arc`. The +`congee::epoch::Guard` path was deliberately preserved across the port, and the +lifetime the new guard carries elides in reference position. + +The port was not the mechanical rename described here. A literal swap would have +kept one global epoch; what shipped gives each tree its own `Domain`, adds a +bounded pending-retire batch, checks guard provenance so a guard from another +tree panics rather than corrupting, and drains the tree's own domain on `Drop`. +Worth knowing because the new `Guard` is `!Send` and tree-scoped, so a guard may +not be created outside the thread and tree that uses it. Nothing in either +repository does; every threaded test builds its guard inside the spawned +closure. + +### Persistence event gap: three leak sites found, instrumented, and fixed + +Updated 2026-09-07. This section previously said the cause was unknown and that +the next step was instrumentation rather than a repro hunt. The instrumentation +was built, and reading for it found the leaks. + +`IndexChangeEventId` is `indexset::cdc::change::Id`, allocated by +`event_id.fetch_add` in the same statement that stamps the event, so indexset +never consumes an id without emitting its event. Every leak is on our side: an +event handed back and then dropped. Three sites, all in generated persisted +query code, all on secondary streams, all confirmed by reading: + +- `codegen/src/generators/persist/queries/update.rs:569`. The + `IndexError::NotFound => Err(WorkTableError::NotFound)` arm returns with no + acknowledge, while its sibling `AlreadyExists` arm immediately above builds an + `Acknowledge` carrying `merged_events` and applies it. The events from + `process_difference_insert_cdc` are dropped on the `NotFound` path. This + asymmetry between two adjacent arms is the clearest of the three. +- `codegen/src/generators/persist/queries/update.rs:593`, + `gen_process_diffs_remove_on_index`: `let (secondary_keys_events_remove, res) + = ...; res?;`. On `Err` the `?` returns before + `op.extend_secondary_key_events`, dropping the events bound on the line above. +- `codegen/src/generators/persist/queries/delete.rs:101`: the same `res?;` shape + after `delete_row_cdc`. + +Checked and NOT leaks: the rollback arms of `insert_cdc`, `insert_many_cdc` and +`reinsert_cdc` in `src/table/mod.rs` all merge forward and rollback events into +an `Acknowledge`, as does the data-delete-failure restore path. Vacuum's +`update_index_after_move` takes the non-CDC branch only when `persistence` is +`None`, so the one case commit `c0c06ba` named is closed for persisted tables. + +The only structurally possible primary-stream leak is a refused +`apply_operation` after the index mutation already consumed ids, and more +generally any `?` between a CDC index mutation and its `apply_operation`. + +**The failure text quoted in earlier versions of this section is stale.** It +said "attempt 9"; `GIVE_UP_AFTER_ATTEMPTS` is now 120, and +`COLLECT_WHOLE_QUEUE_AFTER_ATTEMPTS` plus its regression test +`collection_recovers_when_event_order_and_operation_order_disagree` were added +since, for a symptom that reads identically but is a collection failure rather +than a leak. Telling those two apart is exactly what the new ledger does. + +`src/persistence/event_ledger.rs` records queued, collected, requeued, trimmed +and applied per stream in a bounded 8192-id window, and the guard's message now +ends in a verdict: either ASSIGNED BUT NEVER QUEUED with the id range and the +producer sites either side of the gap, or QUEUED BUT NOT APPLIED with the per-id +stage history. It says so plainly when part of the gap fell outside the retained +window, so it never claims "never queued" about an id it cannot answer for. +Always compiled, gated at run time on `debug_assertions` or `WT_EVENT_LEDGER`, +which puts it on exactly where the bug appears, since the stall needs a full +debug `--all-features` run. + +**Fixed 2026-09-08.** All three sites now do what the rollback arms already did: +build an `Acknowledge` carrying the orphaned events and apply it before +propagating the error. The two `res?` sites became `if let Err(e) = res` so the +events are moved into the acknowledge and the error is returned explicitly, and +the `NotFound` arm acknowledges the events its sibling arm was already +acknowledging. + +The events are **moved** into the acknowledge rather than cloned, and that is +load-bearing rather than tidy. Cloning them fails to compile: the events type is +still an inference variable at that point in the generated code, pinned only by +the `op.extend_secondary_key_events` call further down, and method resolution for +`.clone()` needs the type resolved where the call is written. The result is an +`E0282` reported against the `worktable!` invocation with no inner span, which is +an expensive thing to diagnose twice. + +Covered by emitted-token assertions in both generators +(`indexed_update_write_failure_unwinds_and_acknowledges` and +`delete_data_failure_restores_indexes_and_acknowledges`), which assert the +acknowledge is emitted **before** the return or the extend rather than merely +present somewhere in the output. The write failure itself is not forcible through +the public API, so the wiring is pinned on the tokens, which is the same approach +those tests already took. + +The in-memory generator has the same two `NotFound` arms +(`codegen/src/generators/in_memory/queries/update.rs:528` and `552`) and they are +correctly untouched: there is no persistence stream behind them to gap. ## Housekeeping -- `CHANGELOG.md` stops at 0.4.1, long before the 1.0.0-beta line. +- `CHANGELOG.md` was backfilled to 0.3.10 on 2026-09-08. It previously stopped at + beta.18. - `.github/workflows/rust.yml` has no `cargo fmt --check` job, so formatting drift accumulates unnoticed; `scripts/ci-local.sh` does check it, which makes the script stricter than CI rather than equal to it. diff --git a/docs/cell-lock-registry.md b/docs/cell-lock-registry.md new file mode 100644 index 00000000..4f0411e8 --- /dev/null +++ b/docs/cell-lock-registry.md @@ -0,0 +1,70 @@ +# Archived-row lock stripes + +Each 16 KiB data page keeps 256 fixed reader/writer states outside its archived +image. Every archived-row offset is mixed across all of its bits before it is +assigned a stripe. Readers increment the stripe's reader count. A writer sets +its writer bit, which stops new readers, and waits for existing readers to +leave. + +Rows that collide may read concurrently because neither mutates bytes. A write +waits for every reader or writer on the same stripe, including an unrelated +row that happens to collide. This is conservative exclusion: it can delay an +operation, but it cannot let a reader overlap a write to the same row. + +## Released-collision bug + +Review on 12 September 2026 reproduced this interleaving on commit 85113ce: + +1. A reader of row A occupies the home slot. +2. Row B hashes to the same home slot, so its reader occupies the next slot. +3. A's last reader releases the home slot. +4. A new reader of B sees the empty home slot and claims it, without finding + B's existing reader in the next slot. + +The two readers then protect one row with different atomic states. A writer +can acquire one state while the other still has readers. This violates the +archived-byte synchronization contract. + +## Why stripes replace registration + +The first repair assigned vacant exact-row entries under a per-page mutex. A +random lookup normally outlived its entry for only one guard, so almost every +read needed that mutex. On the twelve-client read control, throughput fell +from roughly 140 to 147 million operations per second to roughly 48 million. + +Fixed stripes have no key assignment, reclamation, scan or registration lock. +The stripe is a pure function of the row offset, so all access to one row +always reaches one atomic state. Mixing matters because archived row starts +are aligned: masking the low offset bits directly would collapse common row +sizes into only a few stripes. + +The states occupy 1 KiB per 16 KiB data page. There is no heap allocation on +acquisition. They remain runtime-only, so no lock state is serialized and the +binary format and grammar do not change. + +## Verification + +Native regressions cover stable mapping for colliding offsets, mixing of offsets +that share low bits, page serialization and reset. The ordinary workspace CI +sequence also exercises concurrent publication, updates, deletion, vacuum and +reopen. + +The production acquisition/drop code substitutes Loom atomics under `wt_loom`. +Two bounded models check same-row read/write exclusion and colliding-offset +exclusion against a Loom-tracked payload, with two preemptions. These are +bounded safety checks, not an exhaustive liveness proof or a claim about all +possible workloads. + +Run from the WorkTable checkout, with the matching release dependencies: + +```sh +cargo test --lib in_memory::data::tests +RUSTFLAGS='--cfg wt_loom' cargo test --release --lib cell_lock_models +scripts/ci-local.sh +``` + +The corrected stripe implementation restores single-client reads to the +pre-bug baseline. At twelve clients, two reversed-order repetitions measured +about 157 to 167 million shared-table reads per second across balanced, +almost_tokio and Tokio, roughly 10 to 16 percent above the pre-bug baseline. +The exact report and source hashes live in the companion performance suite. diff --git a/docs/columnar-fields-and-indexes-guide-v3.md b/docs/columnar-fields-and-indexes-guide-v3.md new file mode 100644 index 00000000..7fb66481 --- /dev/null +++ b/docs/columnar-fields-and-indexes-guide-v3.md @@ -0,0 +1,404 @@ +# WorkTable tabular + columnar side indexes + +> **v3 review boundary:** this guide describes the implemented tabular + columnar side-index +> flavor, and labels the separate full-columnar roadmap explicitly. + +WorkTable can maintain selected fields in derived, chunked **side indexes** while its ordinary row +store and primary key remain authoritative. Point-oriented application code keeps the existing +WorkTable model, while analytical code gains cheaper columnar-flavored scans and projections over +the duplicated selected values. + +The first implementation is intentionally format-compatible and uncompressed. It establishes the +DSL, identity, mutation, validation, and recovery boundaries before adding sealed chunks, +compression, a vector execution engine, or a new disk format. + +## The three WorkTable storage flavors + +| Flavor | Authoritative representation | What it provides | Status | +|---|---|---|---| +| **Tabular** | Existing WorkTable rows | Point lookup, mutation, ordinary indexes and persistence | Already well covered | +| **Tabular + columnar side indexes** | Existing WorkTable rows, plus derived side structures | Cheap columnar-flavored field scans, projections and ordered lookup at the cost of duplicating selected values/index metadata | **This proposal and PR** | +| **Columnar** | A true vector/column representation | Vector batches, encoded or compressed segments, columnar-native execution and persistence | Not covered | + +The middle flavor is not a full column store. Calling it one would erase the most useful property +of the proposal: it adds a bounded, opt-in analytical acceleration structure without replacing the +tabular engine. It is analogous to adding an index, except that a useful side index must duplicate +the selected field values as well as ordering metadata. If it stored only keys and slot positions, +every projection would still perform random gathers from tabular rows and lose most of its +columnar flavor. + +## Complete schema + +```rust +use worktable::prelude::*; +use worktable::worktable; + +// WorkTable's current column parser accepts a single type identifier. +type DiagnosticBlob = Vec; + +worktable!( + name: HistoricalCpu, + persist: true, + + columns: { + id: u128 primary_key, + + // Bare `columnar` uses the table defaults. + host_id: u64 columnar, + captured_at_ns: u64 columnar, + cpu_percent: f32 columnar, + + // A non-indexed columnar field. It has contiguous field storage and + // can be scanned or projected even though no index clusters by it. + status: String columnar(chunk_rows(16_384)), + + // An ordinary row-only field. + diagnostic_blob: DiagnosticBlob, + }, + + columnar_indexes: { + host_time: { + cluster_by: [host_id, captured_at_ns], + }, + }, + + config: { + columnar_slot_id: ColumnSlotId32, + columnar_chunk_rows: 65_536, + }, +); +``` + +Both columnar `config` entries shown above are defaults and may be omitted. + +There is deliberately no table-level `layout: columnar`: this proposal does not select the third +flavor. A field opts into a derived side index. There is also deliberately no `columns: [...]` list inside a columnar index: fields named +by `cluster_by` are its key, and projected values come from base column stores. + +## Three independent choices + +### 1. Which fields have column storage? + +Add `columnar` to a non-primary-key field: + +```rust +cpu_percent: f32 columnar, +status: String columnar(chunk_rows(16_384)), +latency_ns: u64 optional columnar(compression(none)), +``` + +This duplicates that field in a chunked derived side index. It does not create an ordered search index and it +does not change row persistence. A field may be columnar without appearing in any +`columnar_indexes` entry; `status` in the complete example is one such field. + +`columnar` occurs after `optional` and before `using`. The table default is 65,536 rows per chunk. +A per-field `chunk_rows(N)` override must be a power-of-two multiple or divisor of the table +default, which keeps cross-column chunk boundaries nestable. + +Mutable chunks are currently unencoded. Omitted compression and `compression(none)` mean the same +thing. `auto`, `delta`, `rle`, and `dictionary` are reserved and fail macro expansion instead of +silently behaving like `none`. + +### 2. Which access paths are ordered? + +```rust +columnar_indexes: { + host_time: { + cluster_by: [host_id, captured_at_ns], + }, +}, +``` + +`cluster_by` orders the index metadata, not the physical base columns. In the current +implementation, the access path is a `BTreeMap` from the composite key to a set of column slot IDs. +The base columns remain in canonical slot order. + +Every `cluster_by` field must itself declare `columnar`. `include` is reserved for a future genuine +covering projection and is rejected today. + +### 3. How wide is the compact slot position? + +The table-wide setting is: + +```rust +config: { + columnar_slot_id: ColumnSlotId16, +}, +``` + +Available types and theoretical live-slot capacities are: + +| Type | Positions | Typical reason to choose it | +|---|---:|---| +| `ColumnSlotId8` | 256 | tests or a strictly bounded tiny table | +| `ColumnSlotId16` | 65,536 | embedded or hard-bounded live window | +| `ColumnSlotId32` | 4,294,967,296 | default; broad capacity with compact metadata | +| `ColumnSlotId64` | 18,446,744,073,709,551,616 | explicit very-large logical range | + +These are capacity bounds, not promises that the process can allocate that many rows. Address +space, memory, and other table structures impose practical limits first—especially for 64-bit +slots. + +Choosing a width that covers the maximum number of simultaneously live columnar rows is the +schema author's responsibility. WorkTable does not silently widen, truncate, wrap, evict another +row, or reinterpret the setting. A write beyond the selected range returns: + +```rust +WorkTableError::ColumnSlotIdExhausted(bits) +``` + +The failed insert is rolled back and existing rows remain valid. Operators can monitor: + +```rust +table.columnar_slots_in_use(); +table.columnar_slots_high_water(); +``` + +## A slot is not an identity or sort key + +The primary key remains load-bearing. A `ColumnSlotId` is only a compact position shared by the +derived field chunks and columnar indexes. It is not: + +- a replacement primary key; +- the row's rank in `cluster_by` order; +- a stable external identifier; +- a durable identifier across restart; or +- a public tuple value applications should store. + +Generated results carry an opaque reference: + +```rust +pub struct ColumnarRowRef { /* private */ } + +impl ColumnarRowRef { + pub fn primary_key(&self) -> &PrimaryKey; +} +``` + +`ColumnarRowRef` deliberately does not implement serialization. Durable application references +must store the primary key. + +### Delete/reinsert and ABA safety + +A bounded slot allocator must reuse positions. Primary key plus slot alone is insufficient: delete +and reinsert of the same primary key into the same slot would make an old reference appear valid. + +WorkTable therefore validates four pieces of state: + +```text +primary key + slot position + u64 slot generation + table incarnation +``` + +The generation is separate from the configured slot width, so choosing `ColumnSlotId8` still gives +256 live slots rather than carving generation bits out of those eight bits. A delete increments the +slot's generation before reuse. Generation never wraps: if its `u64` counter is ever exhausted, +that slot is permanently retired. A process-local table incarnation invalidates references created +by another table instance or before a persisted table is reopened. + +This is stronger for retained owned references than relying only on an epoch grace period: an +epoch protects active readers, but it cannot protect a reference an application stores after the +read guard ends. + +## Current side-index physical model + +For each generated table with at least one columnar field, WorkTable keeps the tabular row and adds: + +```text +authoritative primary key -> authoritative WorkTable row/link + -> (ColumnSlotId, generation) directory + -> chunked base field replicas + -> zero or more clustered BTreeMap access paths +``` + +Each duplicated side field currently uses `Vec>>`. The outer vector holds chunks and the inner +vector is indexed by the slot offset. A separate primary-key column supports reference validation. +This first layout is deliberately generic: + +- fixed-width values are not yet exposed through SIMD/vector kernels; +- optional fields currently store the Rust `Option` representation rather than a validity + bitmap; +- `String` values are owned values rather than offsets into a byte arena; and +- chunks are mutable and uncompressed. + +These are implementation boundaries, not compression or vectorization claims. + +## Generated API in this implementation + +The current generated methods return owned collections: + +```rust +// Exact lookup requires the complete composite clustered key. +let refs = table.columnar_select_host_time(host_id, captured_at_ns)?; + +// Gather a selected field through validated opaque references. +let cpu = table.columnar_project_cpu_percent(&refs)?; + +// A direct scan needs no columnar index. +let statuses = table.columnar_scan_status()?; + +// Scan in the clustered index's key order. +let ordered_refs = table.columnar_scan_host_time()?; +``` + +The owned `Vec` results keep locks out of the public return type and may be retained safely. They +also materialize the result, so this API is not the final high-volume execution surface. + +Not yet implemented: + +- prefix equality and range predicates on a composite `cluster_by` key; +- namespaced predicate/projection builders; +- zero-copy or callback-based `scan_batches`; +- a single multi-field projection API; and +- row-gather fallback for row-only fields. + +Those remain the next query-API slice. Documentation and benchmarks must not present them as +shipping behavior. + +## Mutation and consistency behavior + +Insert, ordinary update, delete, and reinsert maintain the derived directory, field chunks, and +clustered metadata before the mutation call completes. A same-primary-key update retains its slot +and generation. Vacuum may move the authoritative physical link without changing the columnar +slot. + +The complete derived state is protected by one table-local read/write lock. Consequences: + +- an individual scan or projection observes one coherent columnar snapshot; +- all columnar fields changed by one maintenance operation publish together; +- concurrent columnar writers are table-serialized; +- ordinary row-only selects do not take the columnar lock; and +- two separate API calls are two snapshots, not a transactionally pinned multi-call view. + +Some archived in-place mutation paths cannot apply a typed column delta yet. They mark the +derived replica dirty. The next columnar read rebuilds it from authoritative rows. Applications +can see and schedule that cost explicitly: + +```rust +if table.columnar_is_dirty() { + table.rebuild_columnar()?; +} +``` + +The current rebuild is whole-table and holds the columnar writer lock. Per-chunk dirty tracking is +planned; it is not implemented in this release. + +## Persistence and recovery + +For `persist: true`, ordinary WorkTable rows and indexes retain their existing formats. The +columnar side index is marked derived, omitted from the persisted index structure, and rebuilt from +authoritative rows after load. Therefore this change introduces no new on-disk columnar format. + +`ColumnSlotId` assignments are not promised to survive restart. The process/table incarnation in +`ColumnarRowRef` prevents an old in-memory reference from being accepted by a reopened instance. +Use primary keys for durable identity. + +Native columnar checkpoints are a later design choice. They should be added only if benchmarked +restart time or row-store gather cost justifies another durable format and recovery protocol. + +## Relationship to SAP HANA's unified table + +Sikka et al.'s SAP HANA paper is the most relevant architectural contrast because it explains how +a true column-oriented system can serve transactional and analytical work on one logical table. +HANA primarily informs WorkTable's possible third flavor, while this PR deliberately implements +the cheaper middle flavor. The similarity is the use of different physical representations for +different access patterns; the authority and execution models differ. + +| Dimension | SAP HANA (SIGMOD 2012) | WorkTable side indexes / PR implementation | +|---|---|---| +| Logical goal | OLTP and OLAP through one unified table interface | Preserve the tabular engine and add opt-in columnar-flavored side indexes | +| Write path | Uncompressed row-oriented L1 delta | Existing authoritative WorkTable row storage | +| Intermediate form | Dictionary-encoded, unsorted column L2 delta | Mutable, uncompressed side-index vectors | +| Read-optimized form | Compressed main store with sorted dictionaries and bit-packed values | Not implemented yet | +| Record position | RowId created on entry; positional alignment across columns | Primary key is authoritative; opaque slot aligns derived columns | +| Reorganization | Asynchronous L1→L2 and snapshot-safe L2→main merges | Synchronous maintenance; whole-table rebuild only for dirty fallback paths | +| Readers during merge | Old/new versions retained until transactions using the old version finish | One table-local columnar `RwLock`; no versioned columnar merge yet | +| Point access | Inverted indexes across delta and main structures | Generated `BTreeMap` clustered metadata plus the normal WorkTable indexes | +| Execution | Row/column iterators and vectorized block-at-a-time operators | Owned `Vec` scans/projections in this first API | +| Durability | REDO for incoming changes plus savepoints for column structures | Existing WorkTable persistence is authoritative; columnar state rebuilds after load | + +The closest honest analogy is: **WorkTable's tabular engine plays a role similar to HANA's +write-optimized L1, while this PR adds optional uncompressed side indexes. It does not implement +HANA's L2/main column-store lifecycle and does not claim the third, fully columnar flavor.** + +The HANA result points to a defensible next architecture for WorkTable: + +- keep a small mutable delta that accepts foreground mutations cheaply; +- seal cold column chunks into immutable snapshots; +- build dictionaries, bit packing, delta/RLE, and zone metadata off the foreground path; +- publish a new manifest atomically while readers finish on the previous snapshot; +- reclaim old snapshots only after the read grace period; and +- checkpoint sealed chunks separately from the authoritative row representation only when the + recovery and performance measurements justify it. + +That architecture belongs to a future full-columnar effort and would also remove the current +whole-table dirty-rebuild cliff. It is a roadmap informed by HANA's record-lifecycle design, not a +performance claim for this PR or a requirement for the side-index flavor. + +Reference: Vishal Sikka, Franz Färber, Wolfgang Lehner, Sang Kyun Cha, Thomas Peh, and Christof +Bornhövd. “Efficient Transaction Processing in SAP HANA Database: The End of a Column Store Myth.” +SIGMOD 2012, pp. 731–741. DOI: 10.1145/2213836.2213946. + +## Compile-time validation + +The macro rejects: + +- `columnar` on a primary-key field; +- an unknown or non-columnar field in `cluster_by`; +- empty or duplicate `cluster_by` entries; +- a columnar field/index generated-method name collision; +- the removed `columns:` index property; +- the reserved `include:` property; +- unsupported compression policies; +- zero or non-nesting chunk sizes; +- duplicate table config entries; +- unknown or out-of-order column attributes; and +- `columnar_indexes` in `worktable_version!`. + +## Benchmark contract before performance claims + +For each supported primary-index backend (`WorkTablesIndex`, `congee-wt`, and `arctic-wt` where the +`Using` and persistence rules permit it), measure: + +- row select throughput and p50/p95/p99 with no columnar declarations, fields only, and fields plus + clustered metadata; +- insert, same-key update, indexed-key update, delete, and fixed-window delete/reinsert churn; +- full field scan, exact clustered lookup, and projection gather; +- single-thread and 1→core-count concurrent readers/writers; +- memory amplification, allocation count, code size, and slot-directory overhead by width; +- first-read dirty rebuild versus application-scheduled rebuild; +- persisted reload and rebuild time; and +- correctness counters alongside throughput, especially under slot reuse and concurrent mutation. + +Until those measurements exist, the safe statement is that the feature adds correctness-tested +columnar side indexes—not that it is a full column store, faster for every workload, or ready for +latency-sensitive HFT deployment. + +## Staged roadmap + +- **Query surface:** namespaced predicate builders, prefix/range selection, combined projection, + and bounded `scan_batches`. +- **Incremental maintenance:** per-chunk dirtiness and typed in-place deltas. +- **Column encodings:** validity bitmaps, fixed-width vector kernels, and offset/byte storage for + variable-width fields. +- **Separate full-columnar design:** mutable delta, sealed immutable chunks, background merge, + versioned publication/reclamation, and vector execution. This is a different flavor, not a + silent expansion of the side-index feature. +- **Compression:** type-checked dictionary, delta, RLE, bit packing, and an evidence-based `auto`. +- **Optional native persistence:** manifests, checksums, recovery watermarks, and crash tests. +- **Optimizer:** choose row lookup, ordinary index, clustered side-index lookup, or base-field scan + from measured costs. + +This ordering keeps correctness and compatibility ahead of compression claims while leaving the +DSL stable for the later physical evolution. + +## Reviewer decision points + +- Is “tabular + columnar side indexes” the right permanent name for this middle flavor? +- Is full-width `ColumnSlotId8|16|32|64` plus a separate `u64` generation preferable to hiding a + smaller slot/generation bit split inside the configured width? +- Is the owned, fully materialized phase-one API acceptable if `scan_batches` is the next query + slice and no vector-execution claim is made now? +- Is rebuild-on-load the correct compatibility choice until native side-index checkpoints show a + measured recovery benefit? +- Should the future full-columnar flavor receive distinct DSL rather than changing the meaning of + today's field-level `columnar` attribute? diff --git a/docs/columnar-index-plan.md b/docs/columnar-index-plan.md new file mode 100644 index 00000000..c1b3ad40 --- /dev/null +++ b/docs/columnar-index-plan.md @@ -0,0 +1,142 @@ +# Columnar side-index implementation plan + +Status: phase-one implementation in `feat/columnar-fields-indexes`. The complete user and reviewer +guide is [`columnar-fields-and-indexes-guide-v3.md`](columnar-fields-and-indexes-guide-v3.md). + +## Scope boundary + +WorkTable has three distinct storage flavors: + +1. **Tabular** — the existing authoritative row engine. +2. **Tabular + columnar side indexes** — the scope of this branch. Selected field values and + clustered keys are duplicated into derived structures for cheaper columnar-flavored access. +3. **Columnar** — an authoritative vector layout, vectorized execution, sealed/encoded segments, + and native columnar persistence. This is not implemented by this branch. + +The phase-one feature must not be marketed or documented as the third flavor. + +## Accepted DSL + +```rust +worktable!( + name: HistoricalCpu, + persist: true, + columns: { + id: u128 primary_key, + host_id: u64 columnar, + captured_at_ns: u64 columnar, + temperature: i64 columnar(chunk_rows(32_768), compression(none)), + status: String columnar, + diagnostic_blob: DiagnosticBlob, + }, + columnar_indexes: { + host_time: { + cluster_by: [host_id, captured_at_ns], + }, + }, + config: { + columnar_slot_id: ColumnSlotId32, + columnar_chunk_rows: 65_536, + }, +); +``` + +- Bare `columnar` uses defaults. +- `compression(none)` is the only accepted policy until a codec exists. +- `cluster_by` orders index metadata, not the side-field vectors. +- `columns:` inside a columnar index has been removed as semantically redundant. +- Slot width is a table setting, independent of whether the table declares any clustered side + index. + +## Identity and capacity + +`ColumnSlotId8|16|32|64` uses its complete unsigned range for side-index slot positions. It neither +replaces the primary key nor represents sort rank. + +An opaque `ColumnarRowRef` carries: + +```text +primary key + slot + separate u64 generation + table incarnation +``` + +Delete increments the generation before slot reuse. Generation never wraps; an exhausted slot is +retired. Table incarnation invalidates retained refs across a new/reopened instance. The ref is not +serializable and exposes only the authoritative primary key. + +The schema author is responsible for selecting a width that covers maximum simultaneously live +side-indexed rows. Exceeding it returns `WorkTableError::ColumnSlotIdExhausted(bits)` and rolls back +the insert. The implementation never widens, truncates, wraps, or evicts automatically. + +## Implemented side structures + +Generated tables maintain under one table-local `RwLock`: + +- primary-key → `(ColumnSlotId, generation)` directory; +- reusable slot set and generation vector; +- process/table incarnation; +- chunked `Vec>>` for each opted-in field; +- primary-key side column for ref validation; and +- a `BTreeMap>` for each `columnar_indexes` entry. + +Insert/update/delete/reinsert hooks maintain these after the authoritative row mutation. A vacuum +link change does not change the slot. In-place paths without a typed delta mark the side indexes +dirty; `rebuild_columnar()` lets applications pay the whole-table rebuild cost deliberately. + +Persisted tables skip these derived fields in their existing index disk format and rebuild them +from authoritative rows after load. This branch adds no on-disk format. + +## Current generated operations + +```rust +table.columnar_select_host_time(host_id, captured_at_ns)?; +table.columnar_scan_host_time()?; +table.columnar_scan_status()?; +table.columnar_project_temperature(&row_refs)?; +table.columnar_is_dirty(); +table.rebuild_columnar()?; +table.columnar_slots_in_use(); +table.columnar_slots_high_water(); +``` + +They return owned `Vec` collections. Full-key equality is the only clustered predicate in phase +one. + +## Phase-one correctness gates + +- Same-primary-key delete/reinsert into the same slot must invalidate the old ref. +- A ref from another table incarnation must fail validation. +- Slot exhaustion must roll back the authoritative mutation. +- All four slot widths must enforce their numeric range without wrapping. +- Mutation paths must update or dirty side indexes before returning. +- A dirty rebuild must preserve live slot/generation mappings. +- Macro validation must reject primary-key `columnar`, unknown/non-columnar cluster keys, duplicate + keys/names/config, inert compression, non-nesting chunks, `columns:`, and reserved `include:`. +- Persisted load must reconstruct side indexes without changing the current disk format. + +## Performance gates + +Before an HFT-facing claim or default: + +- compare tabular baseline against fields-only and fields-plus-clustered side indexes; +- measure row select, insert, update, delete, churn, exact lookup, scan, and gather; +- report p50/p95/p99, allocations, memory, and code size; +- run 1→core-count concurrency with correctness counters; +- measure first-reader and explicit dirty rebuild costs; and +- repeat across supported WorkTablesIndex, congee-wt, and arctic-wt `Using` configurations. + +## Follow-up within the side-index flavor + +1. Namespaced builders and prefix/range predicates. +2. Bounded `scan_batches` and one-lock multi-field projection. +3. Per-chunk dirty tracking and typed in-place deltas. +4. Validity bitmaps, fixed-width kernels, and variable-width offset buffers. +5. Optional sealed side-index snapshots if benchmarks justify persistence. + +## Separate full-columnar flavor + +SAP HANA's unified-table record lifecycle is useful prior art for a future third flavor: an +uncompressed row write delta, a column delta, a compressed main, asynchronous merge, old/new +snapshot coexistence, and vector/block execution. That is a separate architecture and performance +contract. It must not arrive by quietly changing what `columnar` side indexes mean. + +See the v3 guide for the detailed comparison and citation. diff --git a/docs/crate.md b/docs/crate.md index 993dc840..39fd4312 100644 --- a/docs/crate.md +++ b/docs/crate.md @@ -5,10 +5,27 @@ primary and secondary indexes, generated CRUD/query methods, optional local or S3-backed persistence, and per-table concurrency. It is not a SQL database and does not provide multi-table transactions or multi-process access. +Since 1.9 it also builds without `std`. A consumer with +`default-features = false` can invoke the macro and use `insert`, `select` and +`select_all`; persistence, vacuum and the disk index are the parts that need an +operating system, and they are gated out. + +Three things a declaration can now choose that it could not before: + +- **The index backend**, with `using`. The default is `arctic`, which takes + fixed-width keys only, so an index over an optional or variable-width column + must say `using worktables_index`. +- **Columnar storage**, with `columnar` on a column and a `columnar_indexes` + block. A columnar field is stored column-wise as well as row-wise, so a scan + over it reads only that field's bytes. +- **The async runtime**, with `runtime: nagoya()` or `runtime: tokio`. + The flavors are scheduler tunings over one pool, not different schedulers. + Take the default unless a measurement says otherwise. + ## In-memory quick start ```rust -# fn main() { futures::executor::block_on(async { +# fn main() { nagoya::block_on(async { use worktable::prelude::*; use worktable::worktable; @@ -38,7 +55,7 @@ String and tuple primary keys accept borrowed forms, so callers do not need to write an explicit clone merely to perform a lookup or delete. ```rust -# fn main() { futures::executor::block_on(async { +# fn main() { nagoya::block_on(async { use worktable::prelude::*; use worktable::worktable; diff --git a/docs/hash-backend-survey.md b/docs/hash-backend-survey.md new file mode 100644 index 00000000..9966b5b7 --- /dev/null +++ b/docs/hash-backend-survey.md @@ -0,0 +1,151 @@ +# A hash index backend: what it would be worth, and who could use it + +Asked and answered on 2026-09-11. `perf-benchmarks/benchmarks/fx-index.rs` +measured what a hash-shaped `using` backend would buy. This is the other half: +which declarations we already have could take one. + +**The answer is none of them, today.** That is a useful result rather than a +disappointing one, because the reasons are structural and each of them names +the thing that would have to change first. + +## What it would be worth + +On the `vec: true` shape, a million rows, against the `ArcticIndex` it holds +today: + +| | gain | +|---|---:| +| build | **7.2x to 9.4x** | +| lookup | **3.9x to 7.7x** | +| delete | 17x to 21x (measured before ghosting; see below) | + +Reserving is most of the build win: an unreserved hash map is only 1.8x to 2.5x, +so `with_capacity` on the index is worth a further 3.1x to 5.1x. That matters +because it is the one place pre-allocation has any headroom at all — +`arctic-prealloc.rs` put the ceiling on pooling Arctic's node allocation at +**0.92x**, below one. + +The delete figure is stale in the useful direction: `vec: true` now ghosts a +delete and the 21-millisecond path it was measured against is gone. Do not quote +it. + +## The survey + +Every `worktable!` in the repositories that have real declarations. Counted +from the checkouts in `~/code`, not from memory. + +| repository | declarations | persisted | in-memory | `vec: true` | range sites | ordered scans | +|---|---:|---:|---:|---:|---:|---:| +| `web3.trading-backend` | 24 | 10 | 14 | **0** | 0 | 4 | +| `agencyzero` | 19 | 19 | 0 | **0** | 1 | 0 | +| `pays.online-backend` | 24 | 23 | 1 | **0** | 0 | 0 | +| `nofilter.io-backend` | 16 | 9 | 7 | **0** | 0 | 0 | +| `api.support.cafe` | 11 | 10 | 1 | **0** | 2 | 0 | +| `auth.honey.id-backend` | 14 | 9 | 5 | **0** | 0 | 0 | +| `api.honey.id-backend` | 8 | 7 | 1 | **0** | 0 | 0 | +| **total** | **116** | **87** | **29** | **0** | **3** | **4** | + +`wt-benchmarks` is excluded from the total: it has 41 invocations, it is a +measurement suite rather than an application, and 18 of the repository's range +call sites are in it. + +## Three filters, and what each one removes + +**1. It only fits `vec: true`, and nothing is `vec: true` yet.** `UniqueIndex` +requires `range_values` and `range_links`, which a hash map cannot answer at any +price, so a hash backend cannot be a fifth arm of the existing trait. The +generator for `vec: true` is the one that never calls them. Zero of the 116 +declarations use it, because it shipped in 1.9 and nothing has adopted it. + +**2. Persistence is a hard exclusion, and it removes 87 of 116.** Verified in +`codegen/src/persist_index/generator.rs`: `from_persisted` rebuilds each index +with `attach_node` / `attach_nodes` / `attach_multi_nodes` from B-tree nodes read +off disk. The on-disk form of an index *is* sorted pages. A hash map has no node +structure to attach and no page form to write, so `persist: true` and a hash +index cannot both be true without a second on-disk index format. + +**3. A shared table needs a lock, and the lock is the whole gain.** +`wt-vs-rustc-structures.rs` measured seven readers against writers: a +`RwLock` keeps **22%, 16% and 10%** of its read throughput at one, +two and four writers, where `ArcticIndex` keeps **97%, 88% and 66%** and +overtakes at two writers. Every one of the 29 in-memory declarations is held as +`Arc<...WorkTable>` and shared. + +That leaves the read-only case, where a locked hash map still wins 4.6x because +nothing ever takes the write side. It does not occur here either. Counting call +sites against `web3.trading-backend`'s seven S5 tables: + +| table | read sites | write sites | +|---|---:|---:| +| `signal_table` | 0 | 1 | +| `event_table` | 6 | 2 | +| `position_table` | 1 | 7 | +| `order_table` | 9 | 5 | +| `fill_table` | 0 | 1 | +| `key_table` | 7 | 5 | + +Every table is written. None is the build-once-read-forever shape. + +## What a candidate would look like + +So the filter, stated as something that can be checked against a declaration +rather than argued about: + +1. `vec: true`, or any table with a single writer — no `Arc` sharing with a + writer on the other end. +2. Not `persist: true`. +3. No `select_by_*_range`, no `order_on`, no ordered iteration. +4. More than 20 rows (below that a scan wins outright), and searched more than + about 150 times after each build (below that the build never earns itself + back). + +Points 3 and 4 are already measured; see `docs/small-tables.md`. + +## What changes the answer + +`vec: true` got ranges and ghosted deletes on 2026-09-11, which cuts both ways +and is worth stating plainly. + +It makes the shape **more** likely to be adopted, so candidates may appear where +there are none now: a `vec: true` table is now a credible replacement for an +in-memory paged table that was only paged because nothing else could range. + +It also makes a hash backend **less** attractive on that shape specifically. A +hash backend would have to give the range API back up, so `using fxhash` would +become a per-backend capability question — a table that declares a range cannot +take it — which is exactly what `using` is for, and exactly the kind of +conditional surface that needs sign-off before anything is built. + +--- + +## Postscript, same day: it was built + +The recommendation above was to leave it unbuilt, on the grounds that the survey +found no candidate. That was overruled, and correctly — the survey answers +"who would use it today", which is not the same question as "is it worth having", +and a backend that nothing uses yet is a different thing from one nothing *can* +use. + +`using fxhash` ships. It is accepted on `vec: true` and refused on a paged +table, with the refusal naming both reasons. Emitting `range` and `range_by_` +became conditional on the backend being ordered, which is the per-backend +capability shape this document proposed in its last section; a table using +`fxhash` has no range methods at all, so a caller who needs one gets a compile +error at their own call site. + +Measured through the real macro rather than the hand-written proxy +(`perf-benchmarks/benchmarks/fx-index.rs`), at a million rows against the +default arctic backend: **build 4.9x, lookup 4.0x**. + +One thing the survey did not predict and the wiring found. The generated arm +first measured only 2.4x on build against the hand-written ceiling's 8x, because +`with_capacity` sized the row vector and left the index alone — right for every +other backend and wrong for this one. `with_capacity` reserves an `fxhash` index +now, and that single change took it to 4.9x. **The pre-allocation lever this +whole line of work went looking for turned out to be here**, on the one backend +that had not existed when the question was asked. + +The three filters in this document are unchanged and still exclude every current +declaration. What changed is that the first of them — "it only fits `vec: true`, +and nothing is `vec: true` yet" — is now a statement about adoption rather than +about capability. diff --git a/docs/known-issues.md b/docs/known-issues.md index 92988423..67082858 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -3,14 +3,23 @@ Open defects and accepted limitations, recorded so the next audit starts here instead of rediscovering them. Source: the 2026-08-31 full audit (WorkTable core plus the WorkTablesIndex, congee-wt, and arctic-wt backends) and the fix pass that followed it in -1.0.0-beta.13. Every item below was deliberately deferred, with the mechanism written down; -items fixed in beta.13 are not listed. +1.0.0-beta.13. The 12 September 2026 review updates resolved entries below; +older backend findings retain their stated scope and are not all newly reproduced. Severity words: "corruption" means wrong or lost data, "outage" means a hang or abort, "perf" means measurable cost with no wrong answers. ## Persistence engine +- **Fixed in the 2026-09-11 release review: page capacity on reopen.** The metadata + reader and six generated metadata/index read paths passed the full stride as the + payload capacity. DataBucket layout validation rejected them before reading any + data. The readers now use the inner capacity, preserving the existing file format. + The previous claim that this affected only one call site was wrong: generated + reads must also be reviewed. `perf-benchmarks/wt-persistence` verifies every row + and secondary key after reopening at 8, 16 and 32 KiB strides. The existing + custom-page-size, index-reload and exact-boundary tests cover the same contracts. + - **Reclaim-barrier ordering inversion escalates to a spurious terminal failure.** While a `ReclaimPages` message is pending, the worker stops popping the queue, and reclaim only runs once the analyzer drains. An operation whose CDC event id precedes events already @@ -40,30 +49,31 @@ Severity words: "corruption" means wrong or lost data, "outage" means a hang or ## In-memory storage -- **Every mutation serializes on the table-global `page_access` write lock**, memcpy and - page bookkeeping included. This is the write-throughput ceiling on multicore; - per-page locking is the architectural fix. -- **Every read performs a SeqCst RMW on one shared `active_readers` line** (`read_guard`), - and the hot counters are adjacent with no padding (false sharing). A sharded or epoch - scheme is the fix. -- **Reclamation requires a global zero-reader instant.** Under sustained overlapping reads - the instant may never occur: retired links, pages, and publications accumulate without - bound and deletes stop reclaiming space. When reclamation does trip, the whole backlog - drains inline inside one arbitrary mutating call (millisecond-class latency spike; the - code warns at a backlog of 1024). Epoch-based reclamation is the fix for both halves. -- **The publication cache doubles the resident set**: every live row exists as archived - page bytes and as `Arc` plus lock plus map slot, and every mutation republish pays a - full-row deserialize. Design cost, paid per row. +- **Fixed: table-global page mutation and reader counters.** Pages now have their + own allocation barrier and exact-cell access guards. Reads pin ps-reclaim + epochs rather than incrementing one table-global reader counter; reclamation + no longer requires a simultaneous zero-reader instant. The old per-row + publication cache was removed. These historical findings do not describe + the current read path. +- **Fixed in the 2026-09-12 review: released collisions split cell locks.** + An empty earlier slot could be claimed for a row still locked in a later + slot. Registration now keeps each active key unique, with a conservative + displaced-entry counter for the common path. See [cell-lock-registry.md](cell-lock-registry.md) + for the failing interleaving, invariants and bounded concurrency models. - **`unsafe impl Sync for Data` is broader than its discipline**: safe `&self` methods - mutate the page `UnsafeCell` relying on callers holding `page_access`; `Arc` is - handed to safe code (vacuum), so the soundness boundary lives in convention, not types. + mutate the page `UnsafeCell` under external page/cell coordination; the low-level + API does not encode all of those ownership requirements in types. Generated + table paths and raw Data-page APIs must not be treated as identical safety surfaces. - **A panicking closure inside `with_mut_ref` leaves the archived page image half-mutated** - while the publication keeps the old row (guards do not poison): memory and disk diverge - silently until reload. Closures are generated code today; nothing enforces that. + and guards do not poison. The old publication cache no longer exists, but a + panicking callback can still leave a partial edit; a persisted call that unwinds + before enqueueing its data operation has no durability guarantee. - **`mark_page_full` can race a concurrent failing save's `free_offset` rollback**, leaving `free_offset` slightly below `DATA_LENGTH` on a non-current page. Capacity pessimism only; no double allocation. -- **`row_count` restarts at 0 on reload** (`DataPages::from_data`), upstream TODO. +- **Fixed for table reload: row count restoration.** Loaded-table hydration calls + `set_loaded_row_count` after validating live row links. The low-level + `DataPages::from_data` constructor alone does not infer row boundaries. ## On-disk space layer @@ -71,20 +81,15 @@ Full mechanisms and the pinned data_bucket item list live in [space-layer-known-issues.md](space-layer-known-issues.md); the summary: - **There is no fsync/ordering discipline anywhere except the ART checkpoint writer.** - Every acknowledgement ends at `File::flush()` (tokio buffer to page cache). On power - loss, any acknowledged write may vanish or reorder against any other; only the ART file - has checksums, so torn pages surface as rkyv panics or silently wrong links. This needs + Ordinary drain is not a power-loss commit or transaction boundary. On power + loss, writes may vanish or reorder. DataBucket v3 now validates checksums and + row directories; the old assertion that only ART has checksums is obsolete. This needs one durability design decision (write ordering plus sync points), not per-site patches. -- **data_bucket 0.5.2 (pinned) carries these classes, all fixed at the source in the - 0.5.3 release PR (pathscale/DataBucket#69)**: u32 offset wraps past 4 GiB in the - relative page seek and link bound checks, `update_key` size accounting, and unchecked - over-budget page persists. Once the pin moves to 0.5.3, WorkTable's TableOfContents - wrapper (src/persistence/space/index/table_of_contents.rs) should adopt the new - capacity-checked `try_insert`/`try_update_key` and typed overflow errors: its - size-change re-key workaround can then delegate, and the inherited oversized-entry - own-page fallback (which can still persist an over-budget segment; the last open item - in space-layer-known-issues.md) is closed by the checked insert. The small-DATA_LENGTH - test fixtures that rely on that fallback need regenerating at the same time. +- **The old DataBucket 0.5.2 pin is obsolete.** The release graph uses 0.7 with + checked page bounds and coordinated v3 integrity validation. The WorkTable TOC + wrapper still permits an oversized entry to occupy a segment in memory, but + DataBucket rejects an over-budget persist instead of overwriting the next page. + Early rejection or segment spilling remains a separate API improvement. - **Perf:** every structural index event rewrites every TOC segment (each re-serialized from a cloned BTreeMap); each sized single-event insert performs an on-disk free-slot scan (one read syscall per cell); ART compaction runs synchronously inside @@ -113,16 +118,16 @@ Full mechanisms and the pinned data_bucket item list live in ## Row locking -- **Lock identity is a wrapping u16 id.** Two distinct in-flight locks 65,536 ids apart - dedup in a predecessor `HashSet`, silently dropping a real predecessor. Astronomically - unlikely per row; structurally wrong. +- **Fixed in the 2026-09-12 review: wrapping labels lost predecessors.** Two + distinct live locks with the same u16 label collapsed in the dependency set. + Equality and hashing now use the existing shared flag allocation identity. + Labels remain diagnostic; no counter widening, allocation or API change is needed. - **`mutation_guard` is an unbounded spin** on an async worker thread; correctness depends - on the (honored, but unstated at call sites) invariant that no holder awaits. 64 stripes + on the (documented on the guard conversion and mutation APIs) invariant that no holder awaits. 64 stripes also collide unrelated keys into one FIFO. - **`Lock` waker lists grow per `wait()` call and are never pruned**; unlock wakes every historical waiter (thundering herd on hot rows). -- **`LockGuard::unlock` runs the unlock pair twice** (explicitly and again in Drop); - harmless only because unlock is idempotent. +- **Fixed: explicit guard unlock delegates to Drop.** Cleanup runs once. ## Generated code (accepted semantics and open items) @@ -142,9 +147,32 @@ Full mechanisms and the pinned data_bucket item list live in Beta.12 fixed the metrics scans and added the `partition_ref` borrow API. Still open: -- **`gc(&mut self)` is uncallable through the shared-`Arc` deployment shape**, so removed - partitions accumulate in the retire list for the process lifetime under key churn. - Epoch-based retirement is the fix; until then treat shared routers as append-only. +- **Fixed: shared routers can reclaim retired partitions.** Epoch retirement and + `collect(&self)` work through an Arc. The remaining inline collection cost is + described below; the old append-only restriction is obsolete. +- **`collect` runs inline and its batch is bounded by count, not by cost: a routing + call can pay 3.3 milliseconds.** (perf. Measured 2026-09-11, + `perf-benchmarks/benchmarks/partition-collect-inline.rs`.) `get_or_create` + (`src/partition/mod.rs:456`) and `remove` (`:519`) both call `collect`, which frees up to + `COLLECT_BATCH_LIMIT = 64` retired partitions on the calling thread. The code moves that + work off the growth *lock*, and its comment says so, but not off the *thread*. + + A quiet router never sees it: `remove` queues a clone, defers the grace marker, then + calls `collect`, and with no reader pinned the grace has already expired, so `collect` + drops the queue's reference while the caller still holds the one being returned. The + teardown lands where the caller drops their own handle. + + With a reader pinned across a run of removals — which `partition_ref` and `pinned` both + document as delaying reclamation — every marker is held back, `collect` claims nothing, + the callers drop their handles, and the queue is left holding the last reference to all + of them. `retired_len` then goes 256, 192, 128, 64, 0 across four consecutive routing + calls costing 3,340 / 3,879 / 3,268 / 4,379 microseconds against a 1.8 us median. + + Not a backend problem: a congee-indexed payload is worst at 3.3 ms but the cheapest + payload measured still reaches 2.9 ms, because sixty-four table teardowns is sixty-four + table teardowns. Two directions, neither chosen: bound the batch by elapsed time rather + than by count, or hand the drain to a background task and leave the routing path with + only the queue push. - **`make()` runs under the global growth mutex**: a slow initializer (or a stage-2 persisted load) stalls all creations and removals. (Initializer panics no longer poison the set: beta.12 moved the lock to parking_lot, which unwinds cleanly.) @@ -178,12 +206,11 @@ Beta.12 fixed the metrics scans and added the `partition_ref` borrow API. Still (Additional items fixed or re-documented by the beta.13-era WorkTablesIndex PR are listed in that repo; the following remain by design or await redesign.) -- **Iterator lifetimes are transmuted past the node guard**: collected `&T` borrows - (`iter().collect::>()`) dangle once the iterator advances or drops. Reachable - use-after-free from idiomatic code; needs an API change (owned yields or a lending - iterator). +- **Fixed: concurrent iterators yield owned batches.** The old guard-lifetime + transmute was removed. Point `Ref` values still hold a node read guard and + must be dropped before re-entering the map on the same thread. - **`len()`/`is_empty()`/`capacity()` lock every node**: calling them while holding a live - `Iter` or `Ref` on the same thread self-deadlocks; with `remove_range` in the mix a + point `Ref` on the same thread self-deadlocks; with `remove_range` in the mix a three-party variant hangs writers too. Also O(nodes) cost per call. - **`Operation::commit` is not unwind-safe**: a panic between the index entry removal and the reinsert of the halves silently unlinks a whole node (locks do not poison, and the diff --git a/docs/magic.md b/docs/magic.md new file mode 100644 index 00000000..c00ab8b5 --- /dev/null +++ b/docs/magic.md @@ -0,0 +1,630 @@ +# Historical design discussion + +This document retains the original runtime proposal and measurements. It is not the released API contract. In particular, runtime selection does not change table lock types, profiles must match the declared backend/flavor, and scheduled selects finish with execute_async().await. See [the canonical guide](wt-user-guide.typ) and [the implementation contract](query-runtime-release-gate.md). + +# `worktable!`: the whole DSL + +What the macro accepts today, why the runtime work exists at all, and the syntax +proposed for it. The two halves are separated: under **Today** it compiles now, +under **Proposed** it does not. + +--- + +# Why any of this exists + +## One engine, several concurrency points, opposite answers + +A WorkTable table is not one concurrent thing. It is five, and they want +different scheduling: + +| point | where | shape | +|---|---|---| +| row-lock handoff | `src/lock/` — a writer parks, the releaser wakes it | a chain: the successor wants the cache lines just touched | +| in-place mutation | internal locking, no caller-visible lock | different contention entirely | +| persistence flush | `persistence/task.rs` — one long-lived queue drainer per table | never usefully suspends, just needs a thread | +| vacuum sweep | `table/vacuum/manager.rs` — background, already paced | must **not** disturb the foreground | +| query fan-out | does not exist yet | independent chunks, embarrassingly parallel | + +A single table-wide setting cannot serve those. That is the whole motivation. + +## The trade is real, and it was measured rather than assumed + +Keeping a woken task on the worker that woke it is worth a lot to one workload +shape and costs a lot to another. Measured on YCSB at eight threads, same engine, +against a tokio-driven build: + +| workload | `locality` | `spread` | +|---|---|---| +| 50% read / 50% update | **+6.4%** | −40.2% | +| read-modify-write | **+2.0%** | −38.4% | +| 95% read / 5% update | −19.2% | **+74.4%** | +| 95% read / 5% insert | +137.1% | **+321.6%** | + +Four independent attempts were made to find a static rule that wins both columns: + +1. route only *self*-wakes locally → the update workload went to −41.8% +2. route locally only when no worker is idle → −49.5% +3. let a thief take the LIFO slot on second sight → +97.7% one column, −34.5% the other +4. push local wakes onto the stealable deque → +113.4% one column, −41.2% the other + +Four attempts, four times the same answer: one column or the other, never both. +**That is evidence there is no such rule**, and it is why this is a selection +mechanism rather than a better default. + +## Nobody else exposes it, and everyone has it + +tokio ships this exact switch as `disable_lifo_slot`: one boolean, no guidance on +when to flip it. The mechanism is not novel. What is novel here is that each +setting carries the measurement that produced it, so a schema author can choose on +evidence instead of folklore. + +## No scheduler wins everywhere + +Same engine, only the executor driving the clients changed: + +| benchmark | winner | +|---|---| +| orderbook-arrival, p99 at 50% load | nagoya, 6,042 ns against tokio's 60,875 | +| orderbook-burst, makespan | nagoya, 5.8 ms against tokio's 7.0 | +| timer latency, 1 ms sleep | nagoya, 270 µs p50 against tokio's 1,538 | +| YCSB pure read, 16 threads | tokio, 18,850,424 against nagoya's 14,329,421 | +| YCSB A / B / F at 4-6 threads | thread-per-client, no async scheduler at all | +| p50 at any load | thread-per-client | + +So the design goal is not "make nagoya win". It is "let the schema say which shape +this table's work has", and then be right for that shape. + +## Why the DSL and not a config file + +Because the choice **changes the generated type**. `runtime:` selects the +`RwLock`, `Notify` and `JoinHandle` that `LockMap` and `PersistenceTask` are built +from. That is the same reason `persist:` is in the macro rather than a setting: +it changes what is generated, not how it behaves at run time. + +It also keeps the decision next to the columns and queries it governs, where a +reader can see it. + +--- + +# Today + +## The smallest table + +```rust +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: Simple, + columns: { + id: u64 primary_key autoincrement, + value: String, + } +); +``` + +Generates `SimpleWorkTable`, `SimpleRow`, `SimplePrimaryKey` and the query methods +below. Every generated name derives from `name:`. + +## The grammar + +A **fixed, ordered prefix**, then a free-order section list. + +| position | key | meaning | +|---|---|---| +| 1, required | `name:` | table name, CamelCase | +| 2, optional | `version:` | schema version, for migration | +| 3, optional | `persist:` | `true` writes to disk | +| 4, optional | `partition_by:` | partition key name and unsigned type | +| 5, required with 4 | `partition_max_size:` | rows per partition, as an index width | +| any order | `columns:` | the row and its primary key | +| any order | `indexes:` | secondary indexes | +| any order | `queries:` | generated `update` / `delete` / `in_place` | +| any order | `config:` | `page_size`, `row_derives` | + +The prefix is genuinely ordered: `parse_name` reads the first token and errors if +it is not `name`, so nothing can precede it. + +## Everything at once + +```rust +worktable!( + name: Test, + persist: false, + columns: { + id: u64 primary_key autoincrement, + test: i64, + another: u64, + exchange: String + }, + indexes: { + test_idx: test unique, + exchnage_idx: exchange, + another_idx: another, + }, + queries: { + update: { + AnotherByExchange(another) by exchange, + AnotherByTest(another) by test, + AnotherById(another) by id, + }, + delete: { + ByAnother() by another, + ByExchange() by exchange, + ByTest() by test, + } + } +); +``` + +## Columns + +```rust +columns: { + id: u64 primary_key autoincrement, // the table generates the key + other: u128 primary_key, // the caller supplies it + name: String, + amount: u64, + price: f64, + flag: bool, +} +``` + +Exactly one column takes `primary_key`. `autoincrement` makes the table generate +it and adds `get_next_pk()`. + +## Index backends: `using` + +The mechanism the proposed `runtime:` selection copies, and the reason `runtime` +is a *separate* keyword rather than an overload of this one. + +```rust +columns: { + id: u64 primary_key using worktables_index, + other: u64, +}, +indexes: { + other_idx: other using congee, + name_idx: name unique using arctic, +} +``` + +| backend | notes | +|---|---| +| `arctic` | the default | +| `worktables_index` | the persisted page format earlier releases wrote | +| `congee` | requires explicit persistence | +| `indexset` | the upstream crate, behind the `vanilla-index` feature | + +Omitting `using` gives `arctic` for in-memory lookups while persisted tables keep +the `worktables_index` page format, so an existing file still opens. `unique` is +independent of the backend and combines with it. + +## Queries + +Three kinds. CamelCase in the declaration, snake_case in the generated method. + +```rust +queries: { + update: { + AmountById(amount) by id, + }, + delete: { + ByName() by name, + }, + in_place: { + SomeValueById(some_value) by id, + } +} +``` + +**`update`** generates `update_amount_by_id(AmountByIdQuery { amount }, id)`. The +query struct is the name plus `Query`. + +**`delete`** generates `delete_by_name(name)`. Empty parentheses because a delete +names no columns. + +**`in_place`** generates `update_some_value_by_id_in_place(id, |value| ...)`, which +mutates without selecting first. Its locking is internal, so it is safe from +several threads without the caller holding anything — which is also why it is a +*different concurrency point* from `update`. **Only `by {pk_field}` is +supported.** + +## Selects, which are not declared + +Generated from the columns and indexes: + +```rust +table.select(pk) // by primary key +table.select_by_name("abc".to_string()) // by an indexed column +table.select_by_pk_range(start..=end).execute()? // a range +table.select_all().execute()? +table.select_all() + .order_on(TestRowFields::Test, Order::Desc) + .limit(10) + .execute()? +``` + +Note which of these return a **builder** — `select_all`, `select_by_pk_range` — +and which return a row directly. That distinction is load-bearing for the proposed +`.runtime()`, which can only exist on a builder. + +## Persistence + +```rust +worktable!( + name: Orders, + persist: true, + columns: { id: u64 primary_key autoincrement, symbol: String }, +); +``` + +Adds `load`, `wait_for_ops`, `close` and the persistence engine; files are opened +through `worktable::fsx`. + +## Partitioning + +```rust +worktable!( + name: Price, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { + exchange_id: u8 primary_key, + bid: f64, + ask: f64 + } +); +``` + +`partition_by: : `, and `partition_max_size: ` beside +it, which is required. It composes with everything else: indexes, queries and +config are untouched by it. + +The size is a type rather than a count because it is an index width. `bool` is 2 +rows, `u8` is 256, `u16` is 65,536, and `u32` or `u64` mean unbounded in practice +and generate a full table per partition. There is no `unbounded` keyword: the +widths run out of smallness, so `u64` is the escape. + +A narrow width generates `DenseTable` as the partition payload: the primary +key *is* the row's position, so there is no primary index, no pages and no lock +map, and a lookup is a bounds check and a load. An empty dense partition costs +108 bytes against a full one's 28,404, which is the whole point of the key. + +The width is a bound and not a reservation: the row vector grows to the highest +key used, so a `u16` partition holding three rows holds three slots. + +## Versions and migration + +```rust +mod v1 { + worktable!( + name: User, + version: 1, + persist: true, + columns: { id: u64 primary_key autoincrement, name: String }, + ); +} + +mod v2 { + worktable!( + name: User, + version: 2, + persist: true, + columns: { id: u64 primary_key autoincrement, name: String, email: String }, + ); +} +``` + +`worktable_version!` declares a read-only view of an older layout, for opening +data written by a previous schema: + +```rust +worktable_version!( + name: UserV1, + columns: { + id: u64 primary_key autoincrement, + name: String, + email: String, + }, + indexes: { name_idx: name }, +); +``` + +## Config + +```rust +config: { + page_size: 16384, + row_derives: Clone, Debug, +} +``` + +Tuning **values** live here. Anything that changes the generated *type* goes in +the prefix instead — which is the rule that puts `runtime:` beside `persist:`. + +# Proposed, not implemented + +Nothing below compiles yet. It is here for review before it is built. + +The design follows `using ` as a *mechanism* — an enum in the DSL, a +codegen mapping to a concrete type, a trait the types satisfy — but deliberately +**does not reuse the `using` keyword**. `using` means index backend and only that. +Runtime selection uses `runtime`. + +## The mapping is concurrency points, not tables + +A table is the wrong unit, and so is a single query. A table contains several +concurrency points and they want opposite things: + +| point | where | shape | measured | +|---|---|---|---| +| row-lock handoff | `src/lock/`, a writer parks and the releaser wakes it | a chain; the successor wants the lines just touched | `locality`: YCSB A +6.4%, F +2.0% vs tokio | +| in-place mutation | internal locking, no caller-visible lock | different contention entirely | not separately measured | +| persistence flush | `persistence/task.rs`, one long-lived queue drainer | never usefully suspends, just needs a thread | pool choice barely matters | +| vacuum sweep | `table/vacuum/manager.rs`, already paced | must not disturb the foreground | wants a budget, not a flavor | +| query fan-out | does not exist yet | independent chunks | 1.99x-2.84x on orderbook upsert/delete | + +The `queries:` sections already group by concurrency point: everything in +`update:` goes through the same lock path, and `in_place:` is a different path by +design. So the annotation belongs on the section. + +## Three positions, one keyword + +| position | scope | cost | +|---|---|---| +| `runtime: nagoya(spread)` at the top level | the table's sync types and default pool | changes the generated type | +| `update runtime fast_local:` on a section | that concurrency point | compile time, free | +| `.runtime(wide)` on a builder | one call | runtime, opt-in | + +## Named profiles + +```rust +runtimes! { + tokio_max: tokio, + fast_local: nagoya(locality), + wide: nagoya(spread), +} +``` + +## The whole thing together + +```rust +worktable!( + name: Orders, + persist: true, + runtime: nagoya, // table default = nagoya(locality) + columns: { + id: u64 primary_key autoincrement using arctic, // `using` = index backend + symbol: String, + qty: u64, + }, + indexes: { + symbol_idx: symbol using congee, + }, + queries: { + update runtime fast_local: { // `runtime` = scheduler + Fill(qty) by id, + Cancel(qty) by symbol, + }, + in_place runtime fast_local: { + Bump(qty) by id, + }, + delete runtime wide: { + BySymbol() by symbol, + }, + } +); +``` + +Omitting `runtime` anywhere falls back to the table default, and omitting the +table default gives `nagoya(locality)`. + +## Call site + +```rust +// point read: unchanged, no hop, no way to get it wrong +let row = table.select(pk); + +// long scan: the builder already exists, `.runtime()` is one more link +let rows = table.select_all() + .order_on(OrdersRowFields::Symbol, Order::Desc) + .limit(10_000) + .runtime(wide) + .execute()?; +``` + +`.runtime()` lives only on the builder-returning selects, so it cannot be attached +to a point read. That is deliberate: **dispatching costs 21 ns to spawn +(`null-submit-cost`) plus ~2,250 ns to wake (`null-wake-latency`), against a +~400 ns point read.** The hop is larger than the operation. + +| query | cost | hop as a share | verdict | +|---|---|---|---| +| point read | 400 ns | 560% | never | +| single update | 1.6 us | 140% | never | +| 16k-row scan | 6-17 ms | 0.02% | worth it | + +## Why `runtime:` is top level and not in `config:` + +| goes | what | +|---|---| +| top level | anything that changes the **generated type** | +| `config:` | tuning **values** | + +`persist:` is top level because it changes the type. `page_size` is a number, so +it is in `config:`. `runtime:` selects the `RwLock`, `Notify` and `JoinHandle` +that `LockMap` and `PersistenceTask` are built from, so it goes beside `persist:`. +It also keeps the setting next to the columns and queries it governs. + +PR #58 moved `columnar_slot_id` and `columnar_chunk_rows` into `config:`, which +was right for those: they are tuning values. + +## The flavors, and what each measured + +| flavor | for | vs tokio, YCSB at 8 threads | +|---|---|---| +| `locality` | wakes that are a chain | 50% update **+6.4%**, read-modify-write **+2.0%** | +| `spread` | wakes that are independent | 95% read / 5% update **+74.4%**, 95% read / 5% insert **+321.6%** | +| `throughput` | a firehose from outside the pool | the defaults before local wakes existed | + +No setting wins both columns; four attempts to find one each reproduced a single +column exactly. `locality` is the default because 19% behind on one shape beats +40% behind on two. + +## Two axes, not one + +Research backends are **queue algorithms**, not runtimes. They swap st3's deque +and keep the facade above it. + +| axis | what changes | cost | examples | +|---|---|---|---| +| runtime | sync types, spawn, timers, io | the `Runtime` trait, ~40 signatures | nagoya, tokio, smol | +| queue | the work-stealing algorithm only | one `Pool` impl | st3, BWoS, Chase-Lev | + +Undecided: whether a queue choice is `nagoya(spread, bwos)` or a separate key. + +## Parse-time rules + +Taken from PR #58's review, which rejected inert declarations rather than +accepting them: + +- a profile naming `tokio` cannot be referenced from a table whose `runtime:` is + `nagoya`; the sync types are already fixed +- unimplemented backends must **fail to compile**, not be accepted and ignored +- `runtime` cannot appear before `name:` — the parser's fixed prefix is `name`, + `version`, `persist`, `partition_by`, and everything after is free-order + +## Open question the syntax does not settle + +Whether a section annotation should select a **pool** or a **retry policy**. + +Only the write path spins: the generated update code has a retry loop calling +`yield_now` with exponential backoff, and `yield_now` self-wakes, which is exactly +what `locality` optimises. The read path has no such loop. So "writes want +locality" may be about that loop rather than about pools, in which case the knob +is the backoff curve, which is inline and free. + +The experiment: set `local_wakes: false` while changing the generated retry loop +from `yield_now` to `sleep`, and see whether the update workload's advantage +survives. Not yet run. + +## Precedence: defining both is an error + +| defined | result | +|---|---| +| section **and** call site | **compile error** | +| section only | the section's profile | +| call site only | the call's profile | +| neither | the table's `runtime:`, else `nagoya(locality)` | + +Decided: an error rather than a silent override, so there is one answer to "which +runtime does this query use" and it is visible at the place you are reading. + +**The cost of that choice**, recorded so it is not a surprise: adding a section +annotation becomes a breaking change for callers already using `.runtime()`. The +window is narrow today — `.runtime()` exists only on select builders, and selects +are not declared in `queries:`, so `update` / `delete` / `in_place` annotations +can never collide with it. It only bites if a `select` section annotation is added +later. + +**Implement the error deliberately.** Omitting `.runtime()` from the generated +builder when the section pins one gives *"no method named `runtime` found for +struct `SelectQueryBuilder`"*, which points at the wrong thing. Generate the +method and make it unsatisfiable so the message can say what happened: + +```rust +#[diagnostic::on_unimplemented( + message = "`{Self}` already has a runtime pinned by the schema", + label = "remove this `.runtime()`, or remove `runtime` from the `select` section", +)] +``` + +## The one error that is unconditional + +A call site may only name a profile whose **backend matches the table's**. The +table-level `runtime:` selects types — the `RwLock`, `Notify` and `JoinHandle` +that `LockMap` and `PersistenceTask` are built from — so nothing downstream can +change it. + +```rust +// table is `runtime: nagoya` +table.select_all().runtime(wide).execute()?; // ok, wide is nagoya(spread) +table.select_all().runtime(tokio_max).execute()?; // compile error, always +``` + +This is why `runtimes!` must give each profile a **backend marker type** rather +than a bare name: `.runtime()` takes `P: Profile`, so the +mismatch surfaces as a trait bound naming both backends. Retrofitting that later +is much more expensive than honouring it in the profile macro now. + +**So: the flavor is selectable at the section or the call site, never both. The +backend is fixed at the table and only ever checked downstream.** + +## No parameters, for now + +`runtimes!` takes a backend and a flavor, nothing else. `.runtime()` takes a +profile name, nothing else. No worker counts, no tuning values, no +`.runtime(wide, 12, 100)`. + +Decided. The reasons, so the decision can be revisited on evidence rather than +taste: + +**Every distinct parameterisation is a distinct thread pool.** A profile is not a +value passed to an existing pool, it selects one. Free-form numbers at call sites +mean unbounded pool creation, and nobody reading the call site can see that. With +names only, every pool the process will ever create can be enumerated by reading +one macro. + +**The knobs are counts, not durations.** Anything of the form "100ms stealing" +does not map onto this pool. `promote_every`'s own doc says it is "a count rather +than a clock because the pool has no clock it is willing to read on the hot path". +The real surface: + +| knob | unit | default | where | +|---|---|---|---| +| workers | count | `available_parallelism` | `Pool::new` | +| `rounds_before_park` | empty rounds | 64 | `Tuning` | +| `backoff_spins` | spin-loop hints | 1024 | `Tuning` | +| `injector_batch` | jobs | 1 | `Tuning` | +| `promote_every` | jobs between heartbeats | 64 | `Tuning` | +| `local_wakes` | bool | true | `Tuning` | + +**Positional numbers are unreadable**, and the DSL already settled on the named +form elsewhere: `columnar(chunk_rows(32_768), compression(none))`. + +If parameters are wanted later, the shape that keeps the pool set finite is a +named profile carrying them, not a call-site tuple: + +```rust +runtimes! { + wide: nagoya(spread), + wide_12: nagoya(spread) { workers: 12, backoff_spins: 4096 }, +} +``` + +That also needs a pool cache keyed by `(backend, workers, tuning)`, which does not +exist today: `nagoya::runtime::background()` is a single process-wide pool. + +### Keeping parameters cheap to add later + +Two things keep the door open, and both cost nothing now. + +**Chain, do not widen.** When parameters arrive they go on as further builder +links, not as extra arguments: + +```rust +.runtime(wide).workers(12) // additive, existing calls unaffected +.runtime(wide, 12) // arity change, breaks every existing call +``` + +Same reason `.limit()` and `.order_on()` are separate links. + +**The API is the free part; the pool registry is not.** A parameterised profile +must resolve to a *cached* pool keyed by `(backend, workers, tuning)`. Today +`nagoya::runtime::background()` is a single process-wide pool with no such lookup, +so that registry is the real work — roughly a `OnceLock` and the logic to +start a pool on first use. + +Build the profile as a struct with room to grow rather than a bare enum, so adding +fields later is a struct change and not a signature change. diff --git a/docs/no-std-validation.md b/docs/no-std-validation.md new file mode 100644 index 00000000..ef600419 --- /dev/null +++ b/docs/no-std-validation.md @@ -0,0 +1,3 @@ +# no_std validation + +The canonical feature contract and verification commands are in the “Building without default features” section of the [Typst user guide](wt-user-guide.typ). diff --git a/docs/on-disk-v3-cutover.md b/docs/on-disk-v3-cutover.md new file mode 100644 index 00000000..8eb556df --- /dev/null +++ b/docs/on-disk-v3-cutover.md @@ -0,0 +1,78 @@ +# WorkTable and DataBucket v3 cutover + +Release decision, 2026-09-11: ordinary persisted WorkTable stores will move +from page format v2 to v3. WorkTable and DataBucket must implement and release +that boundary together. This is independent of the table's `version:` schema +number and of either crate's package version. + +## Motivation + +A v2 data page records a high-water mark and row bytes. It does not record +where each row starts or ends. The primary index supplies those locations. +The schema in `SpaceInfoPage` explains how to decode a row, but cannot locate +rows if that index is unavailable or its representation changes. The free +range list is lossy and cannot substitute for a directory of live rows. + +V3 needs a page-local row directory, with integrity validation covering both +the directory and row bytes. This makes data pages independently readable and +provides a basis for scans, index rebuilding, export and future migrations. +The directory changes the byte layout and usable page capacity. DataBucket's +codec and WorkTable's allocation, mutation, vacuum and persistence paths must +agree on it. + +## Rollout policy + +For almost all deployments, the planned migration is an explicit drop of the +old store followed by recreation or regeneration. There is no requirement for +a general v2 reader in the new runtime. An application that must retain data +needs an explicit source-to-target conversion tool; its old reader can remain +isolated from the production runtime. + +Opening incompatible data must report a clear version error. An application +must not silently reinterpret, overwrite or automatically delete an old store. +A rollback to the old binary cannot use the new store. + +## Binary layout and implementation + +Ordinary data pages now use format 3. For a page stride P, the general +header remains bytes 0..28. Row offsets are relative to byte 28. The +directory ends at P-8 and contains little-endian pairs of u32 offset and +u32 length, one per live row. Entries are ordered by offset. The CRC-32 +occupies P-8..P-4; the live-row count occupies P-4..P. Row bytes grow +forward; the directory occupies the tail. The checksum covers the entire +payload, including unused bytes, directory and count, excluding only its +own four-byte word. Header fields are validated separately. + +WorkTable reserves room for the worst-case slot count using the minimum +archived row-wrapper size. This reduces the row allocator capacity. Index +and metadata pages keep their full payload budget, P-28. Updating, deleting, +relocating and reclaiming rows maintains the directory. Clearing a reclaimed +page persists an empty directory before advertising it as reusable. Reload +restores free-range ownership so append allocation cannot overlap it. + +The separate Vec snapshot codec also uses version 3, but has a different +payload: an archived Vec, a count at P-12, a row-type fingerprint at P-8 +and a checksum at P-4. It +starts with an archived-rows page (type 4), a zero space id and a row-type +fingerprint in its 12-byte trailer; an ordinary WorkTable +space starts with a SpaceInfo page and schema metadata. These files are not +interchangeable. Page version alone does not identify the container. + +DataBucket tools can create a sample v3 file and enumerate live row extents +without opening an index. The slotted-page tests independently decode rows +after inserts, deletes and relocation, and reject a preserved real v2 +fixture without changing its bytes. Full release validation remains required +before readiness is declared. + +## Required verification + +Exercise multi-page inserts, variable-length updates, deletes, free-space +reuse, vacuum and reopen using the new layout. Read live rows from data pages +without consulting indexes, then verify rebuilt primary and secondary indexes. +Check directory capacity, row boundaries, page identity, checksums and explicit +refusal of v2 and unknown versions. Use actual old-format bytes for the refusal +test rather than having the new writer synthesize its own supposed old store. + +Rerun persistence and mutation performance measurements after the integrated +change, including page sizes and batching. Measurements of the current v2 +implementation cannot establish the cost of the v3 directory. diff --git a/docs/page-size.md b/docs/page-size.md new file mode 100644 index 00000000..43ffe6aa --- /dev/null +++ b/docs/page-size.md @@ -0,0 +1,55 @@ +# Page sizes in the v3 format + +Three quantities must remain separate: + +| Quantity | Meaning | +|---|---| +| Page stride | Physical bytes per page, including the 28-byte general header. Every page seek uses this value. | +| Payload capacity | Stride minus the header. Index and metadata pages retain this entire budget. | +| Row capacity | For persisted data pages, payload capacity less the worst-case live-row directory reservation and its eight-byte trailer. | + +A persisted table named SomeTable emits SOME_TABLE_PAGE_SIZE and +SOME_TABLE_INNER_SIZE. The latter is calculated by data_page_row_capacity +using the minimum archived wrapped-row size. It is the in-memory row allocator +budget as well as the persisted data-row budget. It must not be reused as an +index node size or index-page payload capacity. + +The directory contains eight bytes per live row. Its fixed trailer contains +CRC-32 at page offset P-8 and the count at P-4. Reserving the worst case before +allocation prevents an insertion from publishing an index and subsequently +discovering that its directory entry does not fit. Variable archives can be +larger than their minimum size and therefore need no more directory entries +than this reservation permits. + +## Implementation boundaries + +- DataBucket page_start_offset and seek helpers take STRIDE explicitly. Persist + helpers check encoded payloads against STRIDE minus GENERAL_HEADER_SIZE. +- DataBucket data-page readers take both a row-buffer bound and physical stride. + They read the entire physical payload to validate the directory and checksum. +- WorkTable SpaceData uses its row bound for DataPage and the full payload bound + for SpaceInfo. A restored free range advances the append cursor so the append + allocator cannot overlap memory owned by the restored free list. +- WorktableNameGenerator::get_disk_page_capacity supplies the full payload + budget to primary and secondary index nodes, persisted index pages, and table + of contents readers. OffsetEqLink still carries the row bound. +- Generated persistence readers and writers pass the same table stride through + data, index and logical-index wrappers. There is no implicit stride default + on WorkTable persistence wrappers. + +## Validation and allowed sizes + +The default stride is 16,384 bytes. Persisted tables accept configured sizes +of at least 512 bytes. Arctic-backed tables reject sizes above 65,535 because +its packed links have 16-bit offset and length fields. These are validation +rules for the existing page_size option; the v3 implementation adds no grammar. + +Custom-page tests assert physical file lengths and reopen every row. String +and UUID index tests exercise the full index budget independently of row +capacity. The slotted-page tests enumerate rows without consulting an index, +including after deletion and variable-length relocation. Vacuum tests cover +reopen followed by reuse of a reclaimed page. + +Page size is part of the store layout. Changing it requires an explicit data +cutover; reopening an existing file with a different size is not a migration. +See [the v3 cutover](on-disk-v3-cutover.md). diff --git a/docs/paper-2-plan.md b/docs/paper-2-plan.md new file mode 100644 index 00000000..89f97794 --- /dev/null +++ b/docs/paper-2-plan.md @@ -0,0 +1,159 @@ +# Paper two: plan and evidence map + +Written 2026-09-06 against master `d5656e6` (1.0.0-beta.19). Companion to the CIDR 2027 +submission (beta.6, submitted for the 2026-08-04 deadline; notification 2026-10-06). + +## Thesis candidates + +The CIDR paper argued *compile-time engine specialization*. It deferred (§5, §7): the lock +discipline scaling comparison, crash consistency, formal checking of the protocols, the cost +of monomorphization, and external baselines beyond redb/LMDB. Paper two should be the paper +those deferrals point at, not a re-statement of the thesis. + +**A. Lifecycle paper (recommended).** "Row lifecycle in a specialized engine: exact-cell +locking, epoch reclamation, and reactive vacuum without a transaction manager." Everything +in it has landed since beta.6 and has numbers. Contributions: + +1. Exact-cell synchronization replacing hashed stripes and the table-global barrier + (`src/in_memory/data.rs` `CellLocks`; `docs/versioned-row-publication.md`). +2. Quiescent-state reclamation via a one-word `!Send` guard (`ps-reclaim`), replacing a + global reader counter; the `Send`-guard use-after-free found on the way is a good + cautionary section (`docs/TODO.md` "ps-reclaim 0.1.1"). +3. Reactive vacuum planned from the free-range registry, gated by a mutation lease and a + quiet epoch; root-cause note that all four reclaim bugs came from indexes storing + physical addresses (`docs/vacuum-design-directions.md`, `src/table/vacuum/`). +4. The 1-32 thread grid across three index backends: the lock-discipline scaling result the + CIDR paper promised (`docs/beta18-validation.md`). +5. `partition_by` with the loom model of the slot protocol (`src/partition/loom_tests.rs`), + the first model-checked component; HFT review that produced `partition_ref` + (`wt-review.md`). + +**B. Persistence paper.** "Index persistence by replaying the tree's own CDC stream, and what +it costs." Needs the durability work first: structural CDC is ~28% of a 126 ns insert +(`docs/wti-dirty-generation-persistence-plan.md`); ART logical WAL exists +(`src/persistence/space/art_index.rs`) but data pages and WTI still end at `flush()`; the +watermark/fsync design is proposal only (`docs/durability-visibility-proposal.md`). Not +ready before a Q1 2027 deadline unless the journal lands. + +**C. Schema-as-IR paper (PL venue).** `worktable_dsl`: schema read as data, diff with a +per-change cost model, declarations baked into generated code, `wt-dsl` CLI, migration +engine with on-disk version detection (`dsl/`, `docs/migration.md`). Fits PEPM / OOPSLA +better than a DB venue; overlaps with the PEPM plan in the other session. + +## Evidence already in hand (all M4 Max, local trees; needs a pinned Linux rerun) + +| Claim | Number | Source | +|---|---|---| +| Hot-page writer, exact-cell vs hashed | 322 ns vs 1,536 ns | beta18-validation | +| Per-page vs table-global barrier | +25% @4, +65% @8 disjoint writers | versioned-row-publication | +| Generation retire, beta.18 vs 15 | 33.6x (WTI), 16.1x (Arctic) | beta18-validation | +| Read scaling best/1T at 16 threads | 3.6x / 3.3x / 2.6x per backend | beta18-validation | +| 10%-write mix ceiling | peaks at 4 threads | beta18-validation (also beta.13/15) | +| Reactive vacuum foreground penalty | -1.1..-5.1% vs 16-41% unpaced | beta17/18-validation | +| Vacuum reclamation | 18/18 cells, 196 pages, 100% | beta17-validation | +| Point lookup vs stripped index | 15.65 ns vs 9.25 ns bare Arctic vs 33.5 Vec+BTreeMap | beta18-validation | +| Memory overhead | 0.14 B/row over control | beta18-validation | +| Partition route | 0.73 ns Vec vs 9.5 ns string hash; `partition_ref` 3.35 ns | TODO.md, partition docs | +| Persisted insert, beta.15 to 18 | -33..-42% (Arctic 6,130 to 3,753 ns/row) | beta18-validation | +| CDC share of insert | ~35 ns of 126 ns | wti-dirty-generation plan | + +Not in hand: `paper-bench/results/` (never committed), `compile_cost.sh` never run, no +sled/SQLite/DashMap baselines, no beta.19 rerun of Table 2. + +## Consumer workloads (surveyed 2026-09-06, all under ~/code) + +| Repo | Shape | What it gives the paper | Open? | +|---|---|---|---| +| `agentcode` | 11 tables, 8 persisted, Arctic on nearly every index, u128 keys. Stress = 8 concurrent 800-file `update_latency` processes. | The concurrency bug that motivates the paper: `docs/known-defects.md:70-137`, torn/corrupt page header in secondary-index batch apply at beta.11, 6/8 runs failing, 28/28 after moving to Arctic. Map it to the beta.18 fix ("torn reads and premature physical-link reuse") and show the same harness clean on beta.19. Also: Arctic vs WTI at 20k rows, insert 2.18M/s vs 0.89M/s, lookup 17.0M/s vs 5.0M/s (`docs/benchmarks/index-backends.md`); state 42.4 to 22.2 MB after u128 keys (`state-growth.md`); the request for a non-unique fixed-width index that became `ArcticMultiIndex`. | Proprietary | +| `agencyzero` | 19 persisted tables, String PKs, migration engine, `LoadMode::Recovery`, single-writer flock, QA fixture of 248 projects (~30 MB store). | The reclamation case study: `docs/store-recovery.md` records four production corruptions, including a variable-width index page that forgot fragmentation across restart (174 live entries, 2,664 B dead, 64 B tail overlap) and beta.5 whole-row rebuilds churning `pr_project_idx` (38 disagreeing rows) fixed by in-place updates. Production migrations via schema fingerprint. This is the "why indexes must not store physical addresses" story with real data. | Private (GitHub) | +| `karen` | 2 persisted tables, tiny (~700 rows). | Row-level, queryable, durable learned session state instead of one blob: the per-turn Confirm write-through gives 15-20 points top-1. One paragraph of motivation, not evaluation. Uses `unload_gracefully`. | Closed | +| `ekopathrs` | No `worktable!` at all. Uses `worktable-vec::AtomicKeyTable` for two in-memory profiling tables. | The honest negative: `docs/STORAGE-REVIEW.md` rejects full WorkTable (318-package resolve, no `no_std`) for 399 entries. Cite as the boundary of the design space; `worktable-vec` is the lock-free, `no_std` sibling. | Private | + +Use agentcode as the headline stress workload in §4 alongside the shadow-state harness; use +agencyzero as the recovery/fragmentation case study; mention karen and ekopathrs in one +paragraph each in the experience section. Get written OK before naming private repos. + +## Gaps to close for option A + +- Rerun the beta.18 grid and `paper-bench` on a quiet pinned x86 box; commit `results/`. +- Lock-discipline ablation as a proper figure: field vs row vs table lock, 1-32 threads, + skewed keys (`paper-bench/src/bin/contention`). +- Semi-formal statement of the cell-lock + reclamation invariants; ideally extend loom + beyond partitions to the cell/retire path (the CIDR reviewers will ask). +- Wart sweep: 9 `todo!()` sites remain (`codegen/.../queries/in_place.rs`, `update.rs`, + `src/features/s3_support.rs`), `Avaiable` typo in 2 files. +- Merge or explicitly exclude `feat/columnar-fields-indexes` (branch, Aug 6, unmerged). + +## Target: EDBT 2027, 3rd cycle (verified 2026-09-06) + +- Submission **2026-10-07, 5pm PST** (31 days out). Author feedback 11-19, notification + Acc/Rej/Revise 12-05, revised paper 2027-01-04, final 01-27, camera-ready 02-10. + Conference Lille, April 6-9, 2027. +- Paper types: Research long (12p) or short (6p, title prefixed "[Short Paper]"), + Experiments & Analysis, Vision (6p). Topics list includes "Concurrency control, recovery, + and transaction management", "Storage, indexing, and physical database design", + "Data management on modern hardware", "Benchmarking and performance evaluation". +- The revise cycle matters: a paper that gets "revise" on Dec 5 has until Jan 4 to add + the pinned-Linux rerun, so the Oct 7 draft can ship on the M4 grid with the caveat stated. +- CIDR notification is Oct 6, one day before: paper two cannot depend on the outcome and + must not overlap the CIDR text (still under review until then). Option A is disjoint by + construction; cite the CIDR paper as "under submission". + +Alternatives if A slips: ICDE 2027 R2 (2026-11-11), PVLDB rolling (monthly to 2027-03-01), +SIGMOD R4 (2026-10-17). DaMoN 2027 CFP not posted. + +## Update 2026-09-11: release evidence and paper scope + +The release checkout is WorkTable 1.9.0-alpha1. Its runtime and row-lock +implementation differs from the earlier paper: inspect `src/lock/map.rs`, +`src/runtime/` and the selected dependency graph at the pinned evaluation +commit before describing them. + +The physical-design paper remains a useful candidate: paged memory and +persistence, dense bounded partitions, Vec storage, ordered versus hash +indexes, and columnar replicas expose different access and lifecycle costs. +The lifecycle paper remains another candidate, requiring renewed contention +and application-level evaluation on this engine. + +The earlier local draft collected useful hypotheses but its numerical table +is not publication-ready evidence. In particular: + +- The original four-way search matrix compiled the default strategy in every + arm through dependency feature unification. Its claimed 5% spread cannot + compare four strategies. +- 20,000 rows do not fit in a 16,384-row columnar chunk. Chunk and slot-width + claims must carry the actual row count and number of chunks. +- Vec ghost deletion, row-value destruction, compaction and whole-table drop + are different operations. A delete/shift ratio does not establish savings + over dropping a generation. +- The dirty-bit checkpoint experiment is a representation experiment, not a + shipped Vec persistence API. +- A benchmark report containing failed reopening tests cannot establish a + complete release pass. The reopened page payload handling has been fixed + and is being checked against multiple page strides and index paths. +- Runtime throughput needs CPU and tail latency beside it; a setting that + spins more is not universally faster or more efficient. + +The release evidence lives in the sibling `perf-benchmarks` repository: +`docs/claim-audit.md`, `docs/performance-feature-audit.md`, the dated reports, +and the benchmark source. Use measured operation definitions and exact +dependency revisions from there. Do not copy the old draft's unlogged ratios, +crate counts or benchmark counts into the paper. + +The release review does not validate `wt-benchmarks`, establish Linux results, +add missing sled/redb/SQLite/DashMap comparisons, or select a publication +venue. Those remain paper work. The physical-design proposal should be +evaluated against the corrected data before choosing between it and the +lifecycle proposal. + +## 31-day schedule for option A (long paper) + +| Week | Dates | Deliverable | +|---|---|---| +| 1 | Sep 7-13 | Freeze the claim list. Run `paper-bench` contention (field/row/table/inplace, 1-32 tasks) and beta.18 grid on the pinned Linux box if available, else M4 with three rotated passes; commit `results/`. Decide long vs short by Sep 13 based on whether the scaling figure holds. | +| 2 | Sep 14-20 | Draft §2 protocols (cell lock, retire, vacuum lease) with invariants stated; §3 partition + loom. Wart sweep PR (`todo!()`, `Avaiable`). | +| 3 | Sep 21-27 | Draft §4 evaluation from `results/`; figures; related work (Hekaton, epoch/QSBR: Fraser, Hart et al., Bw-tree, OLC, DaMoN vacuum/compaction lineage). | +| 4 | Sep 28-Oct 4 | Full read-through, internal review, page trim to 12. | +| 5 | Oct 5-7 | Buffer. Submit by Oct 6 evening local time (Oct 7 5pm PST is 07:00 Oct 8 in Bangkok, but do not use it). | + +Short-paper fallback (6p): contributions 1-3 only, one scaling figure, one vacuum figure. diff --git a/docs/partition-by-one-pager.md b/docs/partition-by-one-pager.md index 7559b39a..bbf73d25 100644 --- a/docs/partition-by-one-pager.md +++ b/docs/partition-by-one-pager.md @@ -56,6 +56,7 @@ worktable!( worktable!( name: OrderBook, partition_by: symbol_id: u16, + partition_max_size: u64, partitions: 1024, columns: { exchange_id: u8 primary_key, bid: f64, ask: f64, ts: i64 } ); @@ -309,6 +310,7 @@ worktable!( name: SymbolPosting, persist: true, partition_by: file_revision: u64, // Mode B, derived from the BLAKE3 revision + partition_max_size: u64, columns: { id: u64 primary_key autoincrement using arctic, posting_hash: u128, diff --git a/docs/partition-models.md b/docs/partition-models.md new file mode 100644 index 00000000..55a578ee --- /dev/null +++ b/docs/partition-models.md @@ -0,0 +1,209 @@ +# What a partition is, in WorkTable and in six other systems + +WorkTable's partitioning is Postgres-shaped: a partition is a complete table +with its own storage, its own index and its own locks. That is a real choice +with a real cost, not an implementation detail, and this page exists so the +choice can be read next to the alternatives. + +## Read this before the table + +**The seven systems below do not mean the same thing by "partition."** A single +ranked cost column would compare incomparable units and imply an equivalence +that is not there: + +| system | what the word names | +|---|---| +| PostgreSQL | a table | +| Kafka | a unit of ordering and parallelism | +| ClickHouse | a logical group of physical *parts* | +| Cassandra | one partition key's rows; you expect billions of them | +| HBase | a shard, called a region | +| Snowflake | a storage block, created for you | +| **WorkTable** | **a complete generated table** | + +A Cassandra partition and a Postgres partition are four orders of magnitude +apart in expected count. Comparing their per-partition costs without saying so +is how a table like this misleads. + +## The comparison + +| system | a partition is | per-partition cost | what dominates it | +|---|---|---|---| +| PostgreSQL | an ordinary table with its own relfilenode, its own child indexes and its own statistics | high | catalog, planner, per-session metadata, lock manager | +| Kafka | a directory, whose every *log segment* carries its own `.log`, `.index` and `.timeindex` | high | file descriptors, page cache, replica fetchers, metadata | +| **WorkTable today** | **a complete generated table** | **~28 KB, measured** | **fixed apparatus allocated at creation** | +| ClickHouse | a logical group; the physical unit is a *part*, a directory of column files plus a sparse primary index | medium, and small parts are merged away | open files, and a hard cap on active parts | +| HBase | a key range, called a region | medium to high | memstore memory per region per column family | +| Cassandra | a hash token on a shared ring; one key's rows live inside shared SSTables | near zero | bloom filters and index summaries, which scale with partition *count* | +| Snowflake | a 50 to 500 MB uncompressed columnar block, created automatically | not a comparable concept | n/a | + +### PostgreSQL + +A partitioned table "is a 'virtual' table having no storage of its own. Instead, +the storage belongs to *partitions*, which are otherwise-ordinary tables." An +index declared on the parent is virtual in the same way, so N partitions and M +indexes are N x M physical index relations. + +**There is no documented fixed byte overhead per partition, and nothing here +invents one.** What the documentation does commit to is that the planner +"is generally able to handle partition hierarchies with up to a few thousand +partitions fairly well, provided that typical queries allow the query planner to +prune all but a small number of partitions", and that "each partition requires +its metadata to be loaded into the local memory of each session that touches +it" — so the memory cost is per session times per partition, not paid once. + +Two traps worth knowing. Autovacuum does **not** analyze the partitioned parent, +only its children, so parent-level statistics need a manual `ANALYZE` +(PostgreSQL 18 revisits this, adding an `ONLY` option and changing the recursion +default). And the sharpest practical limit is the lock manager rather than disk: +every partition and partition index touched takes a relation lock, fast-path +slots were fixed at 16 per backend before PostgreSQL 18 and are sized from +`max_locks_per_transaction` after it, and overflow spills to the shared lock +table and shows up as `LWLock:LockManager` waits. + +### Kafka + +A partition is a directory named `-`, and the file cost is +**per segment inside it**, not per partition: each log segment carries its own +`.log`, `.index` and `.timeindex`, and each index pair is an mmap. Segments roll +at `log.segment.bytes`, one gigabyte by default. + +**The partition-count numbers most often quoted are ZooKeeper-era and should be +labelled as such.** The familiar "limit partitions per broker to `100 * b * r`, +roughly 2,000 to 4,000 per broker" guidance is from a 2015 Confluent post, and +those bounds came from controller failover and unclean-failure availability +rather than steady-state cost. + +KRaft changed this substantially: Confluent's current documentation cites a +benchmark cluster running **two million partitions**, "10 times the maximum +number of partitions for a cluster running ZooKeeper". Note what is *not* +available: neither Apache nor Confluent publishes a current numeric supported +maximum per broker or per cluster under KRaft, only that "Kafka's scalability +still primarily depends on adding nodes". KRaft reached general availability in +3.3, parity in 3.9, and ZooKeeper was removed in 4.0. + +### ClickHouse + +**A partition is not a part, and the two words are not interchangeable.** +`PARTITION BY` defines a *logical* partition; a *part* is the physical on-disk +unit, a directory of column `.bin` files, `.mrk` mark files and `primary.idx`. +One partition contains many parts. Every insert creates at least one part per +affected partition, and parts in different partitions are never merged together. + +The primary index is genuinely sparse: one entry, a "mark", per granule of rows +rather than one per row, with `index_granularity` defaulting to 8,192 rows and +adaptive granularity via `index_granularity_bytes`, default 10 MB. + +Background merges do fold small parts together, and the limits that enforce it +are the clearest statement of what a partition costs there: +`parts_to_delay_insert` at 1,000 and `parts_to_throw_insert` at 3,000 active +parts **per partition**, plus `max_parts_in_total` at 100,000 per table. The +partition-count guidance is explicit: "you shouldn't make overly granular +partitions (more than about a thousand partitions)", because of "an +unreasonably large number of files in the file system and open file +descriptors". + +### Cassandra and HBase are not one row + +Pairing them was an error. They partition differently and cost differently. + +**Cassandra does not use key ranges.** It "partitions data over storage nodes +using a special form of hashing called consistent hashing": the partition key is +hashed by `Murmur3Partitioner` into a 64-bit token, and what maps to nodes is a +token range, a range of *hashes*. Order-preserving partitioning exists and is +strongly discouraged. + +Its per-partition cost is genuinely low, with no file or directory per +partition, but it is not zero: bloom filters and index summaries scale with +partition **count**, and a billion partitions at the default 1% false-positive +rate costs roughly 1.2 GB of off-heap bloom filter memory. The governing limit +there is partition *size* rather than count, around 100 MB. + +**HBase regions are key ranges, and they are not cheap.** The documentation puts +"20-200 regions per RegionServer" as the reasonable range, with the maximum +"mostly determined by memstore memory usage": each region has a memstore per +column family, flush sizes typically 128 to 256 MB, and exceeding the budget +"can cause undesirable consequences such as unresponsive server or compaction +storms". A worked example on a 16 GB server lands near 51 regions. + +### Snowflake + +Verified, and the reason to keep it in this table is a naming one. "Each +micro-partition contains between 50 MB and 500 MB of uncompressed data", and +"micro-partitioning is automatically performed on all Snowflake tables". They +are never declared; the knob a user does control is the clustering key. + +**So do not use "micro-partition" in WorkTable's user-facing text for a 23-row +partition.** The word already means something automatic and enormous to everyone +who has met it. The generated type here is `DenseTable` for this reason. + +## What the cost actually buys, stated carefully + +An earlier version of this comparison claimed that isolation is what the +per-partition cost buys, and that "2,000 independent tables never contend, where +a Cassandra-style shared index under 10,000 writes/sec would". **Both halves of +that are wrong and neither should be repeated.** + +The Postgres half is false: partitions share the buffer pool, the WAL and the +lock manager, every partition and partition index touched takes a relation lock, +and unpruned partition scans under concurrency are a documented way to lose +throughput. Many partitions can contend *more* than few, which is the opposite +of the claim. + +The Cassandra half is unsupported. No source was found for a shared LSM index +contending at 10,000 writes per second; Cassandra's write path is a sequential +commitlog append plus a memtable insert with no read-before-write, and per-node +rates well above that are unremarkable. Its real contention modes are hot +partitions and compaction backpressure, neither of which is a function of +aggregate write rate. + +What per-partition cost genuinely buys is **independent physical objects**: +detaching or dropping one as a near-metadata operation, per-partition indexes, +compression, retention and statistics, and partition pruning. That is a real +benefit and it is worth paying for. It is not "never contends". + +For WorkTable specifically, the isolation is stronger than Postgres's because +there is no shared lock manager to contend on: a partition is an independent +generated table behind its own handle. That is a claim about this system and it +should be made about this system rather than by analogy. + +## What WorkTable's own partition costs, measured + +Not recalled. `tests/dense_partition_memory.rs` counts what the allocator was +asked for, 200 partitions of 23 rows, one declaration at two widths: + +| shape | bytes per partition | +|---|---:| +| full table, empty | 28,404 | +| **dense, empty** | **108** | +| full table, 23 rows of an 88-byte row | 32,900 | +| **dense, same** | **3,180** | + +The empty row is the one to read. The saving is fixed apparatus allocated at +partition creation, so it is about 28 KB per partition whatever the rows weigh: +roughly 56 MB at 2,000 symbols. + +`partition_max_size: u8` is how a declaration asks for the second shape. See +`docs/small-tables.md` for where the 28 KB goes, and the user guide's section 9 +for the grammar. + +## Provenance + +Every claim above about another system was checked against that system's +current documentation before this page was written, because the version of this +comparison it replaces was written from memory and got ClickHouse's central term +backwards, merged two systems that partition differently, and asserted a +contention figure that does not appear to exist. + +Three things are marked unverified above rather than smoothed over: a +byte-level per-partition overhead for PostgreSQL, a current official numeric +partition maximum for Kafka under KRaft, and the 10,000 writes per second +Cassandra figure, which was deleted rather than softened. + +Sources: the PostgreSQL manual on declarative partitioning and `pg_class`, +the PostgreSQL 18 release notes, the Apache Kafka log implementation +documentation, Confluent's 2015 partition-count post and its current KRaft +documentation, the ClickHouse MergeTree, custom-partitioning-key and +sparse-primary-index pages, the Cassandra Dynamo-architecture and bloom-filter +pages, the HBase region-and-capacity guide, and Snowflake's table-clustering +and micro-partitions page. diff --git a/docs/partitioned-tables-implementation.md b/docs/partitioned-tables-implementation.md index 332b90df..4a91ca37 100644 --- a/docs/partitioned-tables-implementation.md +++ b/docs/partitioned-tables-implementation.md @@ -222,6 +222,7 @@ share one persistence space across partitions and partition only in memory. worktable!( name: Price, partition_by: symbol_id: u16, + partition_max_size: u64, columns: { exchange_id: u8 primary_key, bid: f64, ask: f64 } ); ``` diff --git a/docs/partitioned-tables-proposal.md b/docs/partitioned-tables-proposal.md index c7bbf597..dd91a0bd 100644 --- a/docs/partitioned-tables-proposal.md +++ b/docs/partitioned-tables-proposal.md @@ -127,6 +127,7 @@ generated router is the **partition set**. worktable! ( name: Price, partition_by: symbol_id: u16, + partition_max_size: u64, partitions: 1024, columns: { exchange: Exchange primary_key, @@ -258,6 +259,7 @@ worktable!( worktable!( name: Price, partition_by: symbol_id: u16, + partition_max_size: u64, partitions: 1024, columns: { exchange: Exchange primary_key, bid: f64, ask: f64, ts: u64 } ); diff --git a/docs/partitioned-tables-worked-example.md b/docs/partitioned-tables-worked-example.md index 87181b18..d8c2879f 100644 --- a/docs/partitioned-tables-worked-example.md +++ b/docs/partitioned-tables-worked-example.md @@ -121,6 +121,7 @@ whole game. worktable!( name: OrderBook, partition_by: symbol_id: u16, + partition_max_size: u64, partitions: 1024, columns: { exchange_id: u8 primary_key, diff --git a/docs/persistence-durability.md b/docs/persistence-durability.md index bc37ac23..12ad929a 100644 --- a/docs/persistence-durability.md +++ b/docs/persistence-durability.md @@ -17,7 +17,7 @@ This is an explicit product boundary, not an implied durability guarantee. | Graceful process exit after `close()` | The WorkTable worker completed all writes it reported. | Survival of a subsequent power loss before the operating system commits buffered writes. | | Process crash or `SIGKILL` | No row-fidelity guarantee for an interrupted batch. The next load either returns a state whose primary links and rows validate, or returns `PersistenceLoadError`. | Preservation of the latest acknowledged changes. | | Power loss | The next load applies the same validation/refusal boundary. | Any acknowledged-change retention window; current batches do not call `fsync`. | -| S3 synchronization | Successful calls report completion of the configured upload path. | A transactionally consistent multi-file snapshot. Treat independently uploaded objects as best-effort unless an application-managed snapshot generation protects them. | +| S3 synchronization | A successful persistence operation has uploaded every new immutable segment and then committed one checksummed manifest covering the data, primary index, and secondary indexes. Restore validates all referenced segments before atomically installing the local directory. | S3 makes the local disk engine's completed state remotely recoverable; it does not make the local multi-file update power-loss atomic, call `fsync`, or provide multi-writer coordination between processes. | Call `close()` during orderly shutdown. If `wait_for_ops()` is used before a non-consuming shutdown path, stop application writers first; otherwise a writer can @@ -57,6 +57,37 @@ The strict audit is proportional to the number of primary-index entries. It runs during `load()` and adds no branch, lock, or scan to steady-state insert, select, update, or delete paths. +## S3 generation protocol + +The S3 engine compares each table file in 16 KiB DataBucket-page units and names each +uploaded segment by its BLAKE3 content hash. Adjacent changed pages are coalesced up to +the 4 MiB throughput target, but that target is not a minimum. One isolated page change +uploads one 16 KiB segment plus the small manifest. The integration fixture measures +16,842 uploaded bytes for a one-row update to a 14,385,146-byte table, or 0.117% of the +local table size. A mutation still scans and hashes the local files after the disk engine +completes. Dirty-page reporting from DataBucket can remove that local scan in a future +compatible optimization. + +The mutable `manifest.v1` object is the only remote commit point. It is written after +all referenced immutable segments. A failed manifest PUT leaves the preceding generation +visible; a failed response is resolved by reading the manifest back and comparing its +exact bytes. Startup refuses a corrupt manifest, a missing segment, a length mismatch, or +a hash mismatch. It restores into a sibling staging directory and renames that directory +into place only after every table file validates, so a failed remote restore leaves the +existing local table untouched. The stable object name remains `manifest.v1`; its +checksummed body carries the format version, and the reader accepts the prior fixed-chunk +body as well as the page-extent body. + +Segments no longer referenced by the current manifest are retained. This prevents a +concurrent restore that already read the prior manifest from losing a segment underneath +it. Object reclamation therefore belongs in an explicit offline or lease-aware garbage +collector; the alpha engine does not delete remote segments automatically. + +When `manifest.v1` is absent, startup lists and restores the former whole-file layout. +The next successful mutation uploads segments and establishes the first manifest. Once a +manifest exists, its failure is fatal; WorkTable will not silently continue from stale +local files and overwrite a newer remote generation. + ## Offline index recovery `PersistedWorkTable::load_with(engine, LoadMode::Recovery)` is a low-level diff --git a/docs/pr105-source-fixes.md b/docs/pr105-source-fixes.md new file mode 100644 index 00000000..a310eb00 --- /dev/null +++ b/docs/pr105-source-fixes.md @@ -0,0 +1,59 @@ +> Historical source review at the revision below. The September release audit has since compiled and tested its retained fixes; current release evidence is in perf-benchmarks/docs/release-readiness.md. Its historical query-profile warning is resolved by the final owned-execution follow-up at the end of this document. + +# PR 105 source-only follow-up + +These changes are best-effort source fixes against revision +`5fa24e1b1dd85a7a44b57b60ee247dbe4f54639d`. No compilation, tests, or +benchmarks were performed. + +## Behavioral changes + +- Columnar tables use a shared publication gate around insert/reinsert/delete + index maintenance and primary-row publication. Dirty rebuilds acquire it + exclusively. Mutation stripes must be acquired before this gate; the gate + precedes the columnar data lock. Non-columnar implementations return no guard. + Low-level callers that manually combine secondary-index and primary-row changes + must also hold the publication guard through the whole operation. +- Clean columnar reads avoid taking an exclusive rebuild lock. A dirty rebuild + still blocks relevant writers for its entire scan. The additional shared + acquisition on columnar mutation paths and contention cost are unmeasured. +- Live and persisted index capacities are capped at the u16 slot-count limit. + The byte stride itself is not capped at 64 KiB. +- Persisted CDC deletion clones a secondary key before consuming it, retaining + the complete row for columnar removal. +- Canonical schemas preserve columnar fields, per-field settings, clustered + indexes, slot width, and default chunk size. Columnar changes are classified + as derived-index rebuilds by the planner. The checker validates clustered keys. +- Snapshot-loading opens request read permission only. +- TOC fallback recovery uses the configured stride. +- A persistence engine owns one private worker for blocking HostFile operations, + separate from compute pools. A scope guard shuts its pool down when the task + completes or is dropped. This costs one thread per live persistence engine; + synchronous file operations remain blocking, including foreground loaders. +- Disabled event-ledger batch submission no longer constructs the event-ID vector. + +The DSL version and exact consumer pins are advanced to the proposed +`1.0.0-beta.19`. This is a local source proposal, not a published release. +Check version availability and release all dependent packages together before +publishing. The new schema field and diff variant require downstream consumers +with struct literals or exhaustive matches to adapt. + +## Still open + +Runtime selectors and per-query/section profiles are not fully connected to +execution. This patch does not implement that wiring or remove those APIs. +In particular, the private persistence worker deliberately does not follow a +compute-pool profile. Do not interpret an accepted selector as evidence of +runtime isolation or changed scheduling behavior. + +## Regression source + +`dsl/tests/columnar_schema.rs` covers metadata round trips, rebuild classification, +and invalid clustered keys. Columnar integration tests cover the gate contract +and representable index capacity. Existing concurrent-reinsert tests remain. +The gate test checks exclusion, not every race interleaving; no test result or +latency improvement is claimed. + +## Final runtime execution follow-up + +The per-query scheduling gap above is resolved by owned asynchronous select execution and Arc-based annotated mutations. See query-runtime-release-gate.md for the precise ownership, cancellation and validation contract. The schema grammar is unchanged. diff --git a/docs/query-runtime-release-gate.md b/docs/query-runtime-release-gate.md new file mode 100644 index 00000000..544ca11b --- /dev/null +++ b/docs/query-runtime-release-gate.md @@ -0,0 +1,14 @@ +# Owned query runtime execution + +The release review found that select profiles only recorded tuning and mutation section profiles were ignored. The implementation now uses an explicit ownership boundary without changing grammar. + +- Generated hosted paged rows carry their declared backend and flavor. A named profile must match the backend family. Nagoya profiles may select another flavor without changing the table declaration. +- Synchronous execute remains available. An explicitly selected runtime requires execute_async().await; execute returns RuntimeRequiresAsync rather than ignoring the selection. +- execute_async materializes borrowed iteration and predicates on the caller before constructing the future. Owned range filtering, sorting, offset and limit execute on the selected pool. It defaults to the table's executor, materializes all input rows and does not fan out a query across workers. +- Runtime-annotated update/delete/in-place methods require an Arc table receiver and owned Send/static arguments. Unannotated methods retain their borrowed signatures. Portable table locks and private persistence I/O workers are unchanged. +- Pending owned tasks are cancelled when their waiting future is dropped. Synchronous work already running can complete; cancellation is not rollback. Panics propagate to the caller. +- Vec tables reject query profiles. Without default features, owned select execution remains inline and hosted profile markers are unavailable. + +The generated-table tests in tests/runtime_execution.rs verify worker identity, query results with borrowed non-Send predicates, same-pool nesting on one worker, cancellation, panic propagation, persistence/reopen and optional Tokio execution. The wt-owned-runtime benchmark measures full materialization, synchronous versus scheduled sorting, empty dispatch roundtrips and scheduled mutations; it checks equal results. The separate full CI run covers existing callsites, no-default consumers and Clippy. + +The old scoped-fork experiment is not used. See Nagoya's deferred-experiments note for its independent panic/progress defects. Canonical user-facing documentation is docs/wt-user-guide.typ. diff --git a/docs/remote-page-stores-and-partial-hydration.md b/docs/remote-page-stores-and-partial-hydration.md new file mode 100644 index 00000000..d991becf --- /dev/null +++ b/docs/remote-page-stores-and-partial-hydration.md @@ -0,0 +1,1027 @@ +# Remote page stores and partial hydration + +**Status:** accepted, 2026-09-12 + +**Scope:** DataBucket storage domains, WorkTable partial hydration, Upstash Redis, +Tigris object storage, a dual-write backend using both services, and measured +S3-compatible provider alternatives. + +## Decision + +DataBucket becomes the storage-facing API. It owns stable page identities, the +physical system catalog, mutation generations, page reads and writes, and the +backend contract. WorkTable remains the typed table and query layer above it. + +WorkTable will no longer require every row page to be resident. A spillable +table starts fully resident and uses the same in-memory path while it remains +below its configured memory high-water mark. After it crosses that boundary, +it evicts eligible pages and faults them back from a local file or a configured +remote page store when a query needs them. Fully resident +tables keep their current synchronous API and hot path. Spillable tables use a +distinct generated wrapper with asynchronous, fallible query and mutation +callsites. This requires no DSL grammar change. + +The three remote configurations are: + +| Configuration | Primary purpose | Commit authority | Read shape | +|---|---|---|---| +| Upstash | Optional batched page cache and metadata service | Upstash generation head | Direct page keys, batched with `MGET` | +| Tigris | Default durable capacity and scans | Conditional Tigris generation head | Range reads from immutable page segments | +| Hybrid | Optional Upstash serving tier plus a Tigris durable copy | Upstash live head plus a Tigris commit marker | Upstash first for point reads, Tigris for scans and repair | + +Hybrid means every acknowledged generation is written to both services. It is +not a cache with an optional backup. + +The 2026-09-12 provider gate changed the implementation priority. Upstash's +temporary Redis service had a roughly 216 ms request floor from Fly Singapore +and did not scale independent page requests with concurrency. It is not on the +first durable write path, and the hybrid backend is deferred until a paid, +region-selected Upstash deployment passes the same gate. Tigris, Bunny Storage +and Cloudflare R2 all passed exact-read and conditional-head tests. Tigris and +Bunny passed the complete performance gate. Tigris is the first backend +because it had the stronger sustained write shape. Bunny is supported by the +same S3 adapter as a read-strong alternative. R2 remains adapter-compatible +but is excluded from the first production path by its range and segment +results. + +## Current boundary + +The current persisted WorkTable is restored completely into memory. Primary +and secondary indexes contain DataBucket `Link` values, and those links are +resolved against an owning in-memory page list. This gives point reads their +current inexpensive synchronous path, but it also makes available memory a +hard table-size limit. + +The database-wide S3 engine runs above the local disk engine. After a persisted +batch it walks each table file on the private persistence runtime, hashes data +pages and stable 16 KiB index chunks, and submits only changes to one shared +storage domain. DataBucket stages immutable segments and WorkTable prepares one +generated catalog checkpoint for the database before a conditional head update. +The stateful adapter fixture measures 33,016 uploaded bytes for an isolated page +mutation with a small catalog and 49,544 bytes after the catalog grows beyond one +page. This removes the 4 MiB network floor. The full local scan remains a local +CPU and disk cost until exact dirty-page reporting is connected. + +DataBucket already knows the affected `Space`, `PageId`, physical stride and +row extent at `persist_page`, `persist_pages_batch`, and `update_at`. It should +report mutations at that point so the S3 engine can skip the scan. A raw +`AsyncWrite` wrapper cannot recover the same meaning reliably from byte offsets. + +## Dependency direction + +The runtime relationship is deliberately two-way while the Cargo graph stays +one-way: + +```text +application + |-- generated WorkTable API + | `-- data_bucket storage-domain API + `-- data_bucket API directly + +worktable --depends on--> data_bucket + | | + `-- generated catalog -' DataBucket owns its write permit and commit order +``` + +DataBucket defines the bounded catalog records, generation transaction and a +`SystemCatalog` provider interface. WorkTable implements that interface with a +real generated `vec: true` WorkTable. DataBucket receives a private write +permit and publishes the prepared table only after the page objects, catalog +checkpoint and conditional generation head are durable. Applications receive +read-only typed views over the same generated table. + +This does not create a Cargo cycle. WorkTable depends on DataBucket's protocol; +DataBucket never names the WorkTable crate. At runtime, WorkTable supplies the +catalog implementation that DataBucket owns and updates. The hosted S3 adapter +is an optional DataBucket module behind `std` and `s3-support`; the storage +domain records and catalog interface remain `no_std` plus `alloc`. + +## Storage domain and bootstrap + +A storage domain represents one database persistence root. It contains all +persisted table spaces plus reserved system spaces. A WorkTable generated type +registers its spaces with the domain when it opens. + +Every backend has one deterministic bootstrap location: + +```text +bootstrap head + | + v +generation manifest + | + +--> catalog root and catalog segments + +--> table data pages or page segments + +--> primary-index pages + `--> secondary-index pages +``` + +The bootstrap record is intentionally small. It contains the storage-domain +identifier, format version, current generation, parent generation, manifest +identity, manifest checksum and writer epoch. It does not contain the full +catalog. + +The catalog checkpoint cannot require the catalog to locate itself. The small +bootstrap head therefore names that immutable checkpoint directly. This is the +only raw bootstrap record. User data and index pages are located through rows +in the generated catalog. + +The DataBucket v3 data-page format remains the unit validated after a fetch. +The new catalog has its own format version. It should not add fields to the +archived `SpaceInfoPage` shape merely to hold statistics, because that would +unnecessarily change the v3 page layout. This design should land before the v3 +storage-domain contract is declared stable. + +## Physical system catalog + +The authoritative catalog is database-wide. It has one table row per logical +table, one page row per current logical page, one index row per logical index, +and replication rows when the hybrid backend is active. + +Conceptually, its stable records are: + +```rust +struct SystemTableRow { + table_id: SpaceId, + name: String, + schema_version: u32, + row_count: u64, + live_row_bytes: u64, + allocated_data_pages: u64, + live_data_pages: u64, + primary_index_entries: u64, + secondary_index_entries: u64, + applied_generation: Generation, + durable_generation: Generation, +} + +struct SystemPageRow { + table_id: SpaceId, + space_id: SpaceId, + page_id: PageId, + page_kind: PageKind, + generation: Generation, + object: ObjectId, + object_offset: u64, + encoded_length: u32, + decoded_length: u32, + checksum: Checksum, + live_rows: u32, + live_bytes: u32, +} + +struct SystemReplicationRow { + generation: Generation, + upstash: ReplicaState, + tigris: ReplicaState, + last_error: Option, +} +``` + +These are logical shapes. Their persisted representation must use bounded +fields and DataBucket-owned types suitable for `no_std` plus `alloc`. +Human-readable error text belongs in process diagnostics, not the durable +catalog. + +WorkTable exposes generated read-only views such as +`system_tables()`, `system_pages()` and `system_replication()`. The user can +filter and inspect them, but cannot insert, update, delete, vacuum, or define +indexes on them. Generation is automatic and does not add schema grammar. + +### Maintained values + +Exact values that would otherwise require loading or scanning all pages are +updated as part of each mutation generation: + +- live row count; +- live archived-row bytes; +- allocated and live page counts; +- entry count for every primary and secondary index; +- current applied and durable generations; +- tombstone or ghost count when the table representation has that state; and +- per-backend replication state. + +`count()` and `row_count()` read the maintained row count in O(1). They do not +derive it from the number of resident rows or walk the primary index. The +in-process value advances when a mutation is published. The durable value +advances only when that generation commits. Both generations are observable +so an operator can distinguish live state from remotely recoverable state. + +A batch computes one aggregate delta and publishes it once. Failed unique +inserts, rolled-back index operations and abandoned generations do not change +the committed values. Recovery can verify or rebuild the aggregates offline +from v3 row directories and persisted indexes, but ordinary open and query +paths trust the checksummed committed catalog. + +Values such as column minimum and maximum should not be included initially. +They are cheap on insert but can require an unbounded search when the current +extreme is deleted. A maintained statistic belongs here only when every +mutation can update it with bounded work or it is explicitly approximate. + +Process-local cache statistics are exposed through runtime metrics rather than +persisted catalog rows. Cache hits, misses and current resident bytes are not +database facts. + +## DataBucket generation contract + +DataBucket collects physical changes into a generation before an adapter sees +them: + +```rust +struct PageMutation { + address: PageAddress, + kind: MutationKind, + image: PageImage, + checksum: Checksum, +} + +struct GenerationPlan { + id: Generation, + parent: Generation, + writer_epoch: WriterEpoch, + pages: Vec, + catalog_delta: CatalogDelta, +} +``` + +WorkTable opens a DataBucket generation before applying a logical operation +and passes that generation context through every data, primary-index, +secondary-index and catalog write. The context owns the changed page images +until `finish()` produces the plan: + +```rust +let mut generation = domain.begin_generation(expected_parent)?; +data_space.persist_pages(&mut generation, data_pages).await?; +primary_space.persist_pages(&mut generation, primary_pages).await?; +secondary_spaces.persist_pages(&mut generation, secondary_pages).await?; +generation.apply_semantic_delta(index_delta)?; +let plan = generation.finish()?; +``` + +The real signatures may differ, but generation membership cannot be inferred +later from unrelated file writes. Existing low-level DataBucket callers may +use an explicit one-operation generation or a non-durable sink. WorkTable is +responsible for grouping all physical parts of one logical mutation. + +DataBucket derives row-count, live-row-byte and page-count deltas by comparing +the validated old and new page directories. This keeps those facts correct for +direct DataBucket consumers as well as WorkTable. WorkTable supplies semantic +index-entry deltas because DataBucket does not understand every index +operation. DataBucket records those deltas only with the page generation they +describe. + +`PageAddress` contains the storage domain, table, space and page identifiers. +It is stable across cache eviction. A physical object identity is assigned by +the backend and stored in the resulting catalog snapshot. + +The adapter interface needs these operations: + +```rust +trait PageStore { + fn load_head(&self, domain: StorageDomainId) -> Result, StoreError>; + fn load_catalog(&self, head: &Head) -> Result, StoreError>; + fn read_page(&self, page: &PageRef) -> Result; + fn read_pages(&self, pages: &[PageRef]) -> Result, StoreError>; + fn stage(&self, plan: &GenerationPlan) -> Result; + fn stage_catalog(&self, staged: &mut StagedGeneration, checkpoint: &[u8]) + -> Result<(), StoreError>; + fn commit(&self, staged: StagedGeneration) -> Result; +} +``` + +The exact Rust shape may use associated futures to preserve `no_std` and avoid +an `async_trait` allocation. The semantic split between `stage` and `commit` +is required. + +Staging writes immutable page or segment objects and an immutable catalog +snapshot. Commit moves the small bootstrap head from the expected parent to +the new generation. A failed or repeated stage is idempotent. A commit with a +different current parent returns a conflict instead of applying last-writer +wins. + +The first implementation supports one active writer for a storage domain. +The writer epoch makes stale processes detectable. Multi-writer coordination +is a separate protocol and must not be implied by an atomic object PUT or a +Redis transaction. + +## Partial hydration + +### Residency model + +The unit of data residency is a complete validated DataBucket page, not an +individual row. A page frame moves through these states: + +```text +Absent + | fault + v +Loading --> CleanResident --> Evicting --> Absent + | + | mutation + v + DirtyResident --> Flushing --> CleanResident +``` + +Only one load may be in flight for a page. Concurrent faults share its result. +A query or mutation pins the page frame while it decodes or changes a row. +Pinned pages cannot be evicted. Dirty pages cannot be discarded until their +generation is committed or retained in a durable local write-ahead record. + +The cache key includes logical page identity and the committed object checksum +or generation. This prevents a cached old page from satisfying a newer +catalog reference. Every fetched page is checked using DataBucket's v3 header, +page identity, row directory, bounds and checksum before publication. + +The initial cache policy should be a segmented LRU or CLOCK variant with: + +- an explicit byte budget rather than a page-count budget; +- separate data-page and index-page budgets; +- high and low watermarks so eviction is batched; +- pin and dirty-state awareness; +- negative caching only for catalog-proven absence; and +- bounded metadata per non-resident page. + +The cache manager must account for page frames, decoded scratch buffers and +in-flight fetches. A range query cannot evade the budget by issuing thousands +of reads concurrently. Per-query prefetch concurrency and bytes are bounded. + +### Spill mode + +Spill is a residency transition, not a different persisted table format. A +spillable table has three runtime conditions: + +```text +Resident: every current page is in memory +Spilling: resident bytes crossed the high watermark; eviction is active +Spilled: at least one current page is absent and must be faulted on demand +``` + +Most tables remain `Resident` for their complete lifetime. They pay the +spillable wrapper's budget accounting and one predictable resident-page check, +but perform no storage read and run no eviction work. The ordinary resident +table type pays neither cost. + +`SpillConfig` contains at least a soft byte budget, a lower target watermark, +a hard byte ceiling, data/index budget shares, maximum in-flight fetch bytes +and a backing `PageStore`. Crossing the soft high watermark schedules eviction +until resident bytes fall below the lower watermark. Hysteresis prevents a +table from oscillating around one exact byte value. + +The hard ceiling is a correctness boundary. If pinned, dirty and in-flight +pages leave no eligible victim, an allocating query or mutation waits for +flush/eviction progress or returns a typed budget error according to its +deadline. It does not exceed the configured ceiling indefinitely or discard a +dirty page. Allocation failure is not used as the normal signal to start +spilling; eviction begins before that point. + +Clean pages already named by the committed catalog can be dropped immediately. +A dirty page must first become part of a staged generation or a synchronously +durable local WAL record. Newly inserted and recently faulted pages enter the +hot segment. Sequential scans receive weak admission so one scan does not +replace the repeatedly accessed working set. + +Spill works in both directions. A fault makes a page resident again, and a +small table can return to having every page resident after old rows are +deleted. The spill-capable Rust type does not change back into the synchronous +resident type because it may spill again on its next mutation. + +Opening an existing store follows the same budget. If all current pages fit, +the table may hydrate completely and report `Resident`. If they do not, open +loads bootstrap metadata, catalog roots and the configured index working set, +then leaves remaining pages cold. It does not first load the entire table only +to evict it. + +Runtime spill condition, cache hits and resident bytes are process facts and +are not persisted as authoritative table statistics. The system interface may +join them with durable catalog rows for observation. Exact row count, live +bytes, logical page count and index-entry counts remain maintained catalog +values, so they are correct even when almost every page is cold. + +### Index residency + +Partial hydration has two implementation stages, both covered by the target +design: + +1. Keep primary and secondary indexes resident while data pages use the + bounded cache. This removes the dominant row-byte requirement and provides + the first useful release boundary. +2. Keep index roots and selected upper nodes resident, then fault lower WTI, + ART and table-of-contents pages through a bounded index cache. This removes + the remaining requirement that every index entry fit in memory. + +Stage one is not the final claim that arbitrary tables fit in bounded memory. +Documentation must say that indexes still need to fit until stage two lands. + +The pageable index interface cannot expose raw pointers into evictable nodes. +It resolves an equality or range lookup into stable DataBucket links while +holding node pins. The result owns its links before pins are released. Existing +fully resident WTI, ART, Arctic and Congee implementations keep their current +interfaces. + +### Query execution + +A spillable query plans page access before fetching rows: + +1. Resolve the primary or secondary index to stable row links. +2. Group links by `(space_id, page_id)` and deduplicate page requests. +3. Visit resident pages immediately. +4. Fetch missing pages in bounded batches. +5. Validate and publish each fetched page once. +6. Decode all requested rows from that page while it is pinned. +7. Apply filters, ordering, offset and limit according to the existing query + contract. + +Point lookup faults at most the required index path and one data page. A range +lookup prefetches upcoming pages within a small byte window. A table scan uses +the catalog's ordered live-page list and streams pages through the cache. It +does not first create one future or buffer per page. + +Ordering and early termination matter. When an index already supplies the +requested order, `limit` should stop hydration after enough matching rows are +produced. A sort on an unrelated field may require reading every candidate. +If its result cannot fit the query memory budget, the executor needs an +external merge path backed by temporary storage. Partial hydration alone does +not make an unbounded sort bounded. + +Current WorkTable queries are not snapshot-isolated transactions. Partial +hydration preserves the existing concurrency contract. It must still avoid +combining object identities from different committed catalog generations. +Resident dirty pages take precedence over their prior committed backing page. + +### Mutations + +Updating or deleting a cold row first faults and pins its page exclusively. +The mutation updates the row page, all affected indexes and the aggregate +delta under one WorkTable operation. A relocation produces the new page image, +the old page image, every changed index page and one catalog delta in the same +generation. + +An insert may use a resident page with free space or allocate a new logical +page. The free-space summary needed to choose that page is maintained in the +catalog. Choosing an insertion target must not scan or hydrate every page. + +Vacuum follows the same rule. Its candidate summaries are catalog metadata. +It hydrates only selected source and destination pages, emits the complete set +of relocated links and page changes, then updates counts once. Vacuum may not +publish a reclaimed page before every index relocation and catalog change in +its generation is staged. + +### Generated API + +The resident generated type remains source-compatible: + +```rust +let table = UserWorkTable::load(engine).await?; +let row = table.select(id); // Option +let rows = table.select_by_tenant(tenant).execute()?; +``` + +Every persisted declaration also generates a spillable wrapper without new +DSL: + +```rust +let table = UserWorkTable::load_spillable( + engine, + SpillConfig::memory_limit(256 * 1024 * 1024), +).await?; + +let row = table.select(id).await?; // Result, _> +let rows = table + .select_by_tenant(tenant) + .limit(100) + .execute() + .await?; +``` + +The concrete returned type is `UserSpillableWorkTable` unless code-generation +constraints require an opaque equivalent. The important constraint is that it +is a different Rust type. Calling a synchronous `select() -> Option` on a +table that may spill later would otherwise have only three bad choices: block +unexpectedly, report a storage failure as absence, or panic. The spillable +callsite is asynchronous from construction, but a resident hit completes +without scheduling storage I/O. + +The existing `execute_async()` query option selects a runtime for CPU work. It +does not currently mean storage hydration and must not be repurposed silently. +The spillable builder's `execute().await` performs both asynchronous page +access and the selected CPU execution policy. + +Spillable updates and deletes are asynchronous and fallible because they may +fault pages. The resident methods remain unchanged. This is a callsite +extension, not grammar. + +## Upstash backend + +Upstash stores complete encoded DataBucket pages as immutable values. The +default 16 KiB page is well below current record and request limits. Keys use +one Redis hash tag per storage domain so generation-head coordination and its +metadata share a locking domain. + +This remains a defined adapter path, but it did not pass the first provider +gate. From a Fly Singapore Machine, SET and GET medians were both about 216 ms +and a one-page write plus Lua head compare-and-set was about 444 ms. Sending +128 pages in one MSET reached 213 page writes/s, but the generation still +needed a second request and completed only 1.67 times/s. The command API also +requires base64 for binary pages carried in JSON. An implementation must batch +behind the local WAL; it must not synchronously call Upstash for every row +mutation. + +One possible key layout is: + +```text +wt:{domain}:head +wt:{domain}:generation::manifest: +wt:{domain}:page: +wt:{domain}:catalog: +wt:{domain}:writer +``` + +Page values are content addressed. Staging uses `SET ... NX`; an existing key +is accepted only after its length and checksum match. The immutable generation +manifest maps logical page addresses to page hashes. Large manifests are +segmented so no transaction or request approaches the service request limit. + +Cold point reads use `GET`. Queries group page hashes and use `MGET` or a REST +pipeline within a configured byte ceiling. Pipelines reduce round trips but +are not a commit primitive. Upstash's `/multi-exec` transaction endpoint can +atomically update bounded metadata. Because REST `WATCH` is unavailable, the +head compare-and-set uses a small Lua script or an equivalent supported atomic +conditional operation. + +The script verifies the expected parent generation and writer epoch, then +publishes the new head. The head names the immutable catalog snapshot that +already contains the exact aggregate values, so this atomic operation remains +small. Retrying it with the same generation is idempotent. + +Upstash configuration used as durable storage must not enable Redis eviction +or attach TTLs to WorkTable keys. Quota or command-limit failures are storage +errors and cannot be treated as cache misses. Command count and transferred +bytes are first-class backend metrics because they determine both latency and +cost. + +Garbage collection retains every object reachable from the current head and +the configured recovery window. Unreachable staged generations are deleted +only after their writer lease expires and no retained manifest names them. + +### Upstash transport and access security + +A Fly Machine reaches the normal Upstash endpoint over the public network. +Fly's private 6PN does not extend to Upstash. The connection therefore relies +on all of these boundaries: + +1. TLS with normal hostname and certificate validation protects page contents + and credentials in transit. +2. A dedicated Upstash ACL user grants only the commands and key prefix needed + by one WorkTable storage domain. The application must not use the default + full-database token when an ACL token can express the smaller authority. +3. The ACL token is stored as a Fly app secret and sent in the HTTP + `Authorization` header. It is never placed in a URL, image, `fly.toml`, log, + trace field or error message. +4. A paid Upstash database enables an IPv4 allowlist containing only the + app-scoped static egress IPv4 addresses allocated to the Fly app in every + region where it runs. + +Fly's default outbound addresses are not stable enough for an allowlist. +App-scoped egress addresses survive Machine recreation, but they are regional, +so every deployed region must be allocated and allowlisted. Upstash currently +documents IPv4-only allowlisting. Deployment validation must prove that the +client actually exits through an allowed IPv4 address before the public token +path is enabled. + +The minimum writer ACL is expected to include page and manifest reads, bounded +page creation, the generation-head script and explicitly invoked garbage +collection. Its key pattern is limited to `wt:{domain}:*`. Administrative +commands, keyspace-wide scans, configuration changes, subscription commands +and unrelated key prefixes are denied. A read-only process receives a separate +read-only ACL token. Exact commands are fixed after the adapter prototype and +tested by proving required operations pass and forbidden operations fail. + +Upstash advertises VPC peering and AWS PrivateLink, but a normal Fly Machine is +not inside that AWS VPC or PrivateLink endpoint. Using either would require a +separately operated private gateway or tunnel and is not the default design. +The standard production path is TLS plus least-privilege ACL plus static-egress +IP allowlisting. + +Fly secrets protect the token at configuration and deployment time, but the +running application receives it and a person able to deploy arbitrary code or +obtain root access to the Machine can read it. Workloads that should not share +that authority must run as separate Fly apps with separate Upstash ACL users. +Rotation replaces the ACL token in Fly secrets, rolls Machines, verifies the +new credential, and then revokes the old credential. + +The operational flow is: + +1. An administrator creates the restricted ACL user in Upstash. +2. Upstash's `ACL RESTTOKEN ` command issues the REST + token carrying that user's permissions. +3. The operator imports the token into the target Fly app's secret vault. To + keep the literal token out of shell history, it can arrive through a local + environment variable and stdin: + + ```sh + printf 'UPSTASH_REDIS_REST_TOKEN=%s\n' "$UPSTASH_TOKEN" | + fly secrets import --app "$FLY_APP" + ``` + +4. Fly restarts or updates the app's Machines and injects + `UPSTASH_REDIS_REST_TOKEN` into their runtime environment at boot. +5. The WorkTable Upstash adapter reads the variable once at startup, wraps it + in a redacted secret type, and configures the HTTP client to send: + + ```text + Authorization: Bearer + ``` + +6. The adapter never implements `Debug` or tracing output that reveals the + header, token, signed request, or complete client configuration. + +The endpoint URL is not an authentication credential and may be ordinary Fly +configuration. Keeping it beside the token as a secret is also acceptable. +The Upstash token never crosses the application's public API and is unrelated +to an end-user Honey login token. End-user authorization terminates at the +application; the application uses its own storage credential to reach +Upstash. + +Fly app secrets are normally available to every Machine in that app. If only a +storage worker should have this authority, put that worker in a separate Fly +app with its own secret and expose a narrow service over Fly's private 6PN. +Changing Unix environment variables to a file does not protect the token from +root or arbitrary deployed code in the same Machine. + +Upstash's service-side encryption at rest is plan dependent. WorkTable pages +are opaque Redis values, so an optional client-side authenticated-encryption +layer can protect sensitive page and catalog payloads without losing Redis +query features that this backend does not use. Encryption keys remain in a +separate Fly secret. Object identifiers and checksums must be designed so a +malicious substitution, replay or cross-domain page copy fails authentication. + +Relevant provider constraints are documented at: + +- +- +- +- +- + +## Tigris backend + +Tigris stores immutable page segments rather than fixed 4 MiB slices of local +files. A segment is built directly from changed DataBucket page images. It may +contain one page when a flush must happen immediately or many pages when the +background writer coalesces mutations. A target segment size is a batching +goal, never a minimum write size. + +One possible object layout is: + +```text +/head +/generations//manifest +/generations//catalog/ +/segments/ +/commits/ +``` + +The page catalog records the segment hash, byte offset, encoded length and +page checksum. A cold point lookup issues a byte-range GET for the page. A +scan coalesces adjacent requested pages from the same segment. If measurement +shows that small range GETs are inefficient, the cache may fetch the complete +segment, but it still publishes pages individually and charges the fetched +bytes against its budget. + +The initial target is 4 MiB of encoded pages per segment, with a 256 KiB read +window for cold faults. The segment target is not a correctness boundary. An +idle or pressured writer may flush a smaller segment, and adjacent requested +pages may expand a read window within the query budget. + +The background writer may compress a segment when the codec allows bounded +independent page decoding. Compression metadata is stored per page or per +small frame so reading one page does not require expanding a large segment. +Already compressed archived values should be detected by measurement, not +assumed. + +Commit uploads all segments, catalog parts and the immutable generation +manifest before updating `head`. The head update is conditional on the +expected parent and writer epoch. The Rust QA driver verified conditional +creation and replacement against Tigris: stale `If-None-Match` and `If-Match` +requests were rejected with HTTP 412, while the current ETag replacement +succeeded. The first implementation still supports one writer guarded by a +renewable lease; conditional publication makes stale writers fail instead of +silently replacing the head. + +This removes the current full-file scan and 4 MiB mutation floor. A single +changed page stages roughly one page image plus manifest and catalog metadata. +Batching can improve request efficiency without increasing the correctness +unit. + +Garbage collection is manifest based. It computes reachability across every +retained generation and active reader lease before deleting immutable +segments. It never deletes an object merely because the current generation no +longer references it. + +### S3-compatible provider selection + +The same Rust executable ran from Fly Singapore against Tigris, Bunny Storage +and an R2 bucket with the APAC placement hint. Every page and range was checked +before it counted as a result. + +| Measurement | Tigris | Bunny Singapore | R2 APAC hint | +|---|---:|---:|---:| +| 16 KiB PUT p50 | 34.93 ms | 44.57 ms | 170.83 ms | +| 16 KiB GET p50 | 18.22 ms | 4.93 ms | 50.17 ms | +| HEAD p50 | 4.70 ms | 4.35 ms | 38.82 ms | +| 256 KiB range GET p50 | 24.94 ms | 5.14 ms | 49.73 ms | +| 16 KiB writes/s at concurrency 16 | 63.28 | 48.11 | 68.14 | +| 4 MiB PUT | 303.00 Mbit/s | 180.33 Mbit/s | 90.90 Mbit/s | +| 4 MiB GET | 455.59 Mbit/s | 653.57 Mbit/s | 202.37 Mbit/s | + +Bunny significantly outperformed Tigris for colocated reads. Its very high +hot-key concurrent read result is treated as cache-assisted. The lower +concurrency range median is the planning value, but that measurement also +reused one object and is not a cold-store result. The adapter-level gate must +add unique-object cold faults. Tigris had stronger sustained page and segment +writes. This selects Tigris as the first durable backend while preserving +Bunny as a supported alternative through the same S3 contract. + +Bunny replication is not part of the measured or selected protocol. The tested +zone had Singapore as its primary and no replication regions. If a deployment +later enables Bunny geo-replication, the authoritative conditional head must +still be read and written at the primary; asynchronously replicated copies +cannot coordinate writers. + +Cloudflare R2 returned the expected `200/412/412/200` conditional-write +sequence and all 7,888 verified page reads were exact. Its performance misses +the first backend gate: the 256 KiB range median is 49.73 ms, a 4 MiB PUT takes +369.15 ms on average, and that PUT sustains 90.90 Mbit/s. Its concurrent small +writes scale, but its application-facing request and segment shapes are weaker +than Tigris and Bunny from this Fly Singapore client. R2 remains a compatible +configuration of the S3 adapter rather than the first production default. +This provider choice does not alter the durable catalog or WorkTable grammar. + +Cloudflare Pipelines addresses a different boundary. It can durably buffer +HTTP ingestion and deliver records exactly once into an R2 sink. It does not +provide page-key reads, range hydration or conditional generation-head +publication. The current 5 MB/s per-stream ingestion limit and minimum +10-second R2 roll interval also make it unsuitable as the interactive page +store. A later adapter may measure it as an asynchronous mutation or WAL +export channel, with recovery consuming the materialized R2 records. It does +not replace the `PageStore` contract or repair R2's measured fault latency. + +## Hybrid dual-write backend + +The hybrid backend uses the same generation identifier, logical page images, +checksums and catalog contents in both services. Upstash is the live commit +coordinator and preferred point-read source. Tigris is the durable capacity +copy, scan source and repair source. + +No transaction can atomically commit Redis and S3 together. The backend uses +an idempotent state machine instead of claiming cross-service atomicity: + +1. Allocate generation `G` with expected parent `P` and a unique writer epoch. +2. Stage every changed page and catalog part in Upstash. +3. Stage every changed page segment and catalog part in Tigris. +4. Write the immutable generation manifest to both services. +5. Record `G` as prepared in Upstash. +6. Atomically compare `P` and the writer epoch, then move the Upstash live head + to `G` and mark it committed. +7. Write the immutable Tigris commit marker for `G`, then update its advisory + head. +8. Mark the Upstash replication row complete and acknowledge the generation. + +Every step is safe to retry using `G`. A crash before step 6 leaves only +unreachable staged objects. A crash after step 6 resumes steps 7 and 8. It +does not roll the live database back. An ambiguous response is resolved by +reading both commit records and their checksums. + +The default policy is `BothRequired`: a mutation generation is remotely +acknowledged only after both services contain it and the Tigris commit marker +exists. An optional degraded policy may keep serving when one backend is down, +but it must report a degraded generation and cannot claim dual durability. +The policy is an engine configuration, not schema grammar. + +Under `BothRequired`, head commits remain ordered and a later generation may +be staged but cannot advance the live head until its parent has completed +steps 7 and 8. Degraded operation uses the same linear generation chain and +records the missing replica work durably before accepting a child. + +When Upstash is available, reads pin its current committed generation. A page +miss or checksum failure may be repaired from the identical Tigris generation. +Large scans may read Tigris directly. The executor may mix page sources only +when every page reference comes from the same manifest and the returned hashes +match that manifest. + +When Upstash is unavailable, disaster recovery chooses the newest Tigris +generation with a valid hybrid commit marker. Because `BothRequired` writes +the marker before acknowledging, this preserves acknowledged generations. +Prepared manifests without a marker are not promoted automatically. + +Reconciliation walks generation metadata, not user rows. It copies missing +content-addressed objects, validates hashes and advances replication state. +Conflicting bytes under the same hash are corruption and stop repair. + +## Local disk and write-ahead staging + +Partial hydration must work against local files before a remote adapter is +trusted. The local DataBucket store uses `pread`-shaped page access where the +platform adapter permits it, avoiding one shared seek cursor. This provides a +deterministic correctness and performance baseline for cache faults. + +A local write-ahead staging area is the default for every remote backend. It +is required whenever the process acknowledges before the remote generation +commits, and it is what lets Tigris or Bunny coalesce writes without exposing +their request latency to each mutation. It stores complete generation plans +with checksums. Truncation happens only after the configured remote commit +condition is satisfied. A configuration that waits synchronously for the +remote generation commit may omit it, but inherits the measured provider +latency. + +If no synchronously durable local WAL is configured, an enqueue acknowledgment +retains WorkTable's existing best-effort boundary. The API and system catalog +must distinguish: + +- applied in this process; +- staged locally; +- committed to Upstash; +- committed to Tigris; and +- committed to both. + +`wait_for_ops()` waits for the configured commit policy. `close()` stops +intake, drains to that policy and joins the worker. Neither method should use +the vague word "synced" without naming the reached state. + +## Failure and correctness rules + +- A missing remote object named by a committed manifest is corruption, not an + empty page. +- A backend timeout is an availability error, not `None` from a select. +- A query never returns a row before its fetched page passes v3 validation. +- A catalog aggregate and the data/index changes it describes commit in the + same generation. +- A stale writer cannot advance the head after losing its epoch or lease. +- A query pins object identities from one catalog generation even if a newer + generation commits while it runs. +- Dirty or pinned pages are never selected for eviction. +- A failed unique insert and every rollback leave both catalog counts and page + mappings unchanged. +- Vacuum relocation publishes old-page, new-page and index-link changes as one + generation. +- Hybrid recovery never treats an unmarked Tigris prepared generation as + acknowledged. +- Garbage collection is generation-aware and reader-aware. + +## `no_std` boundary + +DataBucket core keeps page identities, catalog record codecs, mutation plans, +page validation, cache state and backend traits available under `no_std` plus +`alloc`. WorkTable's resident core remains available without `std`, and its +spillable core should also compile without `std` when the caller supplies +storage, time and task-wakeup implementations. + +The provided local-file, HTTP, TLS, Redis, S3 and background-thread adapters +live behind `std` features or separate crates. A `no_std` target may implement +the same traits with libc, platform I/O or its own runtime. The core contracts +must not name `std::fs`, Tokio, or a specific HTTP client. + +CI continues to build WorkTable and DataBucket without default features. A +remote adapter is never pulled into that dependency graph accidentally. + +## Observability + +Expose at least these per-domain and per-table metrics: + +- resident, pinned, dirty and in-flight bytes; +- data and index cache hit ratio; +- coalesced fault count; +- fetch latency and bytes by backend; +- pages and bytes staged per generation; +- catalog and manifest bytes per generation; +- Upstash command count and REST request count; +- Tigris GET, range GET and PUT count; +- applied, locally staged and remotely committed generation lag; +- hybrid replication lag and repair count; and +- eviction scans, successful evictions and budget stalls. + +The read-only system views expose durable database facts and generation state. +High-rate cache metrics should use counters and tracing rather than mutating a +persisted system page on every read. + +## Performance and release gates + +Resident tables use their existing type, so this work should add no branch, +lock or page-cache lookup to their point-read path. That claim must be measured +against the existing full suite. + +The spillable release requires: + +- hot point reads measured against resident point reads; +- cold primary-key and secondary-index reads for local disk, Upstash, Tigris + and hybrid; +- bounded-memory scans over a table several times larger than the cache; +- repeated skewed reads proving that hot pages remain resident; +- range queries with useful index order proving early `limit` termination; +- updates, deletes, relocation and vacuum against cold pages; +- O(1) exact `count()` with most data pages absent; +- a single-row remote write showing no 4 MiB file-chunk upload; +- request, command and byte accounting for each backend; +- forced failures between every stage and commit step; +- restart, repair, another mutation and a second restart; +- checksum, missing-object and stale-writer rejection; and +- unchanged `no_std` builds. + +Benchmarks on Apple silicon should run only after competing compiler work is +quiet and use the repository's `taskpolicy` benchmark wrapper. Report the +observed core scheduling conditions with the result. Remote benchmarks report +service region, client region, page stride, cache size, batching window and +resolved dependency versions. + +The first useful performance targets are structural: + +- one cold point data lookup causes at most one data-page fetch after its index + path is resolved; +- concurrent misses for one page cause one backend fetch; +- a scan's resident memory stays within cache and bounded query overhead; +- `count()` performs no page fetch; +- a one-page mutation sends one page image per backend plus bounded metadata; + and +- the fully resident suite has no statistically meaningful regression. + +The first remote provider gate is now measured: + +- conditional generation-head creation and replacement must reject stale + expectations; +- a colocated 256 KiB range GET has a p50 no higher than 25 ms; +- a 4 MiB PUT averages no more than 250 ms and sustains at least 150 Mbit/s of + logical payload; and +- every returned page passes exact byte verification. + +Tigris and Bunny pass this gate. R2 passes the exact-read and conditional-head +checks but misses all three performance thresholds. Upstash does not pass the +synchronous request shape, although large command batches may support a later +serving tier. The committed evidence is in +`perf-benchmarks/data/fly-sin-shared-cpu-1x/2026-09-12-remote-store-gate.md`. +These are transport gates, not application latency promises. The adapter must +still pass WAL acknowledgment, restart, partial hydration and repair tests. + +## Implementation status and order + +1. Done: DataBucket storage-domain identifiers, generation plans, private + catalog write capability and a stateful page-store fixture. The core remains + `no_std` plus `alloc`. +2. Done: Tigris-compatible immutable page segments, range reads, conditional + head publication, restart restore, and a chunked generated-catalog + checkpoint. WorkTable supplies one real generated system table per database. +3. Done as a transition: the database-wide WorkTable engine runs catalog and + page accounting on its private persistence runtime and restores tables from + catalog mappings. It still discovers dirty pages by scanning the local files. +4. Next: feed exact mutations from DataBucket page persistence into generation + plans and remove the remaining local file scan. +5. Add the local bounded data-page cache, spill state machine and generated + `UserSpillableWorkTable` shape. Keep indexes resident for this milestone. +6. Move `count()`, row bytes, page counts and index-entry counts onto maintained + catalog aggregates. Validate them against full offline scans in tests. +7. Run the complete Tigris application-level crash and performance gates. +8. Implement Upstash staging, batching, head compare-and-set, recovery and + garbage collection after a region-selected service passes the gate. +9. Compose both adapters into the hybrid state machine and repair worker. +10. Add pageable lower index nodes and bounded-memory index scans. +11. Run the complete release gates and retire the per-table compatibility + engine. + +The remote adapters share the same catalog and generation fixtures so their +differences remain transport and commit-policy differences. Partial hydration +is still a release blocker: the new catalog and bounded `read_page` path are its +foundation, but generated queries do not yet evict or fault row pages. + +## External constraints to verify during implementation + +- Upstash documents REST pipelines as ordered but non-atomic, `/multi-exec` as + atomic, Lua scripting as available, and REST `WATCH` as unavailable. The + implementation therefore uses pipelines for reads and a bounded atomic + operation for the generation head: + +- Upstash service limits and billing vary by plan. Batch ceilings must be + configuration bounded and command/byte metrics must be retained: + +- Tigris, Bunny and R2 passed conditional-write and exact-read checks with the + exact Rust client. Only Tigris and Bunny passed the full performance gate. + Those operations remain release checks because provider behavior and + configuration can change: , + and + . +- Cloudflare Pipelines currently guarantees exactly-once delivery to its sink, + caps each stream at 5 MB/s and rolls R2 files no faster than every 10 + seconds. It remains an optional asynchronous ingestion investigation: + . + +## Deferred decisions + +These choices need measurements or adapter prototypes, but they do not block +the architecture: + +- exact cache policy and data/index budget split; +- exact coalescing interval around the initial 4 MiB segment target; +- whether point faults fetch one 256 KiB range or a complete small segment; +- local WAL acknowledgment policy defaults; +- retained-generation count and reader-lease duration; +- when pageable indexes become the default rather than an explicit mode. + +None of these require new WorkTable grammar. diff --git a/docs/small-tables.md b/docs/small-tables.md new file mode 100644 index 00000000..df65c25f --- /dev/null +++ b/docs/small-tables.md @@ -0,0 +1,410 @@ +# Small tables: when the index costs more than it saves + +Every `worktable!` declaration gets a primary index, unconditionally. There is +no way to say "this table is small, do not index it". This document measures +what that costs, where the lines are, and what it means for a table of forty +rows. + +All measurements taken 2026-09-11 on an Apple M4 Max (16 logical cores, 12 +performance and 4 efficiency), aarch64, `--release`. Every arm is interleaved +with the others so a machine warming up over the run cannot be charged to +whichever arm ran last, and every figure is a median rather than a mean. + +## The short version + +| question | answer | +|---|---| +| Below how many rows does a linear scan beat the index on **time**? | **32** | +| Below how many rows does the index cost more **memory** than the rows? | about **1,000** | +| What does the index cost at 40 rows? | **601 bytes per row**, against a 24-byte row | +| What does it save at 131,072 rows? | **1,333x** on lookup | +| Do any of our benchmarks measure a small indexed table? | **No. Not one.** | + +## What "the index" is, and what it is not + +Named, typed columns are free. `PointRow { id: u64, value: u64, tag: u64 }` and +`(u64, u64, u64)` are the same bytes in the same order. Tabular shape costs +nothing. + +What costs is answering **"where is the row for key K"** without looking at +every row. That is the index, that is the only thing being measured here, and +it is the only thing a small table can decline. + +A table with no index is not a table with a missing feature. It is a table that +answers a narrower question: it can hand you row number seven, and it can walk +every row, but it cannot find the row for key `K` except by looking. + +## Time: a scan wins below 32 rows + +Median nanoseconds per successful point lookup. Every key is looked up once per +pass, and passes are repeated so that small sizes still take measurable time. +The scan is `rows.iter().find(|r| r.0 == key)`; the index is `ArcticIndex`, +which is what `worktable!` defaults to. + +| rows | scan ns | arctic ns | ratio | winner | +|---:|---:|---:|---:|---| +| 4 | 1.0 | 9.8 | 0.10 | **scan** | +| 8 | 1.1 | 8.9 | 0.13 | **scan** | +| 16 | 2.3 | 10.8 | 0.22 | **scan** | +| 32 | 4.4 | 11.2 | 0.39 | **scan** | +| 64 | 9.1 | 7.6 | 1.20 | arctic | +| 128 | 19.3 | 7.4 | 2.62 | arctic | +| 256 | 35.4 | 7.5 | 4.72 | arctic | +| 512 | 65.4 | 9.6 | 6.84 | arctic | +| 1,024 | 124.1 | 9.2 | 13.44 | arctic | +| 4,096 | 485.5 | 9.6 | 50.47 | arctic | +| 16,384 | 1,956.2 | 8.6 | 226.16 | arctic | +| 65,536 | 8,155.3 | 8.9 | 918.28 | arctic | +| 131,072 | 16,413.5 | 12.3 | **1,332.98** | arctic | + +Two things to read off this. + +**The scan is fast at small sizes for a real reason.** It is sequential, +prefetchable and has no pointer chasing. At four rows it is a single cache +line. The index cannot beat that, because an ART lookup is a few dependent +loads no matter how small the tree is. + +**Arctic does not degrade.** Its lookup is flat at roughly 9 ns from 64 rows to +131,072. That is the point of a radix tree, and it means there is no upper +crossover where the scan comes back. Once the index wins it keeps winning, and +the margin grows without bound. + +## Memory: the index is never free, and is grotesque when small + +Bytes held by the index alone, per row, measured against a counting global +allocator. Arctic's own `allocated_node_bytes` is taken as a floor where it is +larger. **A row is 24 bytes**, so anything above 24 in the arctic column means +the index outweighs the data it indexes. + +| rows | HashMap B/row | BTreeMap B/row | arctic B/row | arctic vs a row | +|---:|---:|---:|---:|---:| +| 64 | 34.1 | 31.5 | **601.4** | **25.06x** | +| 1,024 | 34.0 | 34.4 | 22.2 | 0.93x | +| 16,384 | 34.0 | 34.2 | 22.2 | 0.92x | +| 131,072 | 34.0 | 34.3 | **16.1** | 0.67x | +| 1,048,576 | 34.0 | 34.3 | 16.1 | 0.67x | + +**Arctic has two crossovers and they are at different sizes.** It starts +winning on time at 32 rows. It does not stop being wasteful with memory until +somewhere around a thousand. + +At 64 rows arctic holds 601 bytes for every 24-byte row: a radix tree's fixed +node structure amortised over almost nothing. Both std maps are flat at about +34 bytes a row at every size, so at 64 rows **either std map is 18x smaller +than arctic**, and at 131,072 rows **arctic is 2.1x smaller than either**. + +That reversal is worth remembering. Arctic is the right default for a large +table on both axes at once. It is the worst of the three choices for a small +one. + +## Where the time actually goes + +200,000 rows, build and query timed separately, so the cost of having an index +is separated from the cost of using one. + +| arm | build ms | query ms | total ms | ns/lookup | +|---|---:|---:|---:|---:| +| `Vec` alone, lookup by position | 0.04 | 0.04 | 0.08 | **0.2** | +| `Vec` + `HashMap` | 3.48 | 1.21 | 4.69 | **6.0** | +| `Vec` + `BTreeMap` | 7.61 | 5.32 | 12.93 | **26.6** | +| `Vec` + `ArcticIndex` | 3.70 | 2.06 | 5.77 | **10.3** | +| `Vec`, linear scan (4,000 rows) | - | - | - | **456.9** | + +**Building is the larger half.** 3.70 ms to build against 2.06 ms to run +200,000 lookups. A table filled once and queried many times amortises that; a +table rebuilt constantly does not, and for such a table the crossover sits +higher than 32 rows. + +**The scan row is the honest alternative.** 456.9 ns per lookup at only 4,000 +rows, growing linearly, so at 200,000 rows it would be roughly 23 microseconds, +about 2,000x arctic. The index is not costing 9x. It is saving 2,000x on the +question a bare `Vec` never asks. + +**`HashMap` beats arctic on point lookups: 6.0 ns against 10.3.** Arctic is +paying for ordering, and `BTreeMap` gives the same ordering for 26.6 ns. So +arctic is 2.6x better than `BTreeMap` at equal capability and 1.7x worse than a +hash that cannot do ranges at all. **There is no hash-shaped backend in the +grammar**, and for a table that declares no range queries and no ordered scans +that is 1.7x left on the hottest path at every size above the crossover. + +### A caution about the bare-`Vec` baseline + +The first row above was measured two ways during this work, and it moved by 8x: + +- built with `(0..n).map(..).collect()`, which pre-sizes and vectorises: **0.08 ms** +- built with a `push` loop: **0.63 ms** + +An earlier note in this session quoted "the index costs 9.38x" from the second +form; the same comparison against the first reads about 72x. **Neither number +is wrong and neither is meaningful**, which is why "1:1 with a native `Vec`" is +not a target this project should quote. The stable claim is the one against the +hand-written `Vec`-plus-index pattern an application writes when it has no +table, and there the generated table is at parity: 6.4 ms against +`worktable-vec`'s `ArcticTable` at 6.5, and 13.6 for `Vec` + `BTreeMap`. + +## Could the table decide for itself? + +A prototype: hold the rows, skip the index until the table crosses a threshold, +build it once at the crossing, and branch on `len()` in `select`. Never tear it +down, so a delete that drops back under the line cannot thrash the rebuild. + +| rows | always indexed ns | adaptive ns | ratio | what the adaptive table did | +|---:|---:|---:|---:|---| +| 8 | 9.4 | **1.3** | 0.14x | scanning, no index built | +| 16 | 8.3 | **2.6** | 0.32x | scanning | +| 32 | 8.1 | **4.4** | 0.54x | scanning | +| 64 | 7.6 | 10.7 | **1.40x** | scanning, **and losing** | +| 128 | 7.5 | 7.6 | 1.00x | indexed | +| 1,024 | 9.5 | 9.6 | 1.00x | indexed | +| 16,384 | 8.5 | 8.7 | 1.02x | indexed | +| 131,072 | 12.0 | 11.3 | 0.94x | indexed | + +**The branch is free.** 1.00, 1.00, 1.02, 0.94 at the four large sizes. +Adaptivity would cost real tables nothing measurable. + +**The prototype's threshold was wrong**, and the measurement caught it. It was +set at 64 from a first reading of the crossover table, and at exactly 64 rows +the adaptive table is still scanning and is 1.40x *slower*. Arctic has already +won by then. **The line is 32.** + +## The production case: web3.trading + +Everything above was measured in the abstract. This section is a real +workload, and it moves the conclusion. + +### It is not one table, it is one table per partition + +`web3.trading-backend` declares `OrderBook` keyed `exchange_id: u8`, with a row +carrying four 24-wide depth arrays. **832 bytes a row.** It is read on every +orderbook update and written at the same rate, concurrently, at around ten +thousand a second, and it is partitioned by symbol. + +Each partition holds **one row per exchange, so two or three rows**. The +partition count is expected to be about **2,000**, possibly as low as 200, and +maybe 40 if squeezed. + +That matters because `PartitionSet` holds **a whole table per partition**. +Every partition carries its own `DataPages`, its own index, its own lock map, +its own empty-link registry and its own epoch domain. Three rows per partition +means all of that apparatus, two thousand times over. + +### Measured, at all three partition counts + +Three rows of 832 bytes per partition, memory measured against a counting +global allocator. + +| partitions | total | row payload | overhead | bytes/partition | overhead | +|---:|---:|---:|---:|---:|---:| +| 40 | 1.15 MB | 97 KB | 1.06 MB | 29,508 | 10.8x | +| 200 | 5.56 MB | 487 KB | 5.07 MB | 28,459 | 10.4x | +| 2,000 | **55.5 MB** | 4.9 MB | **50.6 MB** | 28,424 | 10.4x | + +**At 2,000 partitions the process holds 55 MB to store 4.9 MB of rows.** + +### The rows are not the cost. The partition is. + +| | bytes | +|---|---:| +| an **empty** partition, no rows at all | **28,395** | +| the same partition holding three 832-byte rows | 28,459 | +| difference | **64** | + +Three rows, 2,496 bytes of data, add sixty-four bytes. The whole 28 KB is +allocated at partition creation and is fixed. This is not the index being +wasteful at small sizes, which is what the rest of this document is about. It +is the entire table apparatus replicated per partition, and it would cost the +same if the partitions were empty. + +### This is now fixable in the declaration + +`partition_max_size: u8` beside `partition_by` generates `DenseTable` +instead of the full table. The primary key is the row's position, so the index, +the pages, the links, the free list, the lock map and the CDC all go, and a +lookup becomes a bounds check and a load. + +Measured on one declaration at two widths, 200 partitions of 23 rows each, +counting bytes the allocator was actually asked for +(`tests/dense_partition_memory.rs`): + +| shape | bytes per partition | +|---|---:| +| full table, empty | 28,404 | +| **dense, empty** | **108** | +| full table, 23 rows of an 88-byte row | 32,900 | +| **dense, same** | **3,180** | + +The empty row is the one that matters. The saving is the fixed apparatus, so it +is about 28 KB per partition whatever the rows weigh: at 2,000 symbols, roughly +56 MB. The ratio falls for wider rows only because the rows themselves grow. + +`docs/partition-models.md` sets that cost beside how six other systems partition, +and says what it buys. Briefly: it buys independent physical objects, not +freedom from contention, and an earlier version of that comparison overclaimed +in exactly that direction. + +The 28,404 here and the 28,395 above were measured independently and by +different means: the figure above came from process memory across a range of +partition counts, this one from a counting `#[global_allocator]` around a single +construction loop. They agree to nine bytes, which is the strongest thing that +can be said for either of them. + +### Time is not the problem + +| | | +|---|---:| +| `select` of one row through its partition | **48 ns** | +| at 10,000 reads/sec | 0.48 ms/s | +| at 10,000 reads **and** 10,000 writes/sec | **0.96 ms/s** | + +About **a tenth of one percent of a core**. The 48 ns is dominated by copying +an 832-byte row out, because a paged `select` returns an owned row; the index +lookup is roughly 10 ns of it. Nobody should change anything here for speed. + +### What actually helps, in order + +**1. A smaller page. Available today, no change to this crate.** + +Three 832-byte rows are 2.5 KB. The default page is 16 KB, so **84% of every +partition's page is empty**. + +| page size | bytes/partition | at 2,000 partitions | +|---:|---:|---:| +| 16,384 (default) | 28,459 | 54.3 MB | +| **4,096** | **16,171** | **30.8 MB** | + +`config: { page_size: 4096 }` on the declaration is a **43% cut**, 23 MB back, +and it is one line. This is the first thing to do. + +**2. Cache the config in the caller.** A separate table, `S3Config`, is a +single row read from the event-generation and order-placement paths at the same +rate and written perhaps once a day. Measured at 20.16 ns a read for an 11-field +row. Caching it in the strategy struct and invalidating on +`upsert_configuration` takes it to approximately zero and needs nothing from +this crate. + +**3. Direct addressing on a `u8` key.** Both `OrderBook` and `S3Config` are +keyed `u8`, which bounds them at 256 rows *in the type*, at compile time. For +such a key an index can be a 2 KB array rather than a radix tree. + +| shape | ns/read | vs today | +|---|---:|---:| +| paged `worktable!`, owned row (today) | 20.16 | 1.00x | +| `vec: true`, borrowed row | 5.66 | 0.28x | +| direct `[Option; 256]` on a `u8` key | **0.56** | **0.03x** | + +36x, decided from the declared key type, needing **no grammar and no runtime +machinery**. There is no row count at which a radix tree beats a 256-entry +array, so this is not a trade-off. + +**4. A smaller apparatus for small partitions.** After the page, roughly 12 KB +per partition remains: index, lock map, registries, epoch domain. Nothing in +the grammar lets a caller say "this partition holds three rows". This is the +largest remaining number and the least designed. + +### What this changes about the rest of this document + +The earlier sections conclude that adaptive small-table indexing is a +micro-optimisation with no workload behind it. **The first half of that is +still right and the second half is not.** There is a workload. It is just not +the index that is costing it. + +- The index at 1-3 rows is real but small here: ~10 ns of a 48 ns read, and a + fraction of the 28 KB. +- The **page** is 12 KB of the 28, fixable today with one config key. +- The **rest of the table apparatus** is the other 12 KB, and is not addressable + at all right now. + +An adaptive index would have fixed the smallest of the three. + +## What our benchmarks measure, and what they miss + +Every scale constant in both suites, against the two lines: + +| suite | arm | vs 32 rows (time) | vs ~1,000 rows (memory) | +|---|---:|---|---| +| perf-benchmarks | 200,000 | above | above | +| | 100,000 events | above | above | +| | 68,172 | above | above | +| | 4,000 documents | above | above | +| | 3,000 vocabulary | above | above | +| | 1,000 range rows | above | at the line | +| | 256 per partition | above | **below** | +| wt-benchmarks | 100,000 | above | above | +| | 20,000 | above | above | +| | 1,528 (MoE resident) | above | just above | +| | 256 | above | **below** | + +**No arm anywhere is below the time crossover.** The lowest is 256. + +**The one 256-row arm does not use an index at all.** `partition-route` routes +by arithmetic: + +```rust +rows: (0..ROWS_PER_PARTITION).map(|id| (id, id as f64)).collect(), +let at = (id % ROWS_PER_PARTITION) as usize; // a position, not a lookup +``` + +Sixty-four partitions of 256 rows, each a plain `Vec` addressed by position. So +the single place in either suite that sits in the wasteful band independently +arrived at the right answer for that size, in code written before any of this +was measured. + +**The blind spot:** nothing measures a small *indexed* table. The suites either +go large, or go small and skip the index. So if an application declares a +forty-row lookup table with `worktable!`, it pays 25x the row size in memory +and roughly 2x the lookup a scan would have done for free, and **every +benchmark stays green**. + +## What to do about it + +In order. + +1. **Add a small-table benchmark arm.** 8, 16, 32 and 64 rows, indexed, against + the scan. It will not impress, which is the point: it makes the one regime + where our defaults are wrong visible to the suite. +2. **Measure index memory more widely.** Only `moe-resident-memory-ab` measures + index bytes, and only at 1,528 rows. Nothing demonstrates arctic's 16 B/row + at 131,072, which is a real win over both std maps that we currently cannot + show. +3. ~~Find out whether anything real is under 32 rows and indexed.~~ **Found: + see the web3.trading section.** `OrderBook` is 2-3 rows per partition across + up to 2,000 partitions, read and written 10k/s. The index is the smallest + part of what it costs. +4. **Consider a hash-shaped backend** before considering adaptivity. Larger + win, at every size, and no runtime machinery. +5. **Only then consider adaptivity**, and only for the single-writer table + where it is a branch rather than a concurrency problem. + +Until any of that happens, the practical advice for a caller is one line: **a +table under about thirty rows is better as a `Vec` and a scan than as a +`worktable!`**, and nothing in the tooling will tell you so. + +## What is not measured here + +Stated so nobody quotes these numbers past what they cover. + +- **Single-threaded only.** Every figure is one thread. Contention changes the + picture for any concurrent index, and none of this says anything about it. +- **One key type.** `u64` keys throughout. A `String` key changes both the scan + (comparison cost) and the index (arctic's string path is a different shape). +- **Successful lookups only.** Every key looked up is present. A miss is a + different cost, and for a scan it is the worst case: the whole table. +- **One machine.** Apple M4 Max, aarch64. Cache sizes decide where the scan + stops being a cache-line walk, so the 32-row line is this machine's. +- **No deletes, no updates.** Build then query. A churning table amortises the + index build differently and the crossover moves. + +## Reproducing + +The probes live outside the repository, in the session scratchpad, because they +measure alternatives rather than this crate. To rebuild them, the arms are: + +- **crossover**: a `Vec<(u64,u64,u64)>` and an `ArcticIndex` over the + same rows, `iter().find()` against `get_value`, sizes from 4 to 131,072, + passes repeated to `1_000_000 / n`, 9 rounds, median. +- **memory**: a counting `GlobalAlloc`, measuring live bytes across the build of + each index alone, at 64 / 1,024 / 16,384 / 131,072 / 1,048,576 rows. +- **decomposition**: the same arms with the build and query phases timed + separately, 15 rounds. +- **adaptive**: an `Option` built once on crossing a threshold, and + `select` branching on whether it exists. diff --git a/docs/space-layer-known-issues.md b/docs/space-layer-known-issues.md index 1869a866..a25ec37d 100644 --- a/docs/space-layer-known-issues.md +++ b/docs/space-layer-known-issues.md @@ -3,8 +3,8 @@ Open defects and design gaps in `src/persistence/space/**` that are documented here rather than fixed. Durability semantics in general are covered by [persistence-durability.md](persistence-durability.md); this file records the -concrete space-layer mechanisms behind them plus issues pinned inside the -external `data_bucket = "=0.5.2"` dependency. +concrete space-layer mechanisms behind them. The release graph now uses +DataBucket 0.7; the old 0.5.2 findings below are marked as historical. ## 1. Sized batch path panics on transitional TOC identities (fixed) @@ -16,21 +16,20 @@ resolve through the aliases, and every former panic is a typed error. ## 2. Table of contents persisted before the index pages it references (fixed) -Fixed: `process_create_node`, `process_split_node`, and both batch flush -paths (sized and unsized) now write index pages first and persist the table -of contents last. A crash between the two writes leaves only an orphan page, -which reload ignores because the TOC is the sole page authority -(`parse_indexset` and the strict load audit iterate TOC entries only). The -two writes are still not atomic and still not fsynced (see issue 3): the -whole change can be lost, but a durable TOC entry can no longer point at -absent or stale page bytes. +Fixed at the call-order level: `process_create_node`, `process_split_node`, +and both batch flush paths write index pages before persisting their TOC. +Reload uses the TOC as page authority, so an unreferenced page can be ignored. +These writes are not atomic or synchronously committed. A power failure may +still lose or reorder them; issuing the page write first does not by itself +prove that a durable TOC can never reference missing or older page bytes. +See the durability contract and v3 integrity checks before planning recovery. ## 3. No fsync discipline layer-wide -Every write path in the space layer ends with `File::flush()`, which for -`tokio::fs::File` only pushes user-space buffers to the OS; nothing calls -`sync_data`/`sync_all` except the ART checkpoint writer -(`ArtFile::write_new_file`). Data pages, index pages, info pages, and the +Ordinary space-layer writes do not provide a power-loss commit. The current +portable file adapter does not turn `flush()` into a durability barrier; only +the ART checkpoint path calls +`sync_data` (`ArtFile::write_new_file`). Data pages, index pages, info pages, and the table of contents are therefore never synchronously committed: after a power loss every "completed" batch may be partially or wholly absent, and there is no ordering barrier between the TOC write and the index-page writes it @@ -52,26 +51,19 @@ it. on-disk linear scan proportional to page occupancy on top of the write itself. -## 5. Pinned `data_bucket = "=0.5.2"` defects (cannot be fixed here) +## 5. Historical DataBucket 0.5.2 findings -- `seek_to_page_start_relatively` (src/page/util.rs) computes - `(index * PAGE_SIZE as u32) as i64`: the multiply wraps in u32 once a file - passes 4 GiB, so batch parse/persist of high page ids seeks into live early - pages. The same class of bug was fixed on the WorkTable side in - `update_data_length`; the batch helpers still route through this function. -- `update_at` and `DataPage::{update_at, get_at}` compute - `link.offset + link.length` in u32 without overflow checks; adversarial or - corrupted links near `u32::MAX` wrap instead of failing the bounds check. - WorkTable's `save_data` now checks the addition before calling in. -- `TableOfContentsPage::remove_without_record` adjusts `estimated_size` as if - the removed page id were also pushed onto `empty_pages` (it adds one PageId - size back). When called without the push, the estimate over-counts by one - PageId per call. This is conservative (segments look fuller than they are, - causing at worst premature segment growth), and WorkTable's key-update path - accepts the over-count deliberately. -- `IndexTableOfContents::try_insert` keeps `data_bucket`'s historical fallback - for an entry larger than one segment: it gets its own page unchecked, and - persisting that segment overruns the page slot exactly like the update-path - overflow that is now guarded. The guard cannot be added to the insert path - without breaking the small-`DATA_LENGTH` test fixtures that rely on the - fallback; a real fix needs segment-spilling support in `data_bucket`. +The old 0.5.2 pin no longer applies. The coordinated release uses DataBucket +0.7, checked page/link bounds and v3 checksums and row directories. The former +u32 relative-seek and link-addition findings must not be quoted as current +unfixed behavior of this release. + +The WorkTable TOC wrapper still permits an entry larger than one segment to +occupy its own in-memory segment. DataBucket now rejects a persist that +exceeds the page budget rather than writing into the following page. Earlier +rejection or segment spilling would improve this path; the current behavior +is a typed persistence failure, not an oversized successful page write. + +TOC removal size estimates may remain conservative, causing earlier segment +growth. This is capacity accounting, distinct from the bounds checks that +prevent writing outside a page slot. diff --git a/docs/update-and-delete-semantics.md b/docs/update-and-delete-semantics.md new file mode 100644 index 00000000..928ab4bc --- /dev/null +++ b/docs/update-and-delete-semantics.md @@ -0,0 +1,136 @@ +# What an update and a delete actually do, per table shape + +Three table shapes live side by side in this crate and they do not agree about +what an update is or what a delete costs. The differences are deliberate and +none of them are visible from a declaration, so they are written down here. + +Established by test and measurement on 2026-09-11, not by reading. + +## The short version + +| | `vec: true` | dense partition | paged `worktable!` | +|---|---|---|---| +| writers | one (`&mut self`) | many (`&self`, one lock per partition) | many (`&self`, one lock per cell) | +| update | **clobber, in place** | **clobber, in place, per column** | in place if it fits, else `reinsert` to a new link | +| delete | ghost the slot, O(1) | clear the slot | ghost the row, reclaim out of band | +| reclaim | `compact()`, explicit | none needed | `vacuum()`, paced, background | +| row address | position in a `Vec` | position, and the key *is* the position | `Link { page_id, offset, length }` | +| index on disk | not stored, rebuilt on load | none to store | sorted pages, `attach_nodes` | + +## Update is a clobber on both Vec shapes, and there is no bit for it + +`vec: true` writes the row where it already lives: + +```rust +pub fn update(&mut self, key, edit) -> bool { edit(self.row_at_mut(at)); .. } +pub fn upsert(&mut self, row) { self.rows[at] = Some(row); } +``` + +The dense partition does the same one column at a time, through +`mem::replace`. Neither appends a new version and neither ghosts the old one. + +**There is no clobber bit and there should not be one.** A bit would exist to +*choose* between clobbering and retaining the previous version, and that choice +is meaningless on a single-writer table: nothing can be reading the row while it +changes. The paged table needs the choice because it has concurrent readers, and +that is what `reinsert` is. + +So: **ghosts come only from `delete`.** A workload that updates and never +deletes — an order book over a fixed set of exchanges, a counter table, a +config cache — produces no ghosts at all and never needs `compact()`. + +## A fixed-width row's bytes do not move + +`to_pages` lays rows into 16 KiB pages, and `rows_per_page` **searches** for how +many fit rather than computing it, so page boundaries are a property of the data +rather than of the schema. + +For a row whose columns are all fixed width, boundaries are stable: changing +every value in 5,000 rows leaves the page count and every page header +byte-identical (`a_fixed_width_rows_pages_are_byte_stable_under_update`). Row K +stays at byte `K`'s page forever. + +Add one `String`, `Vec` or `Option` of either and it stops being true — the same +test grows the file by changing a label from one byte to sixty-four. + +This is the property that decides whether a page can be written back in place, +and it is checkable from the declaration: **all-fixed-width columns means a +stable on-disk layout.** + +## What a delete costs, and why it changed + +`vec: true` used to close the hole a delete left: `Vec::remove` moved every row +above it, then every index entry above it was rewritten. At a million rows that +was **21 milliseconds per delete**. It now ghosts the slot and moves nothing, +which is **0.497 us** — 23,706x — and `compact()` does the expensive half once, +when asked. Charging a whole compaction to the 200 deletes that caused it still +leaves the new path 108x cheaper (`perf-benchmarks/benchmarks/vec-ghost-and-range.rs`). + +The cost is a slot that stays allocated: `Option` is a word per slot for a +row whose fields all use their whole range, and free for a row carrying any +spare bit pattern — one `bool` is enough. A walk over a half-ghosted table costs +exactly 2.00x per live row, and compaction gives it back. + +## Where the row lives, which is what limits everything else + +The paged table holds a `Link { page_id, offset, length }`. That is why `vacuum` +can relocate a row and repair the index, and why the paged table can have a +persistence engine at all. + +`vec: true` holds a position into a `Vec`. A position means nothing on disk, and +because `rows_per_page` packs variably the table **cannot know which page a row +will land on until it serialises**. So a dirty-page bitmap has nothing to set on +this shape: per-page persistence needs Links, and Links are the paged table. + +What `vec: true` can address is the row, because a writer is holding the slot +when it writes. And what a *partitioned* `vec: true` can address is the +partition, because the router was in the call path. + +## Consequences worth knowing before designing on top of this + +- A partitioned `vec: true` table is **build-then-freeze** today. The router + hands out `Arc` and every mutation wants `&mut self`, so a partition is + populated and then given away. There is no mutable-partition API. +- `unload()` writes the whole table, so a flush is a clobber of the file. Two + unloads concatenated do load as one table + (`two_unloads_concatenate_into_one_table`), so append is possible — but `load` + keeps the **first** of a duplicate key, so a later segment cannot supersede an + earlier row. +- Nothing here fsyncs. `unload` returns bytes; durability is entirely the + caller's, which matters to anything that wants to record what is persisted. + +## If a dirty bit is ever added, the two orderings are opposite + +Recorded before anything is built, because it is the detail that decides whether +a background flush loses writes, and it is easy to write backwards. + +A flush that clones the row and *then* marks it clean has a lost-update window: + +``` +engine clone row -> gets A +writer update to B, set dirty +engine clear dirty +``` + +B is in memory, A is on disk, and the bit says clean, so nothing will ever write +B again. Silent, permanent, one row. + +The safe order is **clear before read on the engine, set after write on the +writer** — the two are reversed relative to each other: + +``` +writer: write the value, THEN set dirty +engine: clear dirty, THEN read the value +``` + +Every interleaving of those either persists the new value or leaves the bit +dirty for the next cycle. The cost is that a row may be written twice; the +guarantee is that none is skipped. This is ordinary clear-then-read dirty +tracking, and it is written here because the obvious order is the wrong one. + +**Which shape can host a background flush at all:** `DenseRows` is an +`RwLock>>` with `update(&self, ..)`, so an engine can hold a read +lock while writers work. Plain `vec: true` cannot — every mutation is +`&mut self`, so the borrow checker forbids a concurrent reader and there is no +window to look in. A sidecar on that shape means putting the table under a lock, +which gives up the property the shape exists for. diff --git a/docs/vec-persistence-design.md b/docs/vec-persistence-design.md new file mode 100644 index 00000000..cff351c7 --- /dev/null +++ b/docs/vec-persistence-design.md @@ -0,0 +1,138 @@ +# Persisting a Vec table: what was decided, measured, and left undone + +A design conversation on 2026-09-11, recorded because most of it is conclusions +that took measurements to reach and would otherwise be re-derived wrongly. + +Nothing here is built. The measurements are real and live in +`perf-benchmarks`; the design is a proposal with its open questions named. + +## The problem + +A checkpoint costs O(table) when the change is O(1). `unload()` writes every +row, so recording ten changed rows in a million-row table costs **110 ms**. + +Every design below is an answer to one question: *what is the unit of change?* + +## What is already true, and surprised us + +**Dense partitions are already mutable through the router.** Every generated +method takes `&self` — `insert`, `upsert`, `update`, `delete`, the per-column +setters — because `DenseRows` is an `RwLock>>` inside. A +background flush can hold a read lock while writers work. This was believed to +be a blocker and is not one. + +**`vec: true` partitions are build-then-freeze.** The router hands out `Arc` +and every mutation wants `&mut self`, so a partition is populated and then given +away. That is the real gap, and the answer is to use a dense partition rather +than to add interior mutability: a `vec: true` table with a lock inside *is* a +dense table, keyed by a hash index instead of by position. + +**Update is already a clobber on both Vec shapes**, in place, no ghost, no +appended version. Ghosts come only from `delete`. See +`docs/update-and-delete-semantics.md`. A workload that updates and never deletes +— an order book over a fixed set of exchanges — produces no ghosts at all. + +**A fixed-width row's pages are byte-stable under update.** Asserted, not +assumed: changing every value in 5,000 rows leaves the page count and every page +header identical, while a `String` column growing from 1 byte to 64 grows the +file. Checkable from the declaration, and it is what decides whether a page can +be written back in place. + +**`vec: true` cannot know which page a row lands on until it serialises**, because +`rows_per_page` searches rather than computes. So a dirty-*page* bitmap has +nothing to set on that shape. Per-page persistence needs `Link { page_id, offset, +length }`, and Links are the paged table. + +## The measurements + +`perf-benchmarks/benchmarks/persisted-bit.rs`, 1,000,000 rows: + +| dirty rows | write | vs full | flip | +|---:|---:|---:|---:| +| 1,000,000 (today) | 110,359 us | 1.00x | — | +| 10 | **4 us** | **28,480x** | 0.0 us | +| 1,000 (0.1%) | 111 us | 993x | 0.4 us | +| 10,000 (1%) | 1,101 us | 100x | 4.5 us | + +The flip never exceeds 0.5% of the write it accompanies. Marking the whole table +is 3 us — a memset over words, not a walk over rows. The bitset costs **0.312%** +of the table; as a `bool` per slot it would be 8x that. + +Segment count is free on load: 1,000 segments restore in 131,846 us against one +blob's 131,308, because `load` walks pages and rebuilds the index either way. It +costs bytes — each segment rounds to a whole 16 KiB page, so a thousand waste 2%. + +Append already works: two `unload()`s concatenated load as one table +(`two_unloads_concatenate_into_one_table`). + +## The design, in order of preference + +**1. The partition is the unit.** If a table is partitioned and its partitions +are small — 2,000 symbols of 23 rows is one page each — then a checkpoint +rewrites the dirty partitions and needs one dirty bit *per partition*, held by +the router, which was already in the call path. No per-row bit, no sidecar, no +segments, no last-wins, no tombstones. One file per partition, atomic rename, +recovery that cannot be subtly wrong, and nothing accumulates. + +This is the recommended shape and it needs the least new machinery. + +**2. The row is the unit.** For one large unpartitioned `vec: true` table, the +per-row dirty bit above. It works and it is measured, but it brings segments, +which bring last-wins, tombstones, a recovery rule, and eventually compaction. + +**3. The page is the unit.** Dirty-page writeback in place, the classic answer. +Not available on `vec: true` for the addressing reason above; it is what the +paged table already does, and the paged table already has a persistence engine. + +## If a dirty bit is built, two rules + +**The orderings are opposite.** The writer sets its bit *after* writing the +value; the engine clears the bit *before* reading it. Clone-then-clear has a +lost-update window that silently loses one row forever. Written out in +`docs/update-and-delete-semantics.md`. + +**Only a shape with interior mutability can host a background flush.** +`DenseRows` can. Plain `vec: true` cannot — `&mut self` mutations mean the borrow +checker forbids a concurrent reader, so there is no window to look in. + +## What this is not + +It is not a write-ahead log and carries no per-transaction guarantee; what +survives a crash is everything as of the last checkpoint. What the numbers say +is that the *window* collapses cheaply: at 4 us for ten rows, a process can +checkpoint every millisecond for under 1% of a core. + +Known risks, all real: + +- **Nothing fsyncs.** `unload` returns bytes; durability is the caller's. A bit + flipped on `write()` returning rather than on fsync can lie, and a lying bit + loses that row permanently because nothing will clear it again. +- **Segment count on object storage is a GET per segment.** Load time is flat + locally and latency is not, so the S3 variant needs bounded segments, which + means compaction, which means a small LSM. Choose it deliberately or not at all. +- **A torn tail must truncate at the first bad segment**, not skip it. Skipping + applies a later update over a missing earlier one. + +## Queries, since it comes up + +A declared query is `Name(columns) by key` — an equality lookup. **A hash index +serves that fine; it is ranges it cannot serve.** + +| query shape | `fxhash` | ordered backends | +|---|---|---| +| `by ` | yes | yes | +| `by ` | yes | yes | +| `by ` | yes | yes | +| range, ordered scan | **no** | yes | + +The restriction today is blunter than any of that: `queries:` is refused on +`vec: true` for every backend, and the dense table accepts them only +`by ` because it has no secondary index. Enabling them on +`vec: true` is parity work rather than performance work — a generated +`update_amount_by_id` would be a named wrapper over `update(&key, |row| ..)`, +which already exists and already takes a closure. + +Mixed backends already work and are tested: a `fxhash` primary key with `arctic` +secondaries gives a table with no `range` and a working `range_by_seq` +(`a_hash_primary_key_leaves_an_arctic_secondary_ordered`). Capability is per +index, not per table. diff --git a/docs/why-worktables.typ b/docs/why-worktables.typ new file mode 100644 index 00000000..dc9ae9bf --- /dev/null +++ b/docs/why-worktables.typ @@ -0,0 +1,119 @@ +#set document(title: "Why WorkTables", author: "PathScale") +#set page(paper: "a4", margin: (x: 2.2cm, y: 2cm), numbering: "1") +#set text(font: ("Helvetica", "Arial"), size: 10pt, fill: rgb("#172833")) +#set par(leading: 0.65em) +#show heading: set block(above: 1.3em, below: 0.6em) +#show heading.where(level: 2): set text(size: 16pt) +#show raw.where(block: true): it => block(fill: rgb("#eff4f5"), inset: 10pt, radius: 3pt, width: 100%, breakable: false, text(size: 8pt, it)) +#show link: set text(fill: rgb("#176a7a")) + +#text(size: 10pt, weight: "bold", fill: rgb("#176a7a"))[PATHSCALE / WORKTABLES] +#v(0.5cm) +#text(size: 32pt, weight: "bold")[Declare the table. +Keep control of the machine.] +#v(0.3cm) +#text(size: 15pt)[Typed storage for the working data inside your Rust application.] +#v(0.4cm) + +A map is a good beginning. Then the application needs another lookup, a range, +a batch update, a memory budget and a way to reopen its state. WorkTable brings +those concerns into a declaration and generates a typed Rust API around them. + +Your data stays in process. Its indexes, storage shape and lifecycle remain +choices you can see in code and measure on your own workload. + +== From a declaration to useful operations + +```rust +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: Snapshot, + vec: true, + columns: { id: u64 primary_key using fxhash, quantity: u64, } +); + +let mut table = SnapshotWorkTable::with_capacity(1024); +table.insert(SnapshotRow { id: 7, quantity: 42 }).unwrap(); +assert_eq!(table.select(&7).unwrap().quantity, 42); +``` + +This is the compact, exclusively mutated Vec shape: borrowed reads and +synchronous writes. The paged shape offers shared access, async mutations, +secondary indexes and declared update/delete queries. Their distinct Rust +callsites preserve the difference in ownership and behavior. + +== Physical design belongs in the API + +Choose ordered indexes for ranges, or a hash index for Vec point lookups. +Use dense partitions when small bounded keys describe the data. Add columnar +replicas and clustered indexes when projection and clustered access are useful. +Tune page size and columnar chunk size against the workload. + +These choices affect memory use, write cost and access patterns. A single +declaration connects the logical table to the structures that implement it, +without hiding every physical decision behind one universal container. + +#pagebreak() +#text(size: 10pt, weight: "bold", fill: rgb("#176a7a"))[WHY WORKTABLES / MEASURED BEHAVIOR] +== A small choice can change the cost of a table + +The suite measures the generated code, alongside hand-written controls. On +one million rows, choosing `using fxhash` together with `with_capacity(rows)` +changed these per-row costs in the local full-suite run: + +#table( + columns: (1.5fr, 1fr, 1fr), inset: 8pt, + stroke: rgb("#d1dcdf"), + table.header([*Generated Vec table*], [*Build / row*], [*Point lookup*]), + [Arctic, grown on demand], [52.19 ns], [57.63 ns], + [FxHash, capacity reserved], [8.04 ns], [12.36 ns], + [Measured ratio], [*6.49× faster*], [*4.66× faster*], +) + +This is a physical-design result: both arms use the real `worktable!` macro. +The build comparison includes reservation as well as the backend change. +The hash table gives up ordered range methods and uses exclusive mutation. +The result supports choosing the right shape for a read-oriented snapshot; +it does not imply that a hash index replaces the concurrent paged table. + +The reserved hand-written hash-map control recorded 11.11 ns per lookup in +the same run. Keeping that control visible helps separate the cost of the +generated table from the cost of the underlying index. + +== Lifecycle is part of performance + +Building a table is only the beginning. Inspect live rows and accounted +storage, delete data, compact Vec slots or pace paged vacuum work around +foreground operations. For persisted tables, observe the disk footprint +as well as memory; memory reclamation does not imply file truncation. + +Persistence is opt-in for the paged shape, with local disk and an S3-backed +tier. Its completion boundaries are explicit. A successful mutation is +accepted and queued; orderly `close().await` drains and joins the engine. +The current alpha does not promise transaction journaling or fsync durability. +That makes it a fit for application-owned working state whose recovery +contract is designed deliberately. + +== Put the application back in charge + +WorkTable is useful when the hard part is maintaining indexed, typed working +data close to computation: routing state, snapshots, simulation state or +application caches. The declaration removes repetitive table plumbing while +leaving the consequential choices inspectable. + +Start with the #link("wt-user-guide.pdf")[WorkTable user guide]: declarations, +every storage shape, queries, callsites, runtimes, persistence and lifecycle +examples. It describes 1.9.0-alpha1; use the reviewed checkout until publication. + +#v(0.35cm) +#text(size: 8pt, fill: rgb("#526873"))[ + *Measurement note.* Apple M4 Max, macOS arm64, 11 September 2026. + `fx-index`: 1,000,000 rows; 100 lookups per timed burst over 2,000 bursts; + mean of three rounds after one discarded round. Values are amortized + per operation, not individual request latency. One machine and one local + full-suite run; no external database comparison is implied. + #link("https://github.com/pathscale/perf-benchmarks/blob/fix/two-ps-st3-in-one-graph/data/apple-m4-max-darwin-arm64/2026-09-11-release-full.md")[Report and provenance]. + #link("https://github.com/pathscale/perf-benchmarks/blob/fix/two-ps-st3-in-one-graph/benchmarks/fx-index.rs")[Benchmark and controls]. +] diff --git a/docs/wt-user-guide.typ b/docs/wt-user-guide.typ new file mode 100644 index 00000000..13ee00bb --- /dev/null +++ b/docs/wt-user-guide.typ @@ -0,0 +1,1212 @@ +#set document(title: "WorkTable User Guide", author: "PathScale") +#set page(paper: "a4", margin: (x: 2.2cm, y: 2.4cm), numbering: "1") +#set text(font: ("Helvetica", "Arial"), size: 10pt) +#set par(justify: true, leading: 0.62em) +#show heading: set block(above: 1.4em, below: 0.7em) +#show heading.where(level: 1): set text(size: 17pt, weight: "bold") +#show heading.where(level: 2): set text(size: 12.5pt, weight: "bold") +#show heading.where(level: 3): set text(size: 10.5pt, weight: "bold") +#show raw.where(block: true): it => block( + fill: rgb("#f4f4f2"), inset: 9pt, radius: 3pt, width: 100%, breakable: false, text(size: 8.5pt, it), +) +#show raw.where(block: false): it => box(fill: rgb("#f0f0ee"), inset: (x: 2.5pt, y: 0pt), outset: (y: 2.5pt), radius: 2pt, text(size: 9pt, it)) +#show link: set text(fill: rgb("#1a4f8a")) +#let note(title, body) = block( + fill: rgb("#fbf6e8"), stroke: (left: 2.5pt + rgb("#c8a13a")), inset: 9pt, radius: 2pt, width: 100%, + [*#title.* #body], +) + +#align(center)[ + #text(size: 26pt, weight: "bold")[WorkTable] + #v(-0.4em) + #text(size: 11pt, style: "italic")[Absolutely not a database.] + #v(0.6em) + #text(size: 9.5pt)[A user's guide to the `worktable!` macro, its queries, its indexes and its + persistence tier. Written against 1.9.0-alpha1.] +] +#v(1.2em) + += What this is + +Embedded table storage for Rust. Declare a table with a macro, get a typed struct back: +a primary key, secondary indexes, generated queries. Rows live in memory as paged, +zero-copy records. Persisting them to local disk or S3 is opt-in. + +#note("What it is not")[No transaction journal, no fsync per batch. A mutation +returning means the change was accepted and queued, not that it is on stable storage. +See #link()[Persistence].] + += Getting started + +```sh +cargo add worktable@1.9.0-alpha1 +``` + +Until this alpha is published, depend on the reviewed checkout with +`worktable = { path = "../WorkTable" }`. A plain `cargo add worktable` selects the +published release and may not include the APIs described here. + +```rust +use worktable::prelude::*; +use worktable::worktable; +``` + +Everything the macro emits resolves through `worktable::prelude`, so that one import is +the whole setup. + += Examples + +Every clause the macro accepts appears below, labelled where it is used. + +== 1. The smallest table + +```rust +worktable! ( + name: Order, // required, and must come first. CamelCase. + columns: { + id: u64 primary_key, // this table has a single-column primary key + total: u64, + }, +); + +let table = OrderWorkTable::default(); +table.insert(OrderRow { id: 1, total: 500 }).await?; // errors if the key exists +table.upsert(OrderRow { id: 1, total: 600 }).await?; // overwrites instead +let row = table.select(1).expect("just inserted"); +``` + +`name: Order` generates `OrderWorkTable`, `OrderRow`, `OrderPrimaryKey`, and for a +persisted table `OrderPersistenceEngine`. + +== 2. Column clauses + +```rust +worktable! ( + name: Account, + columns: { + id: u64 primary_key autoincrement, // the table assigns keys + email: String, // any sized type + nickname: String optional, // becomes Option + balance: i64, + }, +); +``` + +Clause order inside a column is fixed by the grammar and is not the order you might +guess: `: [primary_key [autoincrement|custom]] [optional] [columnar(..)] +[using ]`. The generator binds to `primary_key`, and `optional` follows both. + +```rust +let id = table.insert(AccountRow { + id: table.get_next_pk().into(), + email: "a@b.c".to_string(), + nickname: None, // optional column + balance: 0, +}).await?; +``` + +`custom` replaces `autoincrement` when you generate keys yourself and still want the +table to track the high-water mark. + +== 3. Composite primary keys + +```rust +worktable! ( + name: Quote, + columns: { + exchange: u32 primary_key, // both columns carry primary_key + symbol: u32 primary_key, // one generator is shared between them + price: f64, + }, +); + +let row = table.select((1_u32, 42_u32).into()).expect("present"); +``` + +A composite key keeps `worktables_index` even though the default is `arctic`, because +arctic cannot represent a tuple key. + +== 4. Secondary indexes + +```rust +worktable! ( + name: Customer, + columns: { + id: u64 primary_key, + email: String, + country: u16, + }, + indexes: { + // : [unique] [using ] + email_idx: email unique using worktables_index, // one row back + country_idx: country using arctic, // many rows back + }, +); + +let one = table.select_by_email("a@b.c".to_string()); // Option +let many = table.select_by_country(44).execute()?; // Vec +``` + +`using` is optional and defaults to `arctic`. An index over an +optional column must say `using worktables_index`; Arctic supports `String` keys, +but does not support optional keys. + +== 5. Declared queries + +```rust +worktable! ( + name: Invoice, + columns: { + id: u64 primary_key, + amount: u64, + state: u8, + }, + queries: { + update: { + AmountById(amount) by id, // () by + }, + delete: { + ById() by id, // empty parens: names no columns + }, + in_place: { + StateById(state) by id, // only `by ` is supported + }, + }, +); +``` + +CamelCase declared, snake_case generated: + +```rust +table.update_amount_by_id(AmountByIdQuery { amount: 900 }, 1).await?; // name + "Query" +table.delete_by_id(1).await?; +table.update_state_by_id_in_place(|state| *state = 2.into(), 1).await?; +``` + +`update` reads, changes and writes. `in_place` mutates without selecting first and locks +internally, so it is safe from several threads without the caller holding anything. + +== 6. Selects you do not declare + +Generated from the columns and indexes, so none of these appear in the macro: + +```rust +table.select(id) // primary key +table.select_by_email("a@b.c".to_string()) // unique index +table.select_by_country(44).execute()? // non-unique index +table.select_by_pk_range(10..=20).execute()? // range over the primary key +table.select_by_country_range(40..=50).execute()? // range over an indexed column +table.select_all().execute()? +table.select_all() + .order_on(InvoiceRowFields::Amount, Order::Desc) // generated field enum + .limit(10) + .execute()? +``` + +== 7. Columnar fields and indexes + +```rust +worktable! ( + name: Reading, + columns: { + id: u64 primary_key, // must NOT say columnar: implicit already + host_id: u64 columnar(chunk_rows(2), compression(none)), + timestamp: i64 columnar, // bare form, not the same as columnar(..) + payload: String, // row-wise only + }, + columnar_indexes: { + host_time: { // : { cluster_by: [..] } + cluster_by: [host_id, timestamp], // every field must be columnar + }, + }, + config: { + columnar_slot_id: ColumnSlotId16, // slot width, default ColumnSlotId32 + columnar_chunk_rows: 4096, // default chunk size, default 65536 + }, +); +``` + +A `columnar` column is stored column-wise as well as row-wise, so a scan over that field +reads only that field's bytes. + +== 8. Page size and row derives + +```rust +worktable! ( + name: Small, + columns: { id: u64 primary_key, v: u64 }, + config: { + page_size: 4096, // 512 minimum, 65535 max under arctic + row_derives: Clone, Debug, // bare identifiers, NOT [Clone, Debug] + }, // row_derives must be written last +); +``` + +`row_derives` reads identifiers until it meets another config key, which is why it goes +last. The `config` block takes no trailing comma after its closing brace. + +== 9. Partitioned tables + +```rust +worktable! ( + name: Book, + persist: false, + partition_by: symbol_id: u16, // : , stored per partition + partition_max_size: u8, // required: rows per partition, as an index width + columns: { + exchange_id: u8 primary_key, + bid: f64, + ask: f64, + }, +); +``` + +The partition key is stored once per partition rather than once per row, and no query +can name it. + +`partition_max_size` is required whenever `partition_by` is present, and it is a *type* +rather than a count, because it is an index width. It is how the declaration says how +many rows one partition holds: + +#table( + columns: (auto, auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*width*], [*rows per partition*], [*shape*], + [`bool`], [2], [dense], + [`u8`], [256], [dense], + [`u16`], [65,536], [dense], + [`u32`, `u64`], [unbounded in practice], [a full table per partition], +) + +There is no `unbounded` keyword: the widths run out of smallness, so `u64` is the escape +and generates exactly what a partitioned table generated before this key existed. + +It is required rather than defaulted because without it the declaration says nothing +about the shape being generated. A reader seeing `exchange_id: u8 primary_key` in a +partitioned table reads "big table with a suspiciously tiny key", when the truth is +"twenty thousand little tables, each of which only needs a byte". Two declarations +differing by 28 KB a partition would otherwise look identical. + +A count is not accepted in its place. A count is not an index width, it is not a power +of two, and it duplicates a constant that lives in the caller's code and will drift. + +=== A narrow primary key off a partition is linted + +`u8` or `bool` as the primary key of a table with no `partition_by` means a table that +can never hold more than 256 or 2 rows. That is occasionally what someone means and +usually a key that was meant to be wider, so it warns rather than failing: + +```text +warning: use of deprecated constant `_::NARROW_PRIMARY_KEY`: `id: u8` is the +primary key of an unpartitioned table, so this table can never hold more than +256 rows... +``` + +Beside `partition_by` it is silent, because there it is correct: the routing key does the +spreading and the inner key only separates the rows inside one partition. A narrow key is +what makes the dense shape below possible. + +To keep it, put `#[allow(deprecated)]` on the module holding the declaration. The warning +is a deprecation because a procedural macro cannot emit a warning any other way. + +=== What a dense width actually generates + +`bool`, `u8` and `u16` generate `DenseTable` as the partition payload instead of +the full table. It addresses rows by *position*: the primary key is the row's index, so +there is no primary index, no pages, no links, no free list, no lock map and no CDC. A +lookup is a bounds check and a load. + +Measured on one declaration at two widths, 200 partitions of 23 rows each, counting what +the allocator was asked for: + +#table( + columns: (1fr, auto), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*shape*], [*bytes per partition*], + [full table, empty], [28,404], + [*dense, empty*], [*108*], + [full table, 23 rows of an 88-byte row], [32,900], + [*dense, same*], [*3,180*], +) + +Read the empty row. The saving is fixed apparatus allocated when a partition is created, +so it is roughly 28 KB per partition whatever the rows weigh; the ratio falls for wider +rows only because the rows themselves grow. At 2,000 symbols that is about 56 MB. + +The width is a *bound, not a reservation*. The row vector grows to the highest key used, +so a `u16` partition holding three rows holds three slots, and an empty one allocates +nothing at all. + +Every method takes `&self`, because `partition_or_create` hands out an `Arc`. There is a +generated `update_` per column, which edits one field in place rather than +cloning the row out and back. Writes serialise per partition rather than per cell: the +full table needs cell-level locking because its writes are async and a query can hold a +column across an await, and nothing here is async. + +A dense width is refused, by name, for a primary key that is not a single unsigned +column, for a width the key cannot count to (`u16` beside a `u8` key declares 65,536 rows +into a partition that holds 256), and for `persist: true`, which it has no engine to +honour. + +`queries:` works. An `update` or `delete` keyed by the primary key generates the same +method name and takes the same `Query` struct as the paged table, so the call reads +the same; it is not `async` and does not return `WorkTableError`, so a call cannot move +between the shapes by accident. A query keyed by any other column is refused, because a +dense partition has no secondary index and scanning instead would turn a keyed operation +into a linear one without saying so. `in_place` is refused as a synonym: every update +here is already in place. + +Note that `memory_by_key` and `memory_total` cannot see any of this. They report +`used_bytes`, which is rows plus indexes and excludes the fixed floor by definition, so +both shapes measure the same through them. + +=== A partition here is a whole table, which is a choice + +WorkTable's partitioning is Postgres-shaped: a partition is a complete table with its own +storage, index and locks. That is a real decision with a real cost rather than an +implementation detail, and `docs/partition-models.md` compares it against PostgreSQL, +Kafka, ClickHouse, Cassandra, HBase and Snowflake, with each claim checked against those +systems' current documentation. + +One thing from it belongs here. The isolation is stronger than Postgres's, because there +is no shared lock manager to contend on: a partition is an independent generated table +behind its own handle. What it is *not* is free, which is what `partition_max_size` +exists to let you decline. + +== 9b. `vec: true`, a table with no pages + +```rust +worktable! ( + name: Lookup, + vec: true, // positional: after `version`, before `persist` + columns: { + id: u64 primary_key, + value: u64, + }, +); +``` + +The rows live in one contiguous `Vec` with an index of positions into it. It pays for +none of the paging, archived rows, lock map, change-data-capture or async surface a paged +table carries. + +It is a key rather than a second macro. `worktable_vec!` existed for a day, emitted +`VecRow` and `VecTable`, and is deleted: one macro means one `Row` and +one `WorkTable` whatever the storage is. + +=== The two are deliberately not interchangeable + +Moving a declaration between them breaks every call site, which is the safety property +rather than an omission. A swap that changed a table's concurrency and durability +guarantees while everything still compiled is the hazard worth having: + +#table( + columns: (auto, 1fr, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [], [*paged*], [*`vec: true`*], + [`insert`], [`async fn(&self, Row) -> Result`], [`fn(&mut self, Row) -> Result<(), Row>`], + [`upsert`], [`async fn(&self, Row) -> Result<(), WorkTableError>`], [`fn(&mut self, Row)`], + [`delete`], [`async fn(&self, Pk) -> Result<(), WorkTableError>`], [`fn(&mut self, &Pk) -> Option`], + [`select`], [`fn(&self, Pk) -> Option`, cloned], [`fn(&self, &Pk) -> Option<&Row>`, borrowed], +) + +A missing `.await`, `&self` against `&mut self`, an owned row against a borrowed one: the +compiler rejects the swap four different ways. + +=== What it refuses, and why + +`persist`, `runtime`, `config` and columnar fields are each refused with an +error naming what to use instead, rather than being accepted and ignored. +`partition_by` is *not* refused: see section 9, where partitioning is what makes the +`Vec` shape correct. + +Declared `queries` are supported. They use equality on a primary or secondary index, +including `fxhash`, and run synchronously through `&mut self`. An update declaration +such as `StateById(state) by id` emits +`update_state_by_id(StateByIdQuery { state: 7 }, &id) -> usize`; a delete declaration +`ByOwner() by owner` emits `delete_by_owner(&owner) -> usize`. The return value counts +affected rows. `in_place: { Status(state) by id }` emits +`update_status_in_place(|state| *state = 42, &id) -> usize` and accepts one column. +These methods belong to the table, not mutable wrappers on the shared partition set. + +Vec edits validate a cloned candidate before replacing a row. A primary or unique +secondary-key collision panics with that row and its indexes unchanged; a panicking +edit closure also leaves the stored row unchanged. Replacing an existing row through +`upsert` checks unique secondary keys first. Multi-row queries apply one row at a time +and are not transactions: earlier successful edits remain if a later edit fails. +Cloning owned fields is part of this mutation cost, including Vec `in_place` queries. + +=== Bytes and back: `unload` and `load` + +There is no persistence engine, no background task and no flush. When you want the rows +as bytes you ask for them: + +```rust +let pages: Vec = table.unload()?; // 16 KiB self-describing pages +let table = LookupWorkTable::load(&pages)?; // and back +``` + +Each 16 KiB page has a 28-byte header, an archived row batch and a 12-byte trailer: +row count at byte 16,372, row-type fingerprint at 16,376 and CRC-32 at 16,380. +The CRC covers the header, archive, padding, count and fingerprint. Page type 4 +identifies archived rows; the space id is zero. Ordinary persisted tables use a +different page type and directory, so these containers cannot be interchanged. +The reader checks page links, detects incomplete chains and rebuilds indexes from rows. +The fingerprint hashes Rust's type name; it catches obvious foreign row types, but is +neither a complete schema hash nor stable across compiler versions. Renaming a type +can invalidate a snapshot; changing fields under the same name still requires an +explicit data cutover. The codec is `worktable::vec_hydrate`. + +For an append-only table, save the number of live rows already written and append only +new rows. `first` counts live rows in insertion order, skipping ghosts: + +```rust +let first = table.len(); +let mut bytes = table.unload()?; +// Insert new rows, without updating or deleting earlier rows. +let pages_before = u32::try_from(bytes.len() / worktable::vec_hydrate::PAGE_SIZE)?; +bytes.extend_from_slice(&table.unload_appending(first, pages_before)?); +``` + +`unload_appending` reports oversized rows and page-number overflow. The previous +terminal page stays unchanged. Independent `unload()` segments can also be concatenated. +At load, the first accepted primary or unique key wins; an appended duplicate cannot +replace a row. Updates, deletes or invalidated cursors require a full snapshot. + + +=== Picking an index backend + +`using` selects the physical index, and four of the five choices are ordered trees: + +#table( + columns: (auto, 1fr, auto), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*clause*], [*stores*], [*ranges*], + [absent, or `using arctic`], [`ArcticIndex`, the default], [yes], + [`using worktables_index`], [WTI's `IndexMap`, leaf width tunable at the call site], [yes], + [`using indexset`], [a plain `BTreeMap`], [yes], + [`using congee`], [`CongeeIndex`], [yes], + [`using fxhash`], [`FxHashMap`], [*no*], +) + +`fxhash` is the odd one and is worth what it costs to explain. Measured on a million +rows against the default (`perf-benchmarks/benchmarks/fx-index.rs`): *build 4.9x, lookup +4.0x*. Nothing else in the backend list moves a number that far. + +What you give up is order. A table `using fxhash` has no `range` and no `range_by_` +methods at all — not a method that panics, not one that returns insertion order and calls +it key order; the methods are simply not generated, so asking for one is a compile error +at your call site. + +It is accepted on `vec: true` and *refused on a paged table*, with an error saying so. Two +reasons, neither negotiable: a paged table generates `select_by__range` for every +index, and a persisted index's on-disk form is sorted pages, rebuilt with `attach_nodes` +on load. A hash map has neither an order to walk nor a page form to write. + +`with_capacity` reserves an `fxhash` index along with the row vector, and that is most of +the build win: without it the same table managed 2.4x rather than 4.9x. It reserves +nothing for the tree backends, because there is nothing to gain — making allocation +completely free measures at *0.92x* for Arctic, below one, since it changes where nodes +land and sequential order is worse for a tree walked in key order. +=== Ranges + +The ordered backends expose primary-key ranges. `fxhash` has no range API: + +```rust +for row in table.range(100..200) { .. } // by primary key, in key order +for row in table.range(..).rev() { .. } // backwards +for row in table.range_by_code(&10..&20) { .. } // by a unique secondary index +``` + +This is not a sorted vector. The keys come out in order and the rows they name are +wherever insertion put them, so a long range is a walk of random accesses into the row +vector rather than a sequential read. `range_by_` is emitted for unique secondary indexes +only; a non-unique one holds a posting list per key and has no single row to yield. + +=== Deleting, and the ghosts it leaves + +`delete` empties one slot and removes its index entries. It avoids shifting all later +rows; index removal still has the selected backend's cost: + +```rust +table.delete(&7); // no vector-wide shift; returns the removed row +table.ghost_count(); // 1 +table.slots(); // unchanged +table.compact(); // reclaims the slot, renumbers the indexes +``` + +It used to close the hole with `Vec::remove`, which meant moving every row above it *and* +rewriting every index entry above it. At a million rows that cost 21 milliseconds per +delete, so two hundred deletes took four seconds. + +What you pay instead is a slot that stays allocated until you ask for it back. That is the +paged table's ghost-and-vacuum model applied to a vector, and the same judgement applies: +`ghost_count` and `slots` are there so a caller decides when compaction is worth its cost. +`compact` keeps the row vector's capacity for reuse; `shrink_to_fit` is separate, because a +table that compacts in order to keep inserting wants the capacity it already has. + +`select_all` returns an iterator rather than a `&[Row]` for this reason: with a hole in it +the live rows are not a contiguous slice, and handing one back would mean paying the +compaction the design exists to defer. + +=== Sizing it + +`with_capacity`, `capacity` and `reserve` size the row vector. `with_capacity` also +reserves the primary FxHash index when selected. Tree indexes do not reserve nodes +through this callsite. `with_capacity_and_node_size` combines row reserve with WTI +leaf width on tables that use WTI. Measure build and lookup separately before choosing +capacity or leaf width. + +== 10. Choosing a runtime + +The table declaration selects the default executor for owned async selects and the +backend identity required by named profiles. Ordinary borrowed mutations execute +where their caller polls them. Table locks remain portable; persistence uses a private +I/O pool, and engine background work follows the process runtime setting. + +```rust +runtimes! { scheduled: nagoya(shared_slot), wide: nagoya(spread), } +worktable! { + name: Orders, + runtime: nagoya(shared_slot), + columns: { id: u64 primary_key, total: u64 }, + queries: { + update runtime scheduled: { TotalById(total) by id }, + in_place runtime scheduled: { TotalById(total) by id }, + } +} +let table = Arc::new(OrdersWorkTable::default()); +table.insert(OrdersRow { id: 1, total: 10 }).await?; +table.update_total_by_id(TotalByIdQuery { total: 20 }, 1u64).await?; +table.update_total_by_id_in_place(|total| *total = 21.into(), 1u64).await?; +let rows = table.select_all() + .order_on(OrdersRowFields::Total, Order::Desc) + .limit(100).runtime(wide).execute_async().await?; +``` + +Omitting the declaration defaults to Nagoya locality. A profile must match the declared backend family; Nagoya profiles may select a different flavor at the callsite. Tokio requires the `tokio-runtime` feature and an +entered Tokio runtime. `WT_DEFAULT_RUNTIME` overrides Nagoya flavors process-wide; +`WT_RUNTIME_WORKERS` sets pool size on first use. Keep these fixed when comparing runs. + +`execute()` stays synchronous. With an explicit `.runtime(profile)`, it returns +`RuntimeRequiresAsync` instead of silently ignoring the profile. `execute_async()` +uses the table default when no profile was supplied. It materializes borrowed iterators +and `where_by` predicates on the caller before returning its future; range filters, +sorting, offset and limit execute on the worker over those owned rows. The full input +is materialized even for a small limit. This boundary releases borrowed table guards +and permits predicates that borrow local state, but adds allocation and dispatch cost. +It does not parallelize a scan or split sorting across workers. + +Runtime-annotated update, delete and in-place sections generate methods on +`Arc
`. Pass owned keys and `Send + 'static` closures; the cloned table handle +keeps storage alive. Unannotated methods retain their borrowed receivers and arguments. +Dropping a pending dispatch cancels it at the next suspension. Synchronous work already +running can finish; cancellation is not transaction rollback. Nested async dispatch +progresses even on one worker. Avoid blocking joins from a pool worker. + +Vec tables remain synchronous and reject runtime annotations. Without default features, +explicit hosted profiles are unavailable and `execute_async()` runs its owned plan inline. +The existing dependency closure still needs std; this is not a freestanding-target claim. + +== 11. A persisted table, end to end + +```rust +worktable! ( + name: Ledger, + version: 2, // optional, defaults to 1. Must precede persist. + persist: true, // generates LedgerPersistenceEngine + columns: { + id: u64 primary_key, + amount: i64, + }, +); + +let config = DiskConfig::new_with_table_name( + dir, + LedgerWorkTable::name_snake_case(), + LedgerWorkTable::version(), +); +let engine = LedgerPersistenceEngine::new(config).await?; +let table = LedgerWorkTable::load(engine).await?; // replays what is on disk + +table.upsert(LedgerRow { id: 1, amount: 42 }).await?; // queued, not durable + +table.close().await?; // the only thing that proves the queue drained +``` + +== 12. Everything at once + +The prefix is ordered. Everything after `partition_max_size` is free-order. + +```rust +worktable! ( + name: Kitchen, // 1, required + version: 3, // 2, optional + // vec: true, // 3, optional, and excludes `persist` + persist: false, // 4, optional + partition_by: shard: u16, // 5, optional + partition_max_size: u64, // 6, required with `partition_by` + runtime: nagoya(locality), // free-order from here down + columns: { + id: u64 primary_key autoincrement, + nickname: String optional, + bucket: u32 columnar, + score: i64, + }, + indexes: { + nickname_idx: nickname unique using worktables_index, + score_idx: score, + }, + columnar_indexes: { + by_bucket: { cluster_by: [bucket] }, + }, + queries: { + update: { ScoreById(score) by id }, + delete: { ById() by id }, + in_place: { ScoreById(score) by id }, + }, + config: { + page_size: 4096, + columnar_chunk_rows: 4096, + row_derives: Clone, Debug, + }, +); +``` + +#note("Writing `version` or `persist` late")[The prefix keys are positional and the +error says so rather than reporting an unexpected token. `version` after `columns` is +refused; so is `persist` or `partition_by`.] + += Index backends + +`using` names the physical structure. They differ in what they can express, not only in +speed. + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Backend*], [*When it fits*], + [`worktables_index`], [The general ordered backend, including composite and optional keys.], + [`indexset`], [Vanilla IndexSet, selectable explicitly while keeping the same disk representation.], + [`arctic`], [*The default.* Supported integer keys and `String`; packs a row link into a single `u64`. Page stride must fit its 16-bit offset and length fields.], + [`congee`], [Fixed-width integer keys. Refuses `String` and other variable-width types.], +) + +Rules: + +- Omitting `using` gives `arctic`. A composite primary key keeps `worktables_index`, + because arctic cannot represent a tuple key. +- Congee must state `persist` explicitly. Its persistence uses native checkpoint and WAL + adapters rather than the shared page format. +- Arctic supports `String`, but not optional keys. `nickname_idx: nickname unique` + over a `String optional` is rejected, and the message names the type rather than the + omission. Say `using worktables_index`. +- Arctic caps page size at 65535: it packs a link into 64 bits with 16-bit offset and + length fields. The macro refuses the combination. + += Building without default features + +Set `default-features = false` on the WorkTable dependency for the in-memory +API and generated calls with `no_std` and `alloc`. An allocator and supported +Unix or Windows OS services are required. Locks, entropy and the change-event +clock may use libc or Windows APIs without linking Rust's standard library. + +Hosted persistence, background vacuum, runtime thread creation and the +`worktable_dsl` parser re-export require `std`. Embedded schema strings and +compile-time macro parsing remain available without it: proc macros run on the +build host. `tokio-runtime`, `vanilla-index`, `s3-support`, `perf_measurements` and `wti-superslice-search` enable `std`. + +Point reads retain the fixed page directory. The no-std fallback page-list +snapshot clones an Arc under a short lock and releases the lock before visiting +rows. Standard builds retain ArcSwap. Change-event identifiers retain UUID v7 +ordering, using OS time and a shared context when std is disabled. + +CI removes Rust std from the target sysroot and compiles both the library and +an isolated consumer. A positive core/alloc control and a failing std control +verify the test environment. Host proc macros retain their normal sysroot. + +```sh +sh scripts/check-no-std.sh -p worktable --lib --no-default-features +sh scripts/check-no-std.sh --manifest-path tests/nostd-consumer/Cargo.toml +cargo test --manifest-path tests/nostd-consumer/Cargo.toml +``` + +The consumer runs generated insertion, selection, scanning and deletion, +concurrent growth, and change-event identifier checks. Its tests supply a host +allocator and executor while WorkTable remains built without std. + += Page size + +A page has two sizes and they are not interchangeable. The *stride* is what one page +occupies on disk, header included, and every file offset is computed from it. The +*payload size* is the stride less the 28-byte header. Persisted row pages also +reserve space for a live-row directory and checksum. Their row allocator budget +is smaller and depends on the minimum archived row size. Index and metadata +pages use the full payload budget. + +Set it in the `config` block: + +```rust +worktable! ( + name: Small, + columns: { id: u64 primary_key, v: u64 }, + config: { page_size: 4096 } +); +``` + +#note("Persisted tables have a floor, not a fixed size")[`page_size` works for a +persisted table, and the only rule is a 512-byte minimum: a page on disk carries a +28-byte header, so anything much smaller is mostly header. An Arctic-backed table is +also capped at 65535. In-memory tables have neither limit. + +It was refused outright until recently, because the seeks computed offsets from a +hardcoded constant while the table threaded the configured one. Every location that +decides a page size, and the three silent bugs found while making them agree, are in +`docs/page-size.md`.] + += Columnar rules + +Syntax is in #link()[Example 7]. The constraints: + +- Every field in `cluster_by` must itself declare `columnar`. +- A primary-key column must not declare `columnar`. It participates in columnar identity + implicitly, and declaring it again generates duplicate scan methods. +- `columnar_indexes` requires at least one `columnar` field. +- A columnar index must not take the name of a columnar field, which would generate two + scan methods with one name. +- `columnar_slot_id` and `columnar_chunk_rows` live in `config` because they apply to the + table. Defaults are `ColumnSlotId32` and 65,536. + += Persistence + +Persistence is implemented, not planned. Add `persist: true` and load the table through +an engine. + +```rust +let config = DiskConfig::new_with_table_name(dir, OrderWorkTable::name_snake_case(), OrderWorkTable::version()); +let engine = OrderPersistenceEngine::new(config).await?; +let table = OrderWorkTable::load(engine).await?; +``` + +With `s3-support`, the recommended hosted path groups persisted tables into one database +storage domain. DataBucket owns the generation protocol and S3 adapter; WorkTable supplies +one generated, read-only system catalog that maps every table, index and durable page. +The S3 engine still uses the disk engine as its local working copy. + +== The durability contract + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Boundary*], [*What you actually get*], + [A mutation returns], [The in-memory change was accepted and its persistence operation was queued.], + [`wait_for_ops()` returns], [The engine completed the queued operations. No fsync, no stable-storage guarantee.], + [`close()` returns], [Intake stopped, the queue drained, the engine task joined. Still no fsync guarantee.], + [Process crash or `SIGKILL`], [Acknowledged rows may be lost and the file may be torn.], + [Power loss], [No atomic-batch or stable-storage guarantee.], +) + +Call `close()` on orderly shutdown. `wait_for_ops()` is not a shutdown boundary: it does +not stop another task queueing more work, so it means nothing without writer quiescence. + +Persistence failure is terminal. An event gap, queue-analysis error, batch-apply error or +engine-task failure fails the table, and the original error goes to waiters, to `close()` +and to later mutations. + +== Loading a torn store + +A normal load audits archived rows and both primary and secondary index consistency +before exposing the table, and refuses torn state with `PersistenceLoadError` rather +than opening plausible-but-invented rows. + +`LoadMode::Recovery` exists for offline tools only. It copies individually validated +rows through a surviving index into a clean table, which must then pass a normal strict +load before anyone reads it. It is not an in-place repair and must never serve live +traffic. + +== Vacuum + +Persisted vacuum compacts the in-memory layout and keeps disk indexes consistent with +moved rows. It does not truncate `.wt.data`. Watch physical growth with +`persisted_data_file_size_bytes().await` and decide when to snapshot and rebuild. + += The filesystem + +One module, `worktable::prelude::fsx`, naming no async runtime. The file type is +`std::fs::File` behind `AllowStdIo`: blocking semantics, `futures-io` traits. + +Measured, not assumed. `tokio::fs` ran scattered updates at 12,316 rows per second +against 74,728 on `std::fs`, a factor of 6.1, with bulk insert within noise. A scattered +update is many small IOs and `tokio::fs` pays a thread-pool round trip for each. + +#note("Where to put the work")[The calls block, so a persistence engine should own a +thread rather than share a worker pool. They were never waiting on the disk anyway: 89 +voluntary context switches across 25,000 inserts.] + += Choosing a runtime + +Syntax is in #link()[Example 10]. The parenthesised name is a *flavor*: a set +of scheduler tunings, not a different scheduler. Each selected flavor owns a separate +process-lifetime pool. Reusing a flavor reuses its pool; selecting several starts several +pools. Idle spinning from those pools can interfere with measurements. Compare flavors +in separate processes and report CPU next to throughput and latency. + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Flavor*], [*What it changes*], + [`shared_slot`], [Keeps the local slot and first displaced inbox job private; shares further displaced work while that inbox is occupied.], + [`locality`], [The default. Keeps wakes local with four short spin rounds before parking. Displaced work enters a private inbox, then a local queue that can promote work to peers.], + [`spread`], [Sends every wake through the shared injector instead of keeping it local.], + [`throughput`], [`spread`, taking a larger batch from the injector at a time.], + [`wide_injector`], [`spread`, taking a larger batch still.], + [`low_latency`], [`locality` with a longer idle spin budget before parking.], +) + +#note("Measure before changing policy")[`locality` is the release baseline, using four rounds of 128 spin hints before parking. +Sparse-burst CPU measurements are part of this choice, not only peak throughput. +The earlier YCSB figures and the WorkTable workloads in `perf-benchmarks/runtime-flavours` +are different experiments. They do not establish a universally fastest flavor. Worker +count, update mix, task wake behavior and CPU consumption all matter. Keep the workload +and worker count with any quoted result.] + += Concurrency + +Indexes are lock-free with change-data-capture; a row-level `LockMap` gives ordered +access when you want it. Reads always use immutable row-version publication, including +under `default-features = false`: turning off a feature must never expose a safe API that +races deserialization against page-byte mutation. + +Point lookups use a strict backend-specific visibility contract. WorkTablesIndex pins the +structural mapping until its node is locked, so hits and misses are both definitive. + += Feature flags worth knowing + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Feature*], [*Effect*], + [`std`], [On by default. Off, hosted persistence and runtime pools are excluded. The remaining library graph is checked without Rust std on native and cross targets, while OS services may still use libc.], + [`s3-support`], [The S3 sync engine, and the HTTP stack under it.], + [`logical-index-persistence`], [Moves unique structural CDC work off the mutation path into the background worker. The page format is unchanged either way.], + [`wti-std-search`], [On by default. At the default node width, randomized lookup measured 45.4 ns here and 101.9 ns with predictable search. Four-client memory insertion also favored this policy.], +) + +The three alternative search policies (`wti-predictable-search`, `wti-hybrid-search`, +`wti-superslice-search`) are compile-time gates. Enable one, and only one, for an +unambiguous build. If feature unification turns on several, WorkTablesIndex applies a +documented precedence rather than refusing the graph. + +Predictable search favors ordered write work. In an alternating three-round table A/B, +it reduced persisted insert-and-drain time by about 12-14%, while the standard default +more than halved randomized leaf lookup and was faster for four-client in-memory +insertion. Select that tradeoff at the Cargo callsite: + +```toml +worktable = { version = "^1.9.0-alpha1", default-features = false, + features = ["std", "vanilla-index", "wti-predictable-search"] } +``` + +The `std` in `wti-std-search` names the slice-search algorithm. That search feature does +not itself require Rust std and remains available in no-default-feature builds. + += Reference coverage + +The callsite reference below covers operations beyond the declaration examples. The +executable `examples/guide_check.rs` demonstrates the public table and maintenance APIs; +the tests named there cover persistence, dense storage and columnar identity boundaries. + += Rust callsite reference + +These are existing Rust APIs, not additional grammar. A declaration chooses a storage +shape and therefore an API contract. Do not transfer a call between shapes by removing +an `await` or changing a borrowed key until the ownership and return type are understood. + +== Paged table operations + +`default()` creates an in-memory table. `insert(row).await` rejects duplicate keys and +returns the primary key. `upsert(row).await` inserts or replaces. `select(key)` returns an +owned row in `Option`, while `select_all()` and non-unique-index selects return builders. +Use `execute()` to materialize those builders. Unique secondary-index selects return an +`Option`. Primary-key and secondary-index range methods require an ordered backend. + +`insert_many(Vec).await` validates and publishes the batch atomically to readers; +`BatchInsertError` identifies the rejected row/index. Persisted success means the batch +was queued, not committed to stable storage. `delete_many(Vec).await` and +`delete_range(range).await` return deleted keys and may report a `BatchDeleteError` with +partial progress. Range deletion walks the keys present at that walk; it does not promise +to delete concurrent future inserts into the range. `reinsert(old, new).await` is the +explicit row-replacement operation; ordinary updates should use `upsert` or declared +queries so secondary indexes stay synchronized. + +With `autoincrement`, get a key from `get_next_pk()`, convert it into the row field, then +insert. `reserve_pks(count)` reserves a disjoint range for a bulk producer. Reserved keys +can be unused; allocation is not publication. `custom` lets the application supply its +generator under the generated primary-key trait contract. `name()`, +`name_snake_case()` and generated schema metadata identify a table. Persisted tables +also expose `version()` and `pk_gen_state()`. + +`row_count()` and `count()` report live rows. `used_bytes()` reports accounted row and +index storage; it is not allocator RSS. `system_info()` provides per-index and table +information. `iter_with(callback)` passes each owned row to a callback returning +`Result<(), WorkTableError>`. `iter_with_async(callback).await` accepts a callback +returning a future with the same result type. Both stop on the first error. + +== Select builders and runtime overrides + +Chain `limit(n)`, `offset(n)`, `order_on(Fields::field, Order::Asc)` or `Order::Desc`, and +`range_on(Fields::field, bounds)` before `execute()`. Generated field and range enums are +table-specific. A limit alone does not establish an order. Filtering and ordering can +require more work than the returned row count suggests. + +`runtime(profile).execute_async().await` dispatches the owned plan to a matching +profile declared by `runtimes!`. Example 10 covers materialization, Arc mutation +receivers and cancellation. `execute_async()` without a profile selects the table's +default executor. Runtime initialization reads `WT_DEFAULT_RUNTIME` and +`WT_RUNTIME_WORKERS` once. Changing environment variables afterwards does not rebuild +an already-created pool. `Runtime`, `NagoyaRt`, optional `TokioRt`, flavor marker types, +`run_on`, `run_profile` and `executor_for_flavor` are the lower-level integration surface. +They do not make storage durable or turn synchronous file access into nonblocking I/O. + +Paged custom updates require a single primary key or an indexed predicate. Paged +in-place queries require the single primary key and cannot mutate primary or secondary +indexed columns. Unsupported predicates are rejected during validation rather than +panicking in code generation. Vec query methods have their own synchronous contract. + +== Vec table operations + +`with_capacity(n)` reserves row storage and, for `fxhash`, its primary hash index. +`with_node_size(n)` selects a WorkTablesIndex leaf width where that backend is used; +it is not a reserve. `insert`, `upsert`, `update` and `delete` require `&mut self`, are +synchronous, and return the shape-specific result documented by the generated method. +Lookups borrow rows; concurrent readers can share an immutable table, but mutation needs +exclusive access. An external lock changes the measured concurrency contract. + +`new/default`, `capacity`, `reserve`, `len`, `is_empty`, `select`, `iter`, `select_all`, +`into_rows`, `range`, generated secondary-index lookups/ranges, `slots`, `ghost_count`, +`compact` and `shrink_to_fit` expose the live and physical layout. +`insert` returns `Result<(), Row>` with the rejected row; `upsert` returns `()`. +`update(&key, edit)` returns whether a row was found. `delete(&key)` returns the removed +row in `Option`; its destructor runs when the caller drops that row. It is not merely a +bit flip for rows owning heap allocations. `compact()` moves +survivors and repairs index positions. Measure deletion separately from compaction and +whole-table drop. Hash-indexed access paths do not provide ordered ranges. + +`unload()`, `unload_appending(first, pages_before)` and `load(bytes)` use the page codec +described above. This is a caller- +managed snapshot, with validation errors such as `RowTooLarge`, `NotAnArchive` and +`LoadError` and append `UnloadError`; it is not the paged persistence worker. +`vec_hydrate::{to_pages, to_pages_at, from_pages}` +and `Codec` are the lower-level codec surface. The proposed persisted dirty-bit/sidecar +design in `vec-persistence-design.md` is *not* a shipped Vec durability API. + +== Dense partitions and partition ownership + +Dense tables expose `new/default`, `insert`, `upsert`, `select(&key)`, `contains(&key)`, +`update`, `delete(&key)`, `select_all`, `row_count/len`, `is_empty`, `slots` and +`used_bytes`. Declared primary-key updates/deletes and generated scalar setters operate +synchronously. Capacity is bounded by the declared width and a failure returns +`DenseError`; dense storage does not silently fall back to paged storage or persistence. + +Generated partition sets expose `partition(key)` for an owned `Arc`, +`partition_ref(key)` for a guarded borrowed reference, and `pinned().get(key)` for +several lookups under one read epoch. Keep guards short: long-lived pins defer reclaim. +`partition_or_create` applies where a default constructor exists; +`partition_or_insert_with` accepts a factory. `keys`, `iter`, `contains`, `len` and +`is_empty` inspect the directory. A removed partition remains usable through an already- +owned `Arc`; directory removal is not revocation of those handles. + +`remove(key)` retires a directory entry. `collect()` performs bounded reclamation and +can execute destructors on its caller. `gc(&mut self)` requires exclusive access for +collection. `retired_len`, `retired_bytes`, `memory_by_key`, `memory_total` and +`rows_by_key` distinguish live directories from retired storage. The low-level +`PartitionSet` adds `get_or_create`, `for_each`, `for_each_retired` and memory-stat +methods for integrations without a generated partition wrapper. Do not benchmark only +the directory unlink and call that the total destruction cost. + +== Columnar callsites and identity + +For the `Reading` declaration in Example 7: + +```rust +let values = table.columnar_scan_host_id()?; +let refs = table.columnar_select_host_time(7, 1000)?; +let projected = table.columnar_project_timestamp(&refs)?; +let ordered_refs = table.columnar_scan_host_time()?; +``` + +Field scans return `(ColumnarRowRef, value)` pairs. Exact clustered-index selects and +clustered scans return row references; projection reads only the requested column. +`ColumnarRowRef::primary_key()` exposes the authoritative identity. Its slot, +generation and table incarnation prevent retained references from addressing a different +row after slot reuse or loading another table. Rebuilding a replica preserves references +to surviving rows. Invalidated references are omitted by projection. +They are not serializable durable IDs and not primary-key sort order. + +`columnar_slots_in_use`, `columnar_slots_high_water` and `columnar_is_dirty` expose the +replica's state; `rebuild_columnar()` reconstructs it from authoritative rows. Normal +columnar reads ensure the replica is current. The slot width bounds capacity: 8 bits +cannot represent a 20,000-row table. Reuse and failure behavior are tested in +`tests/worktable/columnar.rs`. `ColumnarColumn`, `ClusteredColumnarIndex`, +`ColumnSlotId8/16/32/64` and `ColumnCompression` are lower-level building blocks. +Only implemented compression policies are accepted; the declaration is not a promise of +an unimplemented codec. + +== Vacuum policy, scheduling and observability + +```rust +let vacuum = table.vacuum_with_pacing(VacuumPacing { + batch_pages: 64, + backoff: std::time::Duration::from_millis(2), + max_backoff: std::time::Duration::from_millis(128), + quiet_samples: 3, +}); +let before = vacuum.analyze_fragmentation(); +let stats = vacuum.vacuum().await?; +let counters = vacuum.diagnostics(); +``` + +`vacuum()` uses the default policy: 8 source pages, 2 ms initial backoff, 128 ms maximum +and three quiet observations. Positive `batch_pages` waits for quiet mutation activity +before the first and subsequent batches. Zero requests an unpaced sweep. The wait can +defer all useful sweeping under sustained writes. Completion after the foreground stops +does not establish reclamation while it was running. A successful sweep may free zero +pages because free space was reused before sweeping. + +`arm_wake(bytes)` sets the reclaimable-space wake threshold; zero disables it. +`wait_until_worth_running().await` waits for that threshold. `diagnostics()` reports +cumulative requests, batches, examined/reclaimed pages and completions. Fragmentation +metadata describes the free-space registry: its `total_pages` is the number represented +there, not necessarily every allocated page. Empty registries require special care when +forming ratios. + +`VacuumManager::new/with_config`, `register`, `diagnostic_snapshot` and +`run_vacuum_task` manage registered sweeps. Its task lifetime must be handled explicitly. +The concrete `EmptyDataVacuum` additionally offers `with_gate`, `gate` and +`with_persistence`; a `VacuumGate` can pause/resume work at batch boundaries and expose +stand-down counts. Generated callsites return `Arc`; choose their +policy at construction using `vacuum_with_pacing`, not a mutation of the trait object. + +== Persistence, recovery, S3 and versioned schemas + +`PersistenceEngine::new(config)` and generated `load(engine)` open a table. +`load_with(engine, LoadMode::Recovery)` is the explicit offline recovery boundary; +normal loads use strict validation. `wait_for_ops`, `close`, +`persisted_data_file_size_bytes` and the error contracts are described above. Stop writers +before waiting for a drain; consume the table through `close()` when shutting down. +For an `Arc
`, release all other owners and use `Arc::try_unwrap` first. + +Create one `S3Database` per database, then clone it into every table engine. This is a +callsite extension; it adds no table grammar. + +```rust +use worktable::{database_s3_persistence, DatabaseS3DiskConfig, S3Database}; + +database_s3_persistence!(OrderWorkTable); + +let database = S3Database::open_s3(domain_id, writer_epoch, s3_config)?; +let engine = OrderDatabaseS3PersistenceEngine::new(DatabaseS3DiskConfig { + disk: DiskConfig::new_with_table_name(dir, "orders", OrderWorkTable::version()), + database: database.clone(), +}).await?; +let orders = OrderWorkTable::load(engine).await?; +``` + +`data_bucket::storage::s3::S3Config` takes `bucket_name`, `endpoint`, `access_key`, +`secret_key`, optional `session_token`, `region`, optional `prefix`, and +`virtual_host_style`. Supply credentials from application configuration. The same database +handle must be shared by every table in that database. `database.catalog()` returns a +cloneable, read-only view with `system_tables`, `system_pages`, `system_indexes`, +`system_replication`, and lookup by catalog key. + +After a local batch completes, the private persistence worker scans and hashes the changed +working copy, stages only page content absent from the preceding generation, prepares the +generated catalog checkpoint, then conditionally replaces a 160-byte domain head. Table +callers only enqueue operations; catalog accounting and blocking HTTP stay on the private +one-worker persistence runtime. Adjacent dirty pages may coalesce up to 4 MiB, but the target +is not a minimum. The stateful adapter fixture measured 33,016 uploaded bytes for one page +with a small catalog and 49,544 bytes after the catalog grew beyond one page. + +Startup restores the committed generated catalog first. It validates each requested object, +rebuilds table files in a sibling staging directory, and only then renames the complete +working copy into place. Content-addressed objects that fall out of the current catalog are +retained because deleting them could race a restore; reclaim them only with an offline or +lease-aware tool. + +`s3_sync_persistence!(TableName)` and `S3DiskConfig` remain the compatibility callsite for +existing per-table manifests. New databases should use `database_s3_persistence!` and +`DatabaseS3DiskConfig`. Both paths use blocking `ureq`, require no Tokio socket reactor, +and add no local `fsync` guarantee. Exact dirty-page reporting remains a future local-work +optimization; the scan and network work are already outside the mutation caller. + +*The v3 format cutover is a storage migration.* Ordinary persisted tables now +write format 3, with a page-local directory that records every live row and a +checksum covering the payload and directory. Version 2 stores are refused +without being modified. For stores that can be regenerated, stop the application, +explicitly remove the old store, deploy the new binary and rebuild its data. +Retained data needs an explicit conversion using the old reader. Changing the +table's `version:` declaration alone does not convert disk bytes. An old binary +cannot reopen a new store. Vec snapshots use a separate codec and cannot be +opened as ordinary WorkTable space files. + +`worktable_version!` and `migration_engine!` describe explicit versioned conversions; +see `docs/migration.md` and the executable `tests/migration` fixtures for each required +trait and transformation. They do not automatically infer data migration from a changed +schema. For this 1.9 alpha, a planned rebuild/data wipe is supported by the release plan; +do not infer cross-version file compatibility from a successful same-version reopen. +`worktable::worktable_dsl` exposes parsing, checking and canonical schema emission for +tools; the TypeScript emitter is tested against that Rust source of truth. + +== Fixed-capacity atomic rows + +`AtomicKeyTable::with_capacity(n)` is a separate Rust type for counter-like rows, +not another macro grammar. `upsert(usize)` claims or finds a slot and returns `Option<&V>`; +`select`, `iter`, `len`, `is_empty` and `capacity` inspect it. `V` provides interior +mutability. There is no removal, resizing or multi-field snapshot. Two atomic fields +can be observed from different logical updates; pack mutually consistent values into one +atomic or choose a locked table. This type requires a 64-bit target. + +== Feature and capability boundaries + +The Cargo feature surface includes `std`, `vanilla-index`, `tokio-runtime`, +`s3-support`, `logical-index-persistence`, `versioned-row-publication` and the four +`wti-*-search` choices. `versioned-row-publication` is a compatibility no-op: safe row +publication is mandatory. `vanilla-index` makes upstream indexset available. +`tokio-runtime` enables that backend; it is independent of merely accepting a runtime +name in schema metadata. No-std support must be checked through a downstream consumer, +not just by disabling features on this crate while another dependency re-enables them. + +Use `cargo tree -e features` to inspect the resolved graph. Search features are additive +and have precedence; disabling defaults on one dependency does not cancel another +dependency's defaults. The release review found exactly this error in the original +four-way benchmark. Performance claims require the measured graph and an appropriate +workload, not only a compile-time feature label. + +`perf_measurements` enables operation instrumentation; measure its overhead separately +when enabling it in an application. `runtime-backends` is an empty compatibility feature; +runtime selection is available through the existing declaration and Rust callsites. + += Where to look next + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Document*], [*Covers*], + [`docs/persistence-durability.md`], [The durability contract in full, and the snapshot-restore procedure.], + [`docs/index-backend-dsl-proposal.md`], [The `using` syntax and the capability matrix per backend.], + [`docs/page-size.md`], [Every location across the four crates that decides a page size.], + [`docs/queries.md`], [The generated query surface and the custom query grammar.], + [`docs/migration.md`], [Moving a store between formats.], + [`docs/known-issues.md`], [What is known to be wrong right now.], + [`docs/small-tables.md`], [Where an index stops paying for itself, what a partition costs, and what reserving capacity is and is not worth.], + [`docs/partition-models.md`], [How WorkTable's partitioning compares with Postgres, Kafka, ClickHouse and the rest, and what the cost buys.], +) diff --git a/dsl/Cargo.toml b/dsl/Cargo.toml index e75af8f7..a674c168 100644 --- a/dsl/Cargo.toml +++ b/dsl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "worktable_dsl" -version = "1.0.0-beta.18.1" +version = "1.0.0-beta.19" edition = "2024" license = "MIT" description = "The worktable! schema language: its model and parser, readable outside the proc macro" diff --git a/dsl/src/bin/worktable-schemas.rs b/dsl/src/bin/worktable-schemas.rs index 32ed1680..fbff507f 100644 --- a/dsl/src/bin/worktable-schemas.rs +++ b/dsl/src/bin/worktable-schemas.rs @@ -44,7 +44,12 @@ fn walk(dir: &Path, entries: &mut Vec, templates: &mut usize, rejected: let name = name.to_string_lossy(); if path.is_dir() { // `target` is build output and would multiply the scan by every vendored crate. - if name == "target" || name == ".git" || name == "node_modules" { + // + // `ui` is the trybuild corpus of declarations the macro must **refuse**. Counting + // those as rejections makes the number meaningless: it reports nine failures on a + // healthy tree, and a consumer checking `rejected == 0` can never pass. The same + // skip, for the same reason, is in `dsl/tests/round_trip.rs`. + if name == "target" || name == ".git" || name == "node_modules" || name == "ui" { continue; } walk(&path, entries, templates, rejected); diff --git a/dsl/src/check.rs b/dsl/src/check.rs index c6df529f..1677db0d 100644 --- a/dsl/src/check.rs +++ b/dsl/src/check.rs @@ -184,8 +184,14 @@ pub fn check(source: &str) -> Checked { // answering "would the macro accept this?" rather than "would a // reimplementation of the macro accept this?". let diagnostics = match model_of(tokens) { - Ok((columns, queries, config, persistence)) => { - crate::validate::all(&columns, queries.as_ref(), config.as_ref(), persistence) + Ok((columns, queries, config, persistence, storage)) => { + let mut errors = crate::validate::all(&columns, queries.as_ref(), config.as_ref(), persistence); + if let Some(queries) = &queries + && let Err(error) = crate::validate::validate_query_storage(&columns, queries, storage) + { + errors.push(error); + } + errors .iter() .map(|error| Diagnostic { message: error.to_string(), @@ -215,6 +221,7 @@ type Model = ( Option, Option, crate::model::Persistence, + crate::model::Storage, ); /// The macro's own top-level dispatch, kept to the parts the rules read. @@ -222,6 +229,12 @@ fn model_of(tokens: proc_macro2::TokenStream) -> syn::Result { let mut parser = crate::Parser::new(tokens); parser.parse_name()?; parser.parse_version()?; + // `vec` sits between `version` and `persist`, and this walk skipped it, so + // every `vec: true` declaration fell through to the block loop below and + // was rejected as "Unexpected token `vec`". That made `wt-check` and + // `wt-dsl` refuse a whole storage the macro accepts, which the TypeScript + // emitter's cross-implementation test found the moment it emitted one. + let storage = parser.parse_storage()?; let persistence = parser.parse_persist()?; parser.parse_partition_by()?; @@ -229,18 +242,37 @@ fn model_of(tokens: proc_macro2::TokenStream) -> syn::Result { let mut indexes = None; let mut queries = None; let mut config = None; + let mut runtime = None; + let mut columnar_indexes = None; while let Some(ident) = parser.peek_next() { match ident.to_string().as_str() { "columns" => columns = Some(parser.parse_columns()?), "indexes" => indexes = Some(parser.parse_indexes()?), + "columnar_indexes" => columnar_indexes = Some(parser.parse_columnar_indexes()?), "queries" => queries = Some(parser.parse_queries()?), "config" => config = Some(parser.parse_configs()?), + "runtime" => { + let span = ident.span(); + if runtime.is_some() { + return Err(syn::Error::new(span, crate::parser::DUPLICATE_RUNTIME)); + } + runtime = Some(parser.parse_runtime()?); + } other => { - return Err(syn::Error::new(ident.span(), format!("Unexpected token `{other}`"))); + return Err(syn::Error::new( + ident.span(), + format!( + "Unexpected token `{other}`; expected one of `columns`, `indexes`, `columnar_indexes`, \ + `queries`, `config`, `runtime`" + ), + )); } } } + // Runtime selection does not affect the shared validation rules. + let _ = runtime; + let mut columns = columns.ok_or_else(|| { syn::Error::new( proc_macro2::Span::call_site(), @@ -250,5 +282,104 @@ fn model_of(tokens: proc_macro2::TokenStream) -> syn::Result { if let Some(indexes) = indexes { columns.indexes = indexes; } - Ok((columns, queries, config, persistence)) + if let Some(indexes) = columnar_indexes { + columns.columnar_indexes = indexes.indexes; + } + Ok((columns, queries, config, persistence, storage)) +} + +#[cfg(test)] +mod dispatch_agreement { + use super::check; + + /// Every top-level section, in one declaration. + /// + /// The order is deliberately not the canonical one: the dispatch is a + /// free-order loop, so a section is only really wired if it is reachable + /// from wherever it appears. + const EVERY_SECTION: &str = " + name: EverySection, + persist: false, + runtime: nagoya(spread), + columns: { + id: u64 primary_key, + host_id: u64 columnar(chunk_rows(2)), + qty: u64, + }, + indexes: { qty_idx: qty }, + columnar_indexes: { host_order: { cluster_by: [host_id] } }, + queries: { update: { Fill(qty) by id } }, + config: { page_size: 4096 }, + "; + + /// There are three copies of the section dispatch: the macro's own in + /// `worktable_codegen`, the schema mirror in `schema::mod`, and `model_of` + /// here. A section wired into one and not another is not a compile error + /// anywhere; it surfaces as `check` rejecting a declaration the macro + /// happily expands, which is precisely backwards for a function whose job + /// is to explain why something will not compile. + /// + /// `columnar_indexes` was missing from this loop and did exactly that. + #[test] + fn check_accepts_every_section_the_macro_does() { + let checked = check(EVERY_SECTION); + assert!( + checked.schema.is_some(), + "check failed to parse a declaration the macro accepts: {:?}", + checked.diagnostics + ); + assert!( + checked.is_acceptable(), + "check rejected a valid declaration: {:?}", + checked.diagnostics + ); + } + + /// The schema mirror has to agree with `model_of` on the same input, since + /// a caller reads the schema out of `Checked` and draws it. + #[test] + fn the_schema_mirror_accepts_every_section_too() { + let schema = crate::Schema::parse(EVERY_SECTION).expect("the schema mirror parses every section"); + assert_eq!(schema.name, "EverySection"); + assert_eq!( + schema.runtime, + crate::model::RuntimeBackend::Nagoya(crate::model::Flavor::Spread) + ); + } + + /// The rejection message names the sections that would have worked. A + /// bare "Unexpected token" is the same text a missing arm produces, so it + /// cannot tell a typo from a section somebody forgot to wire. + #[test] + fn an_unknown_section_is_told_what_was_expected() { + let checked = check("name: Bad, columns: { id: u64 primary_key }, bananas: { x: 1 }"); + let message = &checked.diagnostics[0].message; + for section in ["columns", "indexes", "columnar_indexes", "queries", "config", "runtime"] { + assert!(message.contains(section), "{section} missing from: {message}"); + } + } + + /// Every storage the macro accepts, the checker must also accept. + /// + /// `vec: true` was rejected here as "Unexpected token `vec`", because this + /// module's own walk of the positional prefix skipped `parse_storage`. The + /// macro accepted the declaration and `wt-check` and `wt-dsl` refused it, + /// so the two disagreed about what the language is. Found by the + /// TypeScript emitter's cross-implementation test, which round-trips + /// through `wt-dsl` and hit it the first time it emitted a `vec` table. + #[test] + fn the_checker_accepts_every_storage_the_macro_does() { + for declaration in [ + "name: Paged, columns: { id: u64 primary_key, v: u64 }", + "name: Vecced, vec: true, columns: { id: u64 primary_key, v: u64 }", + "name: Versioned, version: 2, vec: true, columns: { id: u64 primary_key, v: u64 }", + ] { + let checked = crate::check(declaration); + assert!( + checked.is_acceptable(), + "the checker refused `{declaration}`: {:?}", + checked.diagnostics + ); + } + } } diff --git a/dsl/src/model/column.rs b/dsl/src/model/column.rs index 688d4f62..5478837f 100644 --- a/dsl/src/model/column.rs +++ b/dsl/src/model/column.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use indexmap::IndexMap; use crate::model::index::Index; -use crate::model::{GeneratorType, IndexBackend}; +use crate::model::{ColumnSlotIdType, ColumnarFieldConfig, ColumnarIndex, GeneratorType, IndexBackend}; use proc_macro2::{Ident, TokenStream}; use quote::quote; use syn::spanned::Spanned; @@ -24,6 +24,9 @@ pub struct Columns { pub columns_map: IndexMap, pub field_positions: HashMap, pub indexes: IndexMap, + pub columnar_fields: IndexMap, + pub columnar_indexes: IndexMap, + pub column_slot_id: ColumnSlotIdType, pub primary_keys: Vec, pub primary_index_backend: IndexBackend, pub generator_type: GeneratorType, @@ -37,6 +40,7 @@ pub struct Row { pub gen_type: GeneratorType, pub optional: bool, pub index_backend: Option, + pub columnar: Option, } impl Columns { @@ -47,6 +51,7 @@ impl Columns { let mut pk = vec![]; let mut gen_type = None; let mut primary_index_backend = None; + let mut columnar_fields = IndexMap::new(); for (pos, row) in rows.into_iter().enumerate() { let type_ = &row.type_; @@ -60,6 +65,9 @@ impl Columns { }; columns_map.insert(row.name.clone(), type_); field_positions.insert(row.name.clone(), pos); + if let Some(config) = row.columnar { + columnar_fields.insert(row.name.clone(), config); + } if row.is_primary_key { if let Some(t) = gen_type { @@ -110,6 +118,9 @@ impl Columns { is_sized: sized, columns_map, indexes: Default::default(), + columnar_fields, + columnar_indexes: Default::default(), + column_slot_id: Default::default(), primary_keys: pk, primary_index_backend, generator_type: gen_type.expect("set"), diff --git a/dsl/src/model/columnar.rs b/dsl/src/model/columnar.rs new file mode 100644 index 00000000..07d4773e --- /dev/null +++ b/dsl/src/model/columnar.rs @@ -0,0 +1,63 @@ +use proc_macro2::Ident; + +pub const DEFAULT_COLUMNAR_CHUNK_ROWS: usize = 65_536; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum ColumnSlotIdType { + U8, + U16, + #[default] + U32, + U64, +} + +impl ColumnSlotIdType { + pub fn type_name(self) -> &'static str { + match self { + Self::U8 => "ColumnSlotId8", + Self::U16 => "ColumnSlotId16", + Self::U32 => "ColumnSlotId32", + Self::U64 => "ColumnSlotId64", + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum ColumnCompression { + #[default] + None, +} + +impl ColumnCompression { + pub fn name(self) -> &'static str { + match self { + Self::None => "none", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ColumnarFieldConfig { + pub chunk_rows: Option, + pub compression: ColumnCompression, +} + +impl Default for ColumnarFieldConfig { + fn default() -> Self { + Self { + chunk_rows: None, + compression: ColumnCompression::None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ColumnarIndex { + pub name: Ident, + pub cluster_by: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ColumnarIndexes { + pub indexes: indexmap::IndexMap, +} diff --git a/dsl/src/model/config.rs b/dsl/src/model/config.rs index e8d831a3..63bee3ae 100644 --- a/dsl/src/model/config.rs +++ b/dsl/src/model/config.rs @@ -1,9 +1,25 @@ use proc_macro2::{Ident, Span}; -#[derive(Debug, Default)] +use crate::model::{ColumnSlotIdType, DEFAULT_COLUMNAR_CHUNK_ROWS}; + +#[derive(Debug)] pub struct Config { pub page_size: Option, /// Span of the `page_size` value literal, kept for validation errors. pub page_size_span: Option, pub row_derives: Vec, + pub columnar_slot_id: ColumnSlotIdType, + pub columnar_chunk_rows: usize, +} + +impl Default for Config { + fn default() -> Self { + Self { + page_size: None, + page_size_span: None, + row_derives: Vec::new(), + columnar_slot_id: ColumnSlotIdType::default(), + columnar_chunk_rows: DEFAULT_COLUMNAR_CHUNK_ROWS, + } + } } diff --git a/dsl/src/model/index.rs b/dsl/src/model/index.rs index 79fca62e..059aac5c 100644 --- a/dsl/src/model/index.rs +++ b/dsl/src/model/index.rs @@ -11,6 +11,9 @@ pub enum IndexBackend { WorktablesIndex, Indexset, Congee, + /// A hash map. Point operations only: no ordered scan, no range, and no + /// persisted page form. Accepted on `vec: true` and refused elsewhere. + FxHash, #[default] Arctic, } @@ -20,11 +23,22 @@ impl IndexBackend { matches!(self, Self::Congee) } + /// Can this backend answer an ordered scan? + /// + /// Every backend but `fxhash` is a tree, so this is false for exactly one + /// of them today. It exists as a question about the backend rather than as + /// a match on `FxHash` at each call site, because the next hash-shaped + /// backend should not have to find them all. + pub fn is_ordered(self) -> bool { + !matches!(self, Self::FxHash) + } + pub fn name(self) -> &'static str { match self { Self::WorktablesIndex => "worktables_index", Self::Indexset => "indexset", Self::Congee => "congee", + Self::FxHash => "fxhash", Self::Arctic => "arctic", } } diff --git a/dsl/src/model/mod.rs b/dsl/src/model/mod.rs index fd8d9f18..4a1da7dc 100644 --- a/dsl/src/model/mod.rs +++ b/dsl/src/model/mod.rs @@ -1,4 +1,5 @@ mod column; +mod columnar; mod config; mod index; pub mod operation; @@ -6,12 +7,18 @@ mod partition; mod persistence; mod primary_key; mod queries; +mod runtime; pub use column::{Columns, Row}; +pub use columnar::{ + ColumnCompression, ColumnSlotIdType, ColumnarFieldConfig, ColumnarIndex, ColumnarIndexes, + DEFAULT_COLUMNAR_CHUNK_ROWS, +}; pub use config::Config; pub use index::{Index, IndexBackend}; pub use operation::Operation; -pub use partition::{PARTITION_KEY_TYPES, PartitionKey}; -pub use persistence::Persistence; +pub use partition::{PARTITION_KEY_TYPES, PARTITION_MAX_SIZE_TYPES, PartitionKey, PartitionMaxSize}; +pub use persistence::{Persistence, Storage}; pub use primary_key::{GeneratorType, PrimaryKey}; pub use queries::Queries; +pub use runtime::{Flavor, RuntimeBackend}; diff --git a/dsl/src/model/partition.rs b/dsl/src/model/partition.rs index f92b2e69..8935a93d 100644 --- a/dsl/src/model/partition.rs +++ b/dsl/src/model/partition.rs @@ -13,9 +13,106 @@ pub struct PartitionKey { pub name: Ident, /// Unsigned integer type of the key. pub ty: Ident, + /// How many rows a single partition holds at most, declared as an index + /// width. Required: see [`PartitionMaxSize`]. + pub max_size: PartitionMaxSize, } /// Key types routing accepts. Signed and floating types are rejected because /// a routing coordinate is an index, and a `String` key is rejected because /// hashing it costs more than every other part of the lookup combined. pub const PARTITION_KEY_TYPES: [&str; 5] = ["u8", "u16", "u32", "u64", "usize"]; + +/// How large one partition gets, written as the width of its row index. +/// +/// A **type**, not a count, because it is an index width and that is what the +/// generator needs. It matches `columnar_slot_id: ColumnSlotId16` in `config`, +/// which already means slot-width-as-a-type. +/// +/// It is required beside `partition_by` because the declaration otherwise says +/// nothing about the shape being generated. A reader seeing +/// `exchange_id: u8 primary_key` in a partitioned table reads "big table with +/// a suspiciously tiny key", when the truth is "twenty thousand little tables, +/// each of which only needs a byte". Two declarations differing by 28 KB a +/// partition would look identical. +/// +/// There is no `unbounded` keyword: the widths run out of smallness, so `u64` +/// is the escape and it generates exactly what a partitioned table generated +/// before this key existed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PartitionMaxSize { + /// Two rows. + Bool, + /// 256 rows. + U8, + /// 65,536 rows. + U16, + /// Unbounded in practice; a full generated table per partition. + U32, + /// Unbounded in practice; a full generated table per partition. + U64, +} + +/// Widths `partition_max_size` accepts, in the order they are offered in a +/// diagnostic. +pub const PARTITION_MAX_SIZE_TYPES: [&str; 5] = ["bool", "u8", "u16", "u32", "u64"]; + +impl PartitionMaxSize { + /// The width as it is written in a declaration. + pub fn type_name(self) -> &'static str { + match self { + Self::Bool => "bool", + Self::U8 => "u8", + Self::U16 => "u16", + Self::U32 => "u32", + Self::U64 => "u64", + } + } + + /// Parse the token a declaration wrote, or `None` if it is not a width. + pub fn from_type_name(name: &str) -> Option { + match name { + "bool" => Some(Self::Bool), + "u8" => Some(Self::U8), + "u16" => Some(Self::U16), + "u32" => Some(Self::U32), + "u64" => Some(Self::U64), + _ => None, + } + } + + /// Rows one partition holds, where that is a number worth having. + /// + /// `None` for `u32` and `u64`: four billion rows is not a cap anyone is + /// declaring on purpose, and the generator treats those as "no cap" rather + /// than allocating against them. + pub fn rows(self) -> Option { + match self { + Self::Bool => Some(2), + Self::U8 => Some(256), + Self::U16 => Some(65_536), + Self::U32 | Self::U64 => None, + } + } + + /// Whether this width selects the dense per-partition table. + /// + /// True exactly when [`Self::rows`] is `Some`. The two are one decision and + /// are written as one so they cannot drift apart. + pub fn is_dense(self) -> bool { + self.rows().is_some() + } + + /// The `ColumnSlotId*` type a columnar field in this partition would use. + /// + /// `bool` maps to the `u8` slot id: there is no narrower one, and a + /// two-row partition does not need one. + pub fn slot_id_type_name(self) -> &'static str { + match self { + Self::Bool | Self::U8 => "ColumnSlotId8", + Self::U16 => "ColumnSlotId16", + Self::U32 => "ColumnSlotId32", + Self::U64 => "ColumnSlotId64", + } + } +} diff --git a/dsl/src/model/persistence.rs b/dsl/src/model/persistence.rs index ade72936..27f0203d 100644 --- a/dsl/src/model/persistence.rs +++ b/dsl/src/model/persistence.rs @@ -17,3 +17,44 @@ impl Persistence { matches!(self, Self::Persisted) } } + +/// What holds the rows. +/// +/// The two are not variants of one table. A paged table is concurrent, +/// durable and async, bought with an archived row, links into pages, a +/// row-level lock map and change-data-capture. A `Vec` table is a contiguous +/// `Vec` and an index into it, single-writer and synchronous, and pays +/// for none of that. +/// +/// It is a key on `worktable!` rather than a second macro because a second +/// macro means a second set of generated names: `worktable_vec!` shipped for +/// one release emitting `VecRow` and `VecTable`, which is a +/// parallel vocabulary to learn and a redefinition error when one table is +/// declared both ways. One macro means one `Row` and one +/// `WorkTable` whatever the storage is. +/// +/// The grammar says `vec: true`, a flag, because that is the shape `persist:` +/// already has and needs no new noun. This enum exists anyway because +/// everything downstream crosses a boundary where two flags could disagree: +/// the schema is serialized, round-tripped and handed to a TypeScript +/// emitter, and serde enforces no cross-field invariant. One enum cannot say +/// two things. +/// +/// The choice is still loud rather than silent: the two tables have different +/// method signatures, so moving a declaration between them fails to compile at +/// every call site instead of quietly weakening its guarantees. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Storage { + /// Pages behind links, which is what a `worktable!` has always been. + #[default] + Paged, + /// One contiguous `Vec` and an index of positions into it. + Vec, +} + +impl Storage { + pub fn is_vec(self) -> bool { + matches!(self, Self::Vec) + } +} diff --git a/dsl/src/model/queries.rs b/dsl/src/model/queries.rs index 69425b66..46e66c08 100644 --- a/dsl/src/model/queries.rs +++ b/dsl/src/model/queries.rs @@ -8,4 +8,17 @@ pub struct Queries { pub updates: IndexMap, pub deletes: IndexMap, pub in_place: IndexMap, + /// The profile named by `update runtime :`, when the section was + /// annotated. `None` is not a default: it means the section falls back to + /// the table's `runtime`, and the table's own default only after that. + /// + /// The name is stored unresolved because the parser cannot resolve it. A + /// profile is declared by `runtimes!` somewhere else in the crate, so + /// whether it exists, and whether its backend matches the table's, is a + /// question for code generation. + pub update_runtime: Option, + /// The profile named by `delete runtime :`. See `update_runtime`. + pub delete_runtime: Option, + /// The profile named by `in_place runtime :`. See `update_runtime`. + pub in_place_runtime: Option, } diff --git a/dsl/src/model/runtime.rs b/dsl/src/model/runtime.rs new file mode 100644 index 00000000..98f07b54 --- /dev/null +++ b/dsl/src/model/runtime.rs @@ -0,0 +1,100 @@ +/// Tuning applied to the nagoya scheduler for a generated table. +/// +/// The names describe what the table does with its work rather than how the +/// scheduler is built: `Locality` keeps a task on the worker that woke it, +/// `Spread` fans it out, and `Throughput` trades wake-up latency for batching. +/// +/// **This list is a mirror.** The registry is `worktable::runtime::Flavor`, +/// which carries the stable discriminants and the tuning each name selects. +/// This enum exists so the DSL crate can parse a flavor without depending on +/// the runtime crate, and [`Flavor::ALL`] is what a drift test compares +/// against. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Flavor { + #[default] + Locality, + Spread, + Throughput, + LowLatency, + WideInjector, + SharedSlot, +} + +impl Flavor { + /// Every flavor, in the registry's discriminant order. + pub const ALL: [Flavor; 6] = [ + Self::Locality, + Self::Spread, + Self::Throughput, + Self::LowLatency, + Self::WideInjector, + Self::SharedSlot, + ]; + + /// The spelling that selects this flavor, identical to the one + /// `WT_DEFAULT_RUNTIME` takes. + pub fn name(self) -> &'static str { + match self { + Self::Locality => "locality", + Self::Spread => "spread", + Self::Throughput => "throughput", + Self::LowLatency => "low_latency", + Self::WideInjector => "wide_injector", + Self::SharedSlot => "shared_slot", + } + } + + /// The flavor a spelling selects, or `None`. + pub fn from_name(name: &str) -> Option { + Self::ALL.into_iter().find(|flavor| flavor.name() == name) + } + + /// The marker type `worktable::runtime` exports for this flavor. + pub fn type_name(self) -> &'static str { + match self { + Self::Locality => "Locality", + Self::Spread => "Spread", + Self::Throughput => "Throughput", + Self::LowLatency => "LowLatency", + Self::WideInjector => "WideInjector", + Self::SharedSlot => "SharedSlot", + } + } +} + +/// Async runtime a generated table is built against. +/// +/// Nagoya is the default, in the flavor `Flavor::default()` names, so a +/// declaration that says nothing about a runtime gets the same table as one +/// that writes `runtime: nagoya`. +/// +/// There is deliberately no variant for a backend WorkTable cannot generate +/// against. `forte`, `blocking` and `bwos` are recognised by the parser only +/// so that naming one produces a message saying so; they are a list of strings +/// there rather than variants here, because an enum variant is a promise that +/// something downstream can switch on it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum RuntimeBackend { + Nagoya(Flavor), + Tokio, +} + +impl Default for RuntimeBackend { + fn default() -> Self { + Self::Nagoya(Flavor::default()) + } +} + +impl RuntimeBackend { + /// The keyword that selects this backend, without its flavor. The flavor + /// is a separate word in the surface syntax, so it is a separate name + /// here too. + pub fn name(self) -> &'static str { + match self { + Self::Nagoya(_) => "nagoya", + Self::Tokio => "tokio", + } + } +} diff --git a/dsl/src/parser/attribute.rs b/dsl/src/parser/attribute.rs index a4c6afb4..27c21866 100644 --- a/dsl/src/parser/attribute.rs +++ b/dsl/src/parser/attribute.rs @@ -1,7 +1,9 @@ use proc_macro2::TokenTree; use syn::spanned::Spanned as _; -use crate::model::{PARTITION_KEY_TYPES, PartitionKey, Persistence}; +use crate::model::{ + PARTITION_KEY_TYPES, PARTITION_MAX_SIZE_TYPES, PartitionKey, PartitionMaxSize, Persistence, Storage, +}; use crate::parser::Parser; // TODO: Move this to separate attributes section because now it only parses persist. @@ -89,7 +91,118 @@ impl Parser { } self.try_parse_comma()?; - Ok(Some(PartitionKey { name, ty })) + + let max_size = self.parse_partition_max_size(&ident)?; + + self.try_parse_comma()?; + Ok(Some(PartitionKey { name, ty, max_size })) + } + + /// Parse the `partition_max_size: ,` that must follow `partition_by`. + /// + /// Required rather than defaulted. A default would pick one of the two + /// shapes for the author and generate the other one silently, which is the + /// implicitness this key exists to remove. `partition_by` is the span the + /// error points at, because that is the key whose presence made this one + /// mandatory. + fn parse_partition_max_size(&mut self, partition_by: &proc_macro2::Ident) -> syn::Result { + let missing = || { + syn::Error::new( + partition_by.span(), + format!( + "`partition_by` requires `partition_max_size: ,` after it, where is one of {}. \ + It is how many rows one partition holds, written as an index width: `u8` is 256 rows and \ + generates a dense partition, `u64` is the escape and generates a full table per partition", + PARTITION_MAX_SIZE_TYPES.join(", ") + ), + ) + }; + + let Some(TokenTree::Ident(ident)) = self.input_iter.peek().cloned() else { + return Err(missing()); + }; + if ident.to_string().as_str() != "partition_max_size" { + return Err(missing()); + } + let _ = self.input_iter.next(); + self.parse_colon()?; + + let width = self + .input_iter + .next() + .ok_or_else(|| syn::Error::new(self.input.span(), "Expected a `partition_max_size` width."))?; + let TokenTree::Ident(width) = width else { + return Err(syn::Error::new(width.span(), "Expected a `partition_max_size` width.")); + }; + PartitionMaxSize::from_type_name(width.to_string().as_str()).ok_or_else(|| { + syn::Error::new( + width.span(), + format!( + "`{width}` is not a `partition_max_size` width; it is an index width, so it must be one of {}. \ + A row count is not accepted: a count is not a power of two and duplicates a constant that \ + lives in the caller's code and will drift", + PARTITION_MAX_SIZE_TYPES.join(", ") + ), + ) + }) + } +} + +impl Parser { + /// Parse an optional `vec: true,` declaration. + /// + /// Positional, like `version` and `persist`, and for the strongest form of + /// their reason: this does not describe part of the table, it decides + /// which table is generated. A paged table is concurrent, durable and + /// async; a `Vec` table is single-writer and synchronous. Reading it after + /// the blocks would mean reading three screens of columns before learning + /// what they are columns of. + /// + /// # A boolean in the grammar, an enum in the model + /// + /// The author writes a flag, which is the shape `persist:` already has and + /// needs no new noun explained. What comes out is a [`Storage`], because + /// everything downstream of here crosses a boundary where two flags could + /// disagree: the canonical schema is serialized, round-tripped through + /// `to_dsl`, and handed to a TypeScript emitter, and serde will not + /// enforce a cross-field invariant for anybody. One enum cannot say two + /// things, so the illegal combination stops existing after this function + /// rather than being re-checked by each consumer. + /// + /// This briefly read `storage: vec`. The key was invented while drafting an + /// options menu rather than chosen, and a flag turned out to be the better + /// surface once the invariant could be kept without it. + pub fn parse_storage(&mut self) -> syn::Result { + let Some(ident) = self.input_iter.peek().cloned() else { + return Ok(Storage::Paged); + }; + let TokenTree::Ident(ident) = ident else { + return Err(syn::Error::new(ident.span(), "Expected field name identifier.")); + }; + if ident.to_string().as_str() != "vec" { + return Ok(Storage::Paged); + } + let _ = self.input_iter.next(); + self.parse_colon()?; + let value = self + .input_iter + .next() + .ok_or_else(|| syn::Error::new(self.input.span(), "Expected `true` or `false`."))?; + let TokenTree::Ident(value) = value else { + return Err(syn::Error::new(value.span(), "Expected `true` or `false`.")); + }; + let storage = match value.to_string().as_str() { + "true" => Storage::Vec, + "false" => Storage::Paged, + other => { + return Err(syn::Error::new( + value.span(), + format!("expected `true` or `false`, found `{other}`"), + )); + } + }; + self.try_parse_comma()?; + Ok(storage) } } @@ -98,7 +211,7 @@ mod tests { use quote::quote; use crate::Parser; - use crate::model::{PARTITION_KEY_TYPES, PartitionKey, Persistence}; + use crate::model::{PARTITION_KEY_TYPES, PARTITION_MAX_SIZE_TYPES, PartitionKey, PartitionMaxSize, Persistence}; #[test] fn test_empty() { @@ -179,13 +292,14 @@ mod tests { fn partition_by_accepts_every_unsigned_key_type() { for ty in PARTITION_KEY_TYPES { let ty_ident = syn::Ident::new(ty, proc_macro2::Span::call_site()); - let mut parser = Parser::new(quote! { partition_by: symbol_id: #ty_ident, }); + let mut parser = Parser::new(quote! { partition_by: symbol_id: #ty_ident, partition_max_size: u16, }); let key: PartitionKey = parser .parse_partition_by() .unwrap_or_else(|e| panic!("`{ty}` must be accepted: {e}")) .unwrap_or_else(|| panic!("`{ty}` parsed as absent")); assert_eq!(key.name.to_string(), "symbol_id"); assert_eq!(key.ty.to_string(), ty); + assert_eq!(key.max_size, PartitionMaxSize::U16); } } @@ -234,10 +348,79 @@ mod tests { assert!(error.contains("name"), "unexpected reason: {error}"); } + #[test] + fn partition_max_size_is_required_beside_partition_by() { + // The whole point of the key: a partitioned declaration that does not + // say how big a partition gets is refused rather than defaulted, so + // the two shapes can never look identical. + let mut parser = Parser::new(quote! { partition_by: symbol_id: u32, }); + let error = parser + .parse_partition_by() + .expect_err("a partitioned table must declare its partition size") + .to_string(); + assert!( + error.contains("partition_max_size"), + "the refusal must name the missing key: {error}" + ); + for width in PARTITION_MAX_SIZE_TYPES { + assert!(error.contains(width), "`{width}` must be offered: {error}"); + } + } + + #[test] + fn partition_max_size_accepts_every_width_and_maps_it_to_a_row_count() { + // The counts are the contract, not an implementation detail: they are + // what a reader is being told by writing the width. + for (width, rows) in [ + ("bool", Some(2u64)), + ("u8", Some(256)), + ("u16", Some(65_536)), + ("u32", None), + ("u64", None), + ] { + let width_ident = syn::Ident::new(width, proc_macro2::Span::call_site()); + let mut parser = Parser::new(quote! { partition_by: symbol_id: u32, partition_max_size: #width_ident, }); + let key = parser + .parse_partition_by() + .unwrap_or_else(|e| panic!("`{width}` must be accepted: {e}")) + .expect("declared"); + assert_eq!(key.max_size.type_name(), width); + assert_eq!(key.max_size.rows(), rows, "`{width}` row count"); + assert_eq!(key.max_size.is_dense(), rows.is_some(), "`{width}` density"); + } + } + + #[test] + fn partition_max_size_rejects_a_row_count() { + // A literal is the tempting spelling and it is refused, because a count + // is not an index width, is not a power of two, and duplicates a + // constant that lives in the caller's code and will drift. + let mut parser = Parser::new(quote! { partition_by: symbol_id: u32, partition_max_size: 256, }); + let error = parser + .parse_partition_by() + .expect_err("a literal is not a width") + .to_string(); + assert!( + error.contains("partition_max_size"), + "the refusal must name the key: {error}" + ); + } + + #[test] + fn partition_max_size_rejects_a_width_it_does_not_have() { + let mut parser = Parser::new(quote! { partition_by: symbol_id: u32, partition_max_size: u128, }); + let error = parser + .parse_partition_by() + .expect_err("u128 is not a width we generate") + .to_string(); + assert!(error.contains("u128"), "the offending width must be named: {error}"); + assert!(error.contains("u16"), "the accepted widths must be listed: {error}"); + } + #[test] fn partition_by_leaves_the_following_attribute_parseable() { // It is positional, so what follows has to still parse. - let mut parser = Parser::new(quote! { partition_by: venue: u32, persist: false, }); + let mut parser = Parser::new(quote! { partition_by: venue: u32, partition_max_size: u8, persist: false, }); let key = parser.parse_partition_by().unwrap().expect("declared"); assert_eq!(key.ty.to_string(), "u32"); assert_eq!(parser.parse_persist().unwrap(), Persistence::MemoryOnly); diff --git a/dsl/src/parser/columnar.rs b/dsl/src/parser/columnar.rs new file mode 100644 index 00000000..317302bd --- /dev/null +++ b/dsl/src/parser/columnar.rs @@ -0,0 +1,327 @@ +use std::collections::HashSet; + +use indexmap::IndexMap; +use proc_macro2::{Delimiter, Ident, TokenTree}; +use syn::spanned::Spanned as _; + +use crate::Parser; +use crate::model::{ColumnCompression, ColumnarFieldConfig, ColumnarIndex, ColumnarIndexes}; + +impl Parser { + pub(super) fn try_parse_columnar_field(&mut self) -> syn::Result> { + let Some(TokenTree::Ident(attribute)) = self.input_iter.peek() else { + return Ok(None); + }; + if attribute != "columnar" { + return Ok(None); + } + + let attribute_span = attribute.span(); + self.input_iter.next(); + let Some(TokenTree::Group(group)) = self.input_iter.peek() else { + return Ok(Some(ColumnarFieldConfig::default())); + }; + let group = group.clone(); + if group.delimiter() != Delimiter::Parenthesis { + return Err(syn::Error::new(group.span(), "expected `columnar(...)`")); + } + self.input_iter.next(); + + let mut config = ColumnarFieldConfig::default(); + let mut saw_chunk_rows = false; + let mut saw_compression = false; + let mut parser = Parser::new(group.stream()); + + while parser.has_next() { + let option = parser + .input_iter + .next() + .ok_or_else(|| syn::Error::new(attribute_span, "expected a columnar field option"))?; + let TokenTree::Ident(option) = option else { + return Err(syn::Error::new(option.span(), "expected a columnar option name")); + }; + let value = parser + .input_iter + .next() + .ok_or_else(|| syn::Error::new(option.span(), format!("expected `{option}(...)`")))?; + let TokenTree::Group(value) = value else { + return Err(syn::Error::new(value.span(), format!("expected `{option}(...)`"))); + }; + if value.delimiter() != Delimiter::Parenthesis { + return Err(syn::Error::new(value.span(), format!("expected `{option}(...)`"))); + } + + match option.to_string().as_str() { + "chunk_rows" => { + if saw_chunk_rows { + return Err(syn::Error::new(option.span(), "duplicate `chunk_rows` option")); + } + saw_chunk_rows = true; + let mut values = value.stream().into_iter(); + let Some(TokenTree::Literal(rows)) = values.next() else { + return Err(syn::Error::new(value.span(), "`chunk_rows` expects an integer")); + }; + if values.next().is_some() { + return Err(syn::Error::new(value.span(), "`chunk_rows` expects one integer")); + } + let parsed = rows + .to_string() + .replace('_', "") + .parse::() + .map_err(|_| syn::Error::new(rows.span(), "invalid `chunk_rows` integer"))?; + if parsed == 0 { + return Err(syn::Error::new(rows.span(), "`chunk_rows` must be greater than zero")); + } + config.chunk_rows = Some(parsed); + } + "compression" => { + if saw_compression { + return Err(syn::Error::new(option.span(), "duplicate `compression` option")); + } + saw_compression = true; + let mut values = value.stream().into_iter(); + let Some(TokenTree::Ident(compression)) = values.next() else { + return Err(syn::Error::new(value.span(), "`compression` expects a policy name")); + }; + if values.next().is_some() { + return Err(syn::Error::new(value.span(), "`compression` expects one policy")); + } + config.compression = match compression.to_string().as_str() { + "none" => ColumnCompression::None, + "auto" | "delta" | "rle" | "dictionary" => { + return Err(syn::Error::new( + compression.span(), + format!( + "compression({compression}) is declared but not implemented in this release; only compression(none) is currently supported" + ), + )); + } + _ => { + return Err(syn::Error::new( + compression.span(), + "unknown compression; only `none` is currently supported", + )); + } + }; + } + _ => { + return Err(syn::Error::new( + option.span(), + "unknown columnar option; expected `chunk_rows` or `compression`", + )); + } + } + parser.try_parse_comma()?; + } + + Ok(Some(config)) + } + + pub fn parse_columnar_indexes(&mut self) -> syn::Result { + let section = self + .input_iter + .next() + .ok_or_else(|| syn::Error::new(self.input.span(), "expected `columnar_indexes` section"))?; + let TokenTree::Ident(section) = section else { + return Err(syn::Error::new(section.span(), "expected `columnar_indexes`")); + }; + if section != "columnar_indexes" { + return Err(syn::Error::new(section.span(), "expected `columnar_indexes`")); + } + self.parse_colon()?; + + let body = self + .input_iter + .next() + .ok_or_else(|| syn::Error::new(section.span(), "expected `columnar_indexes: { ... }`"))?; + let TokenTree::Group(body) = body else { + return Err(syn::Error::new(body.span(), "expected `columnar_indexes: { ... }`")); + }; + if body.delimiter() != Delimiter::Brace { + return Err(syn::Error::new(body.span(), "expected braces around columnar indexes")); + } + + let mut parser = Parser::new(body.stream()); + let mut indexes = IndexMap::new(); + while parser.has_next() { + let name = parser + .input_iter + .next() + .ok_or_else(|| syn::Error::new(body.span(), "expected a columnar index name"))?; + let TokenTree::Ident(name) = name else { + return Err(syn::Error::new(name.span(), "expected a columnar index name")); + }; + parser.parse_colon()?; + + let definition = parser + .input_iter + .next() + .ok_or_else(|| syn::Error::new(name.span(), "expected a columnar index definition"))?; + let TokenTree::Group(definition) = definition else { + return Err(syn::Error::new(definition.span(), "expected `{ ... }`")); + }; + if definition.delimiter() != Delimiter::Brace { + return Err(syn::Error::new(definition.span(), "expected `{ ... }`")); + } + + let mut definition_parser = Parser::new(definition.stream()); + let mut cluster_by = None; + while definition_parser.has_next() { + let property = definition_parser + .input_iter + .next() + .ok_or_else(|| syn::Error::new(definition.span(), "expected a columnar index property"))?; + let TokenTree::Ident(property) = property else { + return Err(syn::Error::new(property.span(), "expected `cluster_by`")); + }; + definition_parser.parse_colon()?; + match property.to_string().as_str() { + "cluster_by" if cluster_by.is_none() => { + cluster_by = Some(parse_ident_list(&mut definition_parser, property.span())?) + } + "cluster_by" => { + return Err(syn::Error::new(property.span(), "duplicate columnar index property")); + } + "columns" => { + return Err(syn::Error::new( + property.span(), + "`columns` has no independent columnar-index semantics; remove it because projected fields are selected from base column stores", + )); + } + "include" => { + return Err(syn::Error::new( + property.span(), + "`include` is reserved for a future covering columnar projection and is not implemented", + )); + } + _ => { + return Err(syn::Error::new( + property.span(), + "unknown columnar index property; expected `cluster_by`", + )); + } + } + definition_parser.try_parse_comma()?; + } + + let cluster_by = cluster_by + .ok_or_else(|| syn::Error::new(name.span(), "columnar index requires `cluster_by: [...]`"))?; + if cluster_by.is_empty() { + return Err(syn::Error::new( + name.span(), + "columnar index `cluster_by` cannot be empty", + )); + } + ensure_unique(&cluster_by, "columnar index `cluster_by` contains a duplicate")?; + + if indexes.contains_key(&name) { + return Err(syn::Error::new(name.span(), "duplicate columnar index name")); + } + indexes.insert(name.clone(), ColumnarIndex { name, cluster_by }); + parser.try_parse_comma()?; + } + self.try_parse_comma()?; + Ok(ColumnarIndexes { indexes }) + } +} + +fn parse_ident_list(parser: &mut Parser, span: proc_macro2::Span) -> syn::Result> { + let list = parser + .input_iter + .next() + .ok_or_else(|| syn::Error::new(span, "expected `[field, ...]`"))?; + let TokenTree::Group(list) = list else { + return Err(syn::Error::new(list.span(), "expected `[field, ...]`")); + }; + if list.delimiter() != Delimiter::Bracket { + return Err(syn::Error::new(list.span(), "expected `[field, ...]`")); + } + let mut values = Parser::new(list.stream()); + let mut result = Vec::new(); + while values.has_next() { + let field = values + .input_iter + .next() + .ok_or_else(|| syn::Error::new(list.span(), "expected a field identifier"))?; + let TokenTree::Ident(field) = field else { + return Err(syn::Error::new(field.span(), "expected a field identifier")); + }; + result.push(field); + values.try_parse_comma()?; + } + Ok(result) +} + +fn ensure_unique(values: &[Ident], message: &str) -> syn::Result<()> { + let mut seen = HashSet::new(); + for value in values { + if !seen.insert(value.to_string()) { + return Err(syn::Error::new(value.span(), message)); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use quote::quote; + + use crate::Parser; + use crate::model::{ColumnCompression, ColumnarFieldConfig}; + + #[test] + fn parses_columnar_field_options() { + let mut parser = Parser::new(quote! { + columnar(chunk_rows(65_536), compression(none)) + }); + let config = parser.try_parse_columnar_field().unwrap().unwrap(); + assert_eq!(config.chunk_rows, Some(65_536)); + assert_eq!(config.compression, ColumnCompression::None); + } + + #[test] + fn empty_columnar_field_uses_defaults() { + let mut parser = Parser::new(quote! { columnar }); + let config = parser.try_parse_columnar_field().unwrap().unwrap(); + assert_eq!(config.chunk_rows, ColumnarFieldConfig::default().chunk_rows); + assert_eq!(config.compression, ColumnCompression::None); + } + + #[test] + fn parses_columnar_indexes() { + let mut parser = Parser::new(quote! { + columnar_indexes: { + host_time: { + cluster_by: [host_id, timestamp], + }, + }, + }); + let indexes = parser.parse_columnar_indexes().unwrap(); + let index = indexes.indexes.values().next().unwrap(); + assert_eq!( + index.cluster_by.iter().map(ToString::to_string).collect::>(), + ["host_id", "timestamp"] + ); + } + + #[test] + fn rejects_inert_columns_property() { + let mut parser = Parser::new(quote! { + columnar_indexes: { + host_time: { + columns: [host_id, timestamp], + cluster_by: [host_id, timestamp], + }, + }, + }); + let error = parser.parse_columnar_indexes().unwrap_err(); + assert!(error.to_string().contains("no independent columnar-index semantics")); + } + + #[test] + fn rejects_unimplemented_compression() { + let mut parser = Parser::new(quote! { columnar(compression(dictionary)) }); + let error = parser.try_parse_columnar_field().unwrap_err(); + assert!(error.to_string().contains("not implemented")); + } +} diff --git a/dsl/src/parser/columns.rs b/dsl/src/parser/columns.rs index e9554b41..7fb04ddf 100644 --- a/dsl/src/parser/columns.rs +++ b/dsl/src/parser/columns.rs @@ -108,8 +108,18 @@ impl Parser { false }; + let columnar = self.try_parse_columnar_field()?; + let index_backend = self.try_parse_index_backend()?; + if let Some(next) = self.input_iter.peek() + && !matches!(next, TokenTree::Punct(punct) if punct.as_char() == ',') + { + return Err(syn::Error::new( + next.span(), + "unexpected column attribute; expected attributes in `primary_key`, generator, `optional`, `columnar`, `using` order", + )); + } self.try_parse_comma()?; Ok(Row { @@ -119,6 +129,7 @@ impl Parser { gen_type, optional, index_backend, + columnar, }) } } @@ -255,7 +266,7 @@ mod tests { #[test] fn test_row_parse_no_comma() { - let row_tokens = quote! {id: i64 primary_key TreeIndex}; + let row_tokens = quote! {id: i64 primary_key}; let mut parser = Parser::new(row_tokens); let row = parser.parse_row(); @@ -324,6 +335,18 @@ mod tests { assert_eq!(row.index_backend, Some(crate::model::IndexBackend::Congee)); } + #[test] + fn test_columnar_field_parse() { + let row_tokens = quote! { + host_id: u64 columnar(chunk_rows(65_536), compression(none)), + }; + let mut parser = Parser::new(row_tokens); + let row = parser.parse_row().unwrap(); + let config = row.columnar.unwrap(); + assert_eq!(config.chunk_rows, Some(65_536)); + assert_eq!(config.compression, crate::model::ColumnCompression::None); + } + #[test] fn test_using_rejected_on_plain_column() { let tokens = quote! {columns: { diff --git a/dsl/src/parser/config.rs b/dsl/src/parser/config.rs index a3f62adc..6e5c2269 100644 --- a/dsl/src/parser/config.rs +++ b/dsl/src/parser/config.rs @@ -1,10 +1,11 @@ +use std::collections::HashSet; use std::str::FromStr; use proc_macro2::{Delimiter, TokenTree}; use syn::spanned::Spanned; use crate::Parser; -use crate::model::Config; +use crate::model::{ColumnSlotIdType, Config}; const CONFIG_FIELD_NAME: &str = "config"; @@ -49,6 +50,7 @@ impl Parser { let mut parser = Parser::new(tt); let mut config = Config::default(); parser.parse_config(&mut config)?; + self.try_parse_comma()?; // `parse_updates`, `parse_indexes` and `parse_queries` have always // consumed the comma that may follow their block. This one did not, so @@ -62,6 +64,7 @@ impl Parser { } pub fn parse_config(&mut self, config: &mut Config) -> syn::Result> { + let mut seen = HashSet::new(); while self.peek_next().is_some() { let Some(_) = self.input_iter.peek() else { return Ok(None); @@ -73,9 +76,17 @@ impl Parser { return Err(syn::Error::new(ident.span(), "Expected identifier.")); }; + let name_string = name.to_string(); + if !seen.insert(name_string.clone()) { + return Err(syn::Error::new( + name.span(), + format!("Duplicate `{name_string}` config"), + )); + } + self.parse_colon()?; - match name.to_string().as_str() { + match name_string.as_str() { "page_size" => { let value = self.input_iter.next().ok_or(syn::Error::new( self.input.span(), @@ -107,8 +118,53 @@ impl Parser { ) })?) } + "columnar_slot_id" => { + let value = self.input_iter.next().ok_or(syn::Error::new( + self.input.span(), + "Expected ColumnSlotId8, ColumnSlotId16, ColumnSlotId32, or ColumnSlotId64", + ))?; + let TokenTree::Ident(value) = value else { + return Err(syn::Error::new(value.span(), "Expected a column slot ID type.")); + }; + config.columnar_slot_id = match value.to_string().as_str() { + "ColumnSlotId8" => ColumnSlotIdType::U8, + "ColumnSlotId16" => ColumnSlotIdType::U16, + "ColumnSlotId32" => ColumnSlotIdType::U32, + "ColumnSlotId64" => ColumnSlotIdType::U64, + _ => { + return Err(syn::Error::new( + value.span(), + "Expected ColumnSlotId8, ColumnSlotId16, ColumnSlotId32, or ColumnSlotId64", + )); + } + }; + self.try_parse_comma()?; + } + "columnar_chunk_rows" => { + let value = self.input_iter.next().ok_or(syn::Error::new( + self.input.span(), + "Expected a positive columnar chunk row count", + ))?; + let TokenTree::Literal(value) = value else { + return Err(syn::Error::new(value.span(), "Expected an integer.")); + }; + let parsed = value + .to_string() + .replace('_', "") + .parse::() + .map_err(|_| syn::Error::new(value.span(), "Invalid columnar chunk row count"))?; + if parsed == 0 { + return Err(syn::Error::new( + value.span(), + "columnar_chunk_rows must be greater than zero", + )); + } + config.columnar_chunk_rows = parsed; + self.try_parse_comma()?; + } "row_derives" => { - const CONFIG_VARIANTS: [&str; 2] = ["page_size", "row_derives"]; + const CONFIG_VARIANTS: [&str; 4] = + ["page_size", "row_derives", "columnar_slot_id", "columnar_chunk_rows"]; let mut derives = vec![]; diff --git a/dsl/src/parser/index.rs b/dsl/src/parser/index.rs index a495a8cd..944e9be4 100644 --- a/dsl/src/parser/index.rs +++ b/dsl/src/parser/index.rs @@ -18,7 +18,7 @@ impl Parser { let backend = self.input_iter.next().ok_or_else(|| { syn::Error::new( using_span, - "expected an index backend after `using`: `worktables_index`, `indexset`, `congee`, or `arctic`", + "expected an index backend after `using`: `worktables_index`, `indexset`, `congee`, `fxhash`, or `arctic`", ) })?; let TokenTree::Ident(backend) = backend else { @@ -32,10 +32,11 @@ impl Parser { "worktables_index" => Ok(Some(IndexBackend::WorktablesIndex)), "indexset" => Ok(Some(IndexBackend::Indexset)), "congee" => Ok(Some(IndexBackend::Congee)), + "fxhash" => Ok(Some(IndexBackend::FxHash)), "arctic" => Ok(Some(IndexBackend::Arctic)), _ => Err(syn::Error::new( backend.span(), - "unknown index backend; expected `worktables_index`, `indexset`, `congee`, or `arctic`", + "unknown index backend; expected `worktables_index`, `indexset`, `congee`, `fxhash`, or `arctic`", )), } } diff --git a/dsl/src/parser/mod.rs b/dsl/src/parser/mod.rs index e2571e63..e3129176 100644 --- a/dsl/src/parser/mod.rs +++ b/dsl/src/parser/mod.rs @@ -1,14 +1,18 @@ mod attribute; +mod columnar; mod columns; mod config; mod index; mod name; mod punct; pub mod queries; +mod runtime; use proc_macro2::{TokenStream, TokenTree}; use std::iter::Peekable; +pub use runtime::DUPLICATE_RUNTIME; + pub struct Parser { pub input: TokenStream, pub input_iter: Peekable, diff --git a/dsl/src/parser/queries/delete.rs b/dsl/src/parser/queries/delete.rs index 3ea75d0c..42b4ac3f 100644 --- a/dsl/src/parser/queries/delete.rs +++ b/dsl/src/parser/queries/delete.rs @@ -6,7 +6,10 @@ use crate::Parser; use crate::model::Operation; impl Parser { - pub fn parse_deletes(&mut self) -> syn::Result> { + /// The `delete` block, and the profile it was annotated with. See + /// [`Parser::parse_updates`] for why the annotation rides beside the + /// operations. + pub fn parse_deletes(&mut self) -> syn::Result<(Option, IndexMap)> { let ident = self.input_iter.next().ok_or(syn::Error::new( self.input.span(), "Expected `delete` field in declaration", @@ -19,6 +22,8 @@ impl Parser { return Err(syn::Error::new(ident.span(), "Expected field name identifier.")); }; + let runtime = self.try_parse_section_runtime()?; + self.parse_colon()?; let ops = self @@ -31,7 +36,7 @@ impl Parser { // Symmetry with `parse_updates`: consume a comma after the block, // so a `delete` block is not required to be written last. self.try_parse_comma()?; - Ok(operations) + Ok((runtime, operations)) } else { Err(syn::Error::new(ops.span(), "Expected operation declarations")) } @@ -54,7 +59,7 @@ mod tests { } }; let mut parser = Parser::new(tokens); - let ops = parser.parse_updates().unwrap(); + let (_, ops) = parser.parse_updates().unwrap(); assert_eq!(ops.len(), 2); let op = ops.get(&Ident::new("TestQuery", Span::mixed_site())).unwrap(); diff --git a/dsl/src/parser/queries/in_place.rs b/dsl/src/parser/queries/in_place.rs index 2ba2a94b..c10f2f94 100644 --- a/dsl/src/parser/queries/in_place.rs +++ b/dsl/src/parser/queries/in_place.rs @@ -6,7 +6,10 @@ use crate::Parser; use crate::model::Operation; impl Parser { - pub fn parse_in_place(&mut self) -> syn::Result> { + /// The `in_place` block, and the profile it was annotated with. See + /// [`Parser::parse_updates`] for why the annotation rides beside the + /// operations. + pub fn parse_in_place(&mut self) -> syn::Result<(Option, IndexMap)> { let ident = self.input_iter.next().ok_or(syn::Error::new( self.input.span(), "Expected `in_place` field in declaration", @@ -19,6 +22,8 @@ impl Parser { return Err(syn::Error::new(ident.span(), "Expected field name identifier.")); }; + let runtime = self.try_parse_section_runtime()?; + self.parse_colon()?; let ops = self @@ -31,7 +36,7 @@ impl Parser { // Symmetry with `parse_updates`: consume a comma after the block, // so a `in_place` block is not required to be written last. self.try_parse_comma()?; - Ok(operations) + Ok((runtime, operations)) } else { Err(syn::Error::new(ops.span(), "Expected operation declarations")) } @@ -53,7 +58,7 @@ mod tests { } }; let mut parser = Parser::new(tokens); - let ops = parser.parse_in_place().unwrap(); + let (_, ops) = parser.parse_in_place().unwrap(); assert_eq!(ops.len(), 1); let op = ops.get(&Ident::new("TestQuery", Span::mixed_site())).unwrap(); diff --git a/dsl/src/parser/queries/mod.rs b/dsl/src/parser/queries/mod.rs index a74140c0..0664b6da 100644 --- a/dsl/src/parser/queries/mod.rs +++ b/dsl/src/parser/queries/mod.rs @@ -39,16 +39,19 @@ impl Parser { while let Some(ident) = parser.peek_next() { match ident.to_string().as_str() { "update" => { - let updates = parser.parse_updates()?; + let (runtime, updates) = parser.parse_updates()?; queries.updates = updates; + queries.update_runtime = runtime; } "delete" => { - let deletes = parser.parse_deletes()?; + let (runtime, deletes) = parser.parse_deletes()?; queries.deletes = deletes; + queries.delete_runtime = runtime; } "in_place" => { - let in_place = parser.parse_in_place()?; + let (runtime, in_place) = parser.parse_in_place()?; queries.in_place = in_place; + queries.in_place_runtime = runtime; } other => { return Err(syn::Error::new( @@ -67,3 +70,74 @@ impl Parser { Ok(queries) } } + +#[cfg(test)] +mod tests { + use quote::quote; + + use crate::Parser; + + #[test] + fn sections_are_unannotated_by_default() { + let tokens = quote! { + queries: { + update: { Fill(qty) by id }, + delete: { BySymbol() by symbol }, + in_place: { Bump(qty) by id }, + } + }; + let queries = Parser::new(tokens).parse_queries().unwrap(); + + assert!(queries.update_runtime.is_none()); + assert!(queries.delete_runtime.is_none()); + assert!(queries.in_place_runtime.is_none()); + } + + #[test] + fn each_section_takes_a_runtime_annotation() { + let tokens = quote! { + queries: { + update runtime fast_local: { Fill(qty) by id }, + delete runtime wide: { BySymbol() by symbol }, + in_place runtime bulk: { Bump(qty) by id }, + } + }; + let queries = Parser::new(tokens).parse_queries().unwrap(); + + assert_eq!(queries.update_runtime.unwrap(), "fast_local"); + assert_eq!(queries.delete_runtime.unwrap(), "wide"); + assert_eq!(queries.in_place_runtime.unwrap(), "bulk"); + assert_eq!(queries.updates.len(), 1); + assert_eq!(queries.deletes.len(), 1); + assert_eq!(queries.in_place.len(), 1); + } + + #[test] + fn an_annotated_section_sits_beside_an_unannotated_one() { + let tokens = quote! { + queries: { + update runtime fast_local: { Fill(qty) by id }, + in_place: { Bump(qty) by id }, + } + }; + let queries = Parser::new(tokens).parse_queries().unwrap(); + + assert_eq!(queries.update_runtime.unwrap(), "fast_local"); + assert!(queries.in_place_runtime.is_none()); + } + + #[test] + fn a_section_rejects_a_backend_in_place_of_a_profile() { + let tokens = quote! { + queries: { + update runtime nagoya: { Fill(qty) by id }, + } + }; + let error = Parser::new(tokens).parse_queries().unwrap_err().to_string(); + + assert!( + error.contains("`nagoya` is a runtime backend, not a profile name"), + "{error}" + ); + } +} diff --git a/dsl/src/parser/queries/select.rs b/dsl/src/parser/queries/select.rs index a0bfcbac..0140a10b 100644 --- a/dsl/src/parser/queries/select.rs +++ b/dsl/src/parser/queries/select.rs @@ -50,7 +50,7 @@ mod tests { } }; let mut parser = Parser::new(tokens); - let ops = parser.parse_updates().unwrap(); + let (_, ops) = parser.parse_updates().unwrap(); assert_eq!(ops.len(), 2); let op = ops.get(&Ident::new("TestQuery", Span::mixed_site())).unwrap(); diff --git a/dsl/src/parser/queries/update.rs b/dsl/src/parser/queries/update.rs index 3b5fe622..c7563a53 100644 --- a/dsl/src/parser/queries/update.rs +++ b/dsl/src/parser/queries/update.rs @@ -6,7 +6,13 @@ use crate::Parser; use crate::model::Operation; impl Parser { - pub fn parse_updates(&mut self) -> syn::Result> { + /// The `update` block, and the profile it was annotated with. + /// + /// The annotation is returned beside the operations rather than folded + /// into them because it applies to the block: every query in it runs on + /// the same runtime, and saying so once is the point of writing it at the + /// section rather than on each query. + pub fn parse_updates(&mut self) -> syn::Result<(Option, IndexMap)> { let ident = self.input_iter.next().ok_or(syn::Error::new( self.input.span(), "Expected `update` field in declaration", @@ -19,6 +25,8 @@ impl Parser { return Err(syn::Error::new(ident.span(), "Expected field name identifier.")); }; + let runtime = self.try_parse_section_runtime()?; + self.parse_colon()?; let ops = self @@ -27,9 +35,9 @@ impl Parser { .ok_or(syn::Error::new(self.input.span(), "Expected operation declarations"))?; if let TokenTree::Group(ops) = ops { let mut parser = Parser::new(ops.stream()); - let ops = parser.parse_operations(); + let ops = parser.parse_operations()?; self.try_parse_comma()?; - ops + Ok((runtime, ops)) } else { Err(syn::Error::new(ops.span(), "Expected operation declarations")) } @@ -52,7 +60,7 @@ mod tests { } }; let mut parser = Parser::new(tokens); - let ops = parser.parse_updates().unwrap(); + let (_, ops) = parser.parse_updates().unwrap(); assert_eq!(ops.len(), 2); let op = ops.get(&Ident::new("TestQuery", Span::mixed_site())).unwrap(); diff --git a/dsl/src/parser/runtime.rs b/dsl/src/parser/runtime.rs new file mode 100644 index 00000000..e744be5a --- /dev/null +++ b/dsl/src/parser/runtime.rs @@ -0,0 +1,487 @@ +use proc_macro2::{Delimiter, Ident, TokenTree}; +use syn::spanned::Spanned as _; + +use crate::model::{Flavor, RuntimeBackend}; +use crate::parser::Parser; + +/// Backends whose name the parser knows but whose code it cannot generate. +/// +/// Kept as strings rather than [`RuntimeBackend`] variants for the reason +/// given on that enum: a variant is a promise that something downstream can +/// switch on it. Kept at all so that the message can say "not implemented" +/// rather than "no such backend", which are different mistakes and want +/// different next steps from the reader. +const RECOGNISED_UNIMPLEMENTED: &[&str] = &["forte", "blocking", "bwos"]; + +/// Duplicate `runtime:` at the table level. +/// +/// Public because the free-order dispatch is what notices a repeat, and there +/// is more than one of those loops. The text lives here with the rest of the +/// runtime diagnostics so the three sites cannot drift apart. +pub const DUPLICATE_RUNTIME: &str = "duplicate `runtime` section; a declaration selects a runtime at most once"; + +const EXPECTED_BACKEND: &str = "expected a runtime backend after `runtime:`: `nagoya`, optionally flavored as \ + `nagoya(locality)`, `nagoya(spread)` or `nagoya(throughput)`, or `tokio`"; + +/// The flavors, listed from [`Flavor::ALL`] rather than written out. +/// +/// There are four places a flavor name appears in this file's diagnostics. +/// Spelling them by hand is how a flavor gets added to the parser and left +/// out of a message that claims to be exhaustive. +fn flavor_list() -> String { + Flavor::ALL + .iter() + .map(|flavor| format!("`{}`", flavor.name())) + .collect::>() + .join(", ") +} + +fn expected_flavor() -> String { + format!("expected a flavor inside the parentheses: one of {}", flavor_list()) +} + +const TOKIO_HAS_NO_FLAVORS: &str = + "`tokio` has no flavors; write `runtime: tokio`, or select a flavored runtime with `runtime: nagoya(spread)`"; + +const EXPECTED_PROFILE: &str = "expected a profile name after `runtime`, as in `update runtime fast_local:`; \ + profiles are declared with `runtimes!`"; + +impl Parser { + /// Parse a table-level `runtime: ` section. + /// + /// This is an arm of the free-order section loop, beside `columns`, + /// `indexes`, `queries` and `config`, so it consumes its own keyword the + /// way [`Parser::parse_indexes`] does. Duplicate detection belongs to the + /// caller, which is the only thing that knows whether one was already + /// read; [`DUPLICATE_RUNTIME`] is the message to use. + pub fn parse_runtime(&mut self) -> syn::Result { + let ident = self.input_iter.next().ok_or(syn::Error::new( + self.input.span(), + "Expected `runtime` field in declaration", + ))?; + if let TokenTree::Ident(ident) = &ident { + if ident.to_string().as_str() != "runtime" { + return Err(syn::Error::new(ident.span(), "Expected `runtime` field")); + } + } else { + return Err(syn::Error::new(ident.span(), "Expected field name identifier.")); + }; + + self.parse_colon()?; + + let backend = self + .input_iter + .next() + .ok_or(syn::Error::new(self.input.span(), EXPECTED_BACKEND))?; + let TokenTree::Ident(backend) = backend else { + return Err(syn::Error::new_spanned(backend, EXPECTED_BACKEND)); + }; + + let selected = self.parse_backend(&backend)?; + + self.try_parse_comma()?; + + Ok(selected) + } + + /// The backend keyword and its optional postfix flavor. + /// + /// The postfix form is the house one: `columnar(chunk_rows(32_768))` in the + /// `config` block reads the same way, and so does `using ` on an + /// index. A flavor is therefore an argument to the backend rather than a + /// second key, which is what keeps `nagoya` alone meaning the default + /// flavor rather than meaning "unset". + fn parse_backend(&mut self, backend: &Ident) -> syn::Result { + match backend.to_string().as_str() { + "nagoya" => Ok(RuntimeBackend::Nagoya(self.try_parse_flavor()?.unwrap_or_default())), + "tokio" => { + if let Some(TokenTree::Group(group)) = self.input_iter.peek() + && group.delimiter() == Delimiter::Parenthesis + { + let group = group.clone(); + return Err(syn::Error::new_spanned(group, TOKIO_HAS_NO_FLAVORS)); + } + Ok(RuntimeBackend::Tokio) + } + other if RECOGNISED_UNIMPLEMENTED.contains(&other) => Err(syn::Error::new_spanned( + backend, + format!( + "runtime backend `{other}` is recognised but not implemented; the implemented backends are \ + `nagoya` and `tokio`" + ), + )), + other => Err(syn::Error::new_spanned( + backend, + format!("unknown runtime backend `{other}`; expected `nagoya` or `tokio`"), + )), + } + } + + /// `(locality)`, `(spread)` or `(throughput)`, if one was written. + fn try_parse_flavor(&mut self) -> syn::Result> { + let Some(TokenTree::Group(group)) = self.input_iter.peek() else { + return Ok(None); + }; + if group.delimiter() != Delimiter::Parenthesis { + return Ok(None); + } + let group = group.clone(); + self.input_iter.next(); + + let mut inner = group.stream().into_iter(); + let flavor = inner + .next() + .ok_or_else(|| syn::Error::new_spanned(&group, expected_flavor()))?; + let TokenTree::Ident(flavor) = flavor else { + return Err(syn::Error::new_spanned(flavor, expected_flavor())); + }; + if let Some(extra) = inner.next() { + return Err(syn::Error::new_spanned( + extra, + format!("`nagoya` takes a single flavor; write one of {}", flavor_list()), + )); + } + + let name = flavor.to_string(); + Flavor::from_name(&name).map(Some).ok_or_else(|| { + syn::Error::new_spanned( + &flavor, + format!("unknown nagoya flavor `{name}`; expected one of {}", flavor_list()), + ) + }) + } + + /// The optional `runtime ` between a query section's keyword and + /// its colon, as in `update runtime fast_local: { .. }`. + /// + /// The token after `runtime` is a profile name, never a backend literal. + /// A section names a profile because a profile carries tuning as well as a + /// backend, and because the backend is a property of the table rather than + /// of one of its query blocks. Naming a backend here is therefore rejected + /// rather than quietly treated as a profile that happens to be called + /// `nagoya`. + pub fn try_parse_section_runtime(&mut self) -> syn::Result> { + let Some(TokenTree::Ident(keyword)) = self.input_iter.peek() else { + return Ok(None); + }; + if keyword != "runtime" { + return Ok(None); + } + let keyword = keyword.clone(); + self.input_iter.next(); + + let profile = self + .input_iter + .next() + .ok_or_else(|| syn::Error::new_spanned(&keyword, EXPECTED_PROFILE))?; + let TokenTree::Ident(profile) = profile else { + return Err(syn::Error::new_spanned(profile, EXPECTED_PROFILE)); + }; + + let name = profile.to_string(); + if name == "nagoya" || name == "tokio" || RECOGNISED_UNIMPLEMENTED.contains(&name.as_str()) { + return Err(syn::Error::new_spanned( + &profile, + format!( + "`{name}` is a runtime backend, not a profile name; a query section names a profile declared \ + with `runtimes!`, and the backend is selected once for the table with `runtime: {name}`" + ), + )); + } + + Ok(Some(profile)) + } +} + +#[cfg(test)] +mod tests { + use quote::quote; + + use crate::Parser; + use crate::model::{Flavor, RuntimeBackend}; + + /// A bare `nagoya` is whatever flavor is currently the default. Named out + /// of the registry rather than written here, so moving the default does + /// not turn this into a failure about a word. + #[test] + fn parses_bare_nagoya_as_the_default_flavor() { + let mut parser = Parser::new(quote! { runtime: nagoya, }); + assert_eq!( + parser.parse_runtime().unwrap(), + RuntimeBackend::Nagoya(Flavor::default()) + ); + } + + #[test] + fn bare_nagoya_equals_the_default() { + let mut parser = Parser::new(quote! { runtime: nagoya, }); + assert_eq!(parser.parse_runtime().unwrap(), RuntimeBackend::default()); + } + + #[test] + fn parses_all_backends() { + for (tokens, expected) in [ + (quote! { runtime: nagoya, }, RuntimeBackend::Nagoya(Flavor::default())), + ( + quote! { runtime: nagoya(locality), }, + RuntimeBackend::Nagoya(Flavor::Locality), + ), + ( + quote! { runtime: nagoya(spread), }, + RuntimeBackend::Nagoya(Flavor::Spread), + ), + ( + quote! { runtime: nagoya(throughput), }, + RuntimeBackend::Nagoya(Flavor::Throughput), + ), + (quote! { runtime: tokio, }, RuntimeBackend::Tokio), + ] { + let mut parser = Parser::new(tokens); + assert_eq!(parser.parse_runtime().unwrap(), expected); + } + } + + #[test] + fn trailing_comma_is_optional() { + let mut parser = Parser::new(quote! { runtime: nagoya(spread) }); + assert_eq!(parser.parse_runtime().unwrap(), RuntimeBackend::Nagoya(Flavor::Spread)); + assert!(!parser.has_next()); + } + + #[test] + fn leaves_the_next_section_for_the_dispatch_loop() { + let mut parser = Parser::new(quote! { runtime: tokio, columns: { id: u64 primary_key } }); + assert_eq!(parser.parse_runtime().unwrap(), RuntimeBackend::Tokio); + assert_eq!(parser.peek_next().unwrap().to_string(), "columns"); + } + + #[test] + fn rejects_recognised_but_unimplemented_backends() { + for backend in ["forte", "blocking", "bwos"] { + let tokens: proc_macro2::TokenStream = format!("runtime: {backend},").parse().unwrap(); + let error = Parser::new(tokens).parse_runtime().unwrap_err().to_string(); + assert!( + error.contains(&format!("`{backend}` is recognised but not implemented")), + "{error}" + ); + assert!(error.contains("`nagoya` and `tokio`"), "{error}"); + } + } + + #[test] + fn rejects_unknown_backend() { + let error = Parser::new(quote! { runtime: banana, }) + .parse_runtime() + .unwrap_err() + .to_string(); + assert_eq!(error, "unknown runtime backend `banana`; expected `nagoya` or `tokio`"); + } + + #[test] + fn rejects_unknown_flavor() { + let error = Parser::new(quote! { runtime: nagoya(banana), }) + .parse_runtime() + .unwrap_err() + .to_string(); + assert!(error.starts_with("unknown nagoya flavor `banana`;"), "{error}"); + for flavor in Flavor::ALL { + assert!(error.contains(flavor.name()), "{} missing from: {error}", flavor.name()); + } + } + + #[test] + fn rejects_a_flavor_on_tokio() { + let error = Parser::new(quote! { runtime: tokio(spread), }) + .parse_runtime() + .unwrap_err() + .to_string(); + assert!(error.contains("`tokio` has no flavors"), "{error}"); + } + + #[test] + fn rejects_two_flavors() { + let error = Parser::new(quote! { runtime: nagoya(spread, locality), }) + .parse_runtime() + .unwrap_err() + .to_string(); + assert!(error.contains("takes a single flavor"), "{error}"); + } + + #[test] + fn rejects_an_empty_flavor_list() { + let error = Parser::new(quote! { runtime: nagoya(), }) + .parse_runtime() + .unwrap_err() + .to_string(); + assert!(error.contains("expected a flavor inside the parentheses"), "{error}"); + } + + #[test] + fn rejects_a_missing_backend() { + let error = Parser::new(quote! { runtime: }) + .parse_runtime() + .unwrap_err() + .to_string(); + assert!(error.contains("expected a runtime backend after `runtime:`"), "{error}"); + } + + #[test] + fn section_annotation_reads_a_profile_name() { + let mut parser = Parser::new(quote! { runtime fast_local: }); + let profile = parser.try_parse_section_runtime().unwrap().expect("annotated"); + assert_eq!(profile, "fast_local"); + assert_eq!(parser.peek_next().unwrap().to_string(), ":"); + } + + #[test] + fn section_annotation_is_optional() { + let mut parser = Parser::new(quote! { : { Fill(qty) by id } }); + assert!(parser.try_parse_section_runtime().unwrap().is_none()); + assert_eq!(parser.peek_next().unwrap().to_string(), ":"); + } + + #[test] + fn section_annotation_rejects_a_backend_literal() { + for backend in ["nagoya", "tokio", "forte"] { + let tokens: proc_macro2::TokenStream = format!("runtime {backend}:").parse().unwrap(); + let error = Parser::new(tokens).try_parse_section_runtime().unwrap_err().to_string(); + assert!( + error.contains(&format!("`{backend}` is a runtime backend, not a profile name")), + "{error}" + ); + } + } + + #[test] + fn section_annotation_rejects_a_missing_profile_name() { + let error = Parser::new(quote! { runtime }) + .try_parse_section_runtime() + .unwrap_err() + .to_string(); + assert!(error.contains("expected a profile name after `runtime`"), "{error}"); + } + + /// The free-order dispatch, exercised through the whole declaration + /// rather than through `parse_runtime` alone. Position is a property of + /// the loop, so a test that calls the section parser directly cannot see + /// it. + fn schema(source: &str) -> crate::Schema { + crate::Schema::parse(source).unwrap_or_else(|error| panic!("{error}\n{source}")) + } + + #[test] + fn an_omitted_runtime_is_the_default() { + let schema = schema("name: Bare, columns: { id: u64 primary_key }"); + assert_eq!(schema.runtime, RuntimeBackend::default()); + } + + #[test] + fn runtime_may_precede_columns() { + let schema = schema( + " + name: First, + runtime: nagoya(spread), + columns: { id: u64 primary_key }, + ", + ); + assert_eq!(schema.runtime, RuntimeBackend::Nagoya(Flavor::Spread)); + } + + #[test] + fn runtime_may_follow_queries() { + let schema = schema( + " + name: Last, + columns: { id: u64 primary_key, qty: u64 }, + queries: { update: { Fill(qty) by id } }, + runtime: tokio, + ", + ); + assert_eq!(schema.runtime, RuntimeBackend::Tokio); + } + + #[test] + fn runtime_may_sit_between_indexes_and_config() { + let schema = schema( + " + name: Middle, + columns: { id: u64 primary_key, qty: u64 }, + indexes: { qty_idx: qty }, + runtime: nagoya(throughput), + config: { page_size: 4096 }, + ", + ); + assert_eq!(schema.runtime, RuntimeBackend::Nagoya(Flavor::Throughput)); + } + + #[test] + fn rejects_a_second_runtime() { + let error = crate::Schema::parse( + " + name: Twice, + runtime: tokio, + columns: { id: u64 primary_key }, + runtime: nagoya, + ", + ) + .unwrap_err() + .to_string(); + assert_eq!( + error, + "duplicate `runtime` section; a declaration selects a runtime at most once" + ); + } + + #[test] + fn a_section_annotation_survives_the_whole_declaration() { + let schema = schema( + " + name: Annotated, + columns: { id: u64 primary_key, qty: u64, symbol: u64 }, + queries: { + update runtime fast_local: { Fill(qty) by id }, + delete runtime wide: { BySymbol() by symbol }, + in_place: { Bump(qty) by id }, + }, + ", + ); + assert_eq!(schema.queries.update_runtime.as_deref(), Some("fast_local")); + assert_eq!(schema.queries.delete_runtime.as_deref(), Some("wide")); + assert_eq!(schema.queries.in_place_runtime, None); + } + + #[test] + fn a_declared_runtime_survives_the_round_trip() { + let source = " + name: RoundTrip, + columns: { id: u64 primary_key, qty: u64 }, + runtime: nagoya(spread), + queries: { update runtime wide: { Fill(qty) by id } }, + "; + let once = schema(source); + let twice = schema(&once.to_dsl()); + assert_eq!(once, twice); + assert_eq!(twice.runtime, RuntimeBackend::Nagoya(Flavor::Spread)); + assert_eq!(twice.queries.update_runtime.as_deref(), Some("wide")); + } + + #[test] + fn the_default_runtime_is_not_written_back_out() { + // An omitted `runtime` and an explicit `runtime: nagoya` are the same + // table, so the emitter writes neither. + let dsl = schema("name: Quiet, runtime: nagoya, columns: { id: u64 primary_key }").to_dsl(); + assert!(!dsl.contains("runtime"), "{dsl}"); + } + + #[test] + fn backend_names_round_trip() { + assert_eq!(RuntimeBackend::Nagoya(Flavor::Spread).name(), "nagoya"); + assert_eq!(RuntimeBackend::Tokio.name(), "tokio"); + for flavor in Flavor::ALL { + assert_eq!(Flavor::from_name(flavor.name()), Some(flavor)); + } + assert_eq!(Flavor::Locality.name(), "locality"); + assert_eq!(Flavor::LowLatency.name(), "low_latency"); + assert_eq!(Flavor::WideInjector.name(), "wide_injector"); + } +} diff --git a/dsl/src/schema/diff.rs b/dsl/src/schema/diff.rs index b6722a8e..99e650d5 100644 --- a/dsl/src/schema/diff.rs +++ b/dsl/src/schema/diff.rs @@ -37,7 +37,7 @@ use std::collections::BTreeSet; use std::fmt::Write as _; use super::{ColumnSpec, IndexSpec, PartitionKeySpec, Schema}; -use crate::model::{IndexBackend, Persistence}; +use crate::model::{IndexBackend, Persistence, Storage}; /// What applying a change costs. /// @@ -94,6 +94,24 @@ pub enum Change { /// The declared name. to: String, }, + /// The row storage changed; a caller must choose how to convert it. + StorageChanged { + /// Stored representation. + from: Storage, + /// Declared representation. + to: Storage, + }, + /// The page layout changed and existing row links cannot be reused. + PageSizeChanged { + /// Stored setting; absence selects the default. + from: Option, + /// Declared setting; absence selects the default. + to: Option, + }, + /// Derived columnar storage or clustering changed. + ColumnarChanged, + /// Runtime selection changed without changing archived rows. + RuntimeChanged, /// `persist` changed. PersistenceChanged { /// What was stored. @@ -202,8 +220,7 @@ pub enum Change { }, /// The generated queries differ. Nothing on disk depends on them. QueriesChanged, - /// The `config` block differs. `page_size` is pinned to the on-disk page - /// size for persisted tables, so what is left here cannot reach the data. + /// Code-only configuration differs. Layout changes are reported separately. ConfigChanged, } @@ -211,23 +228,26 @@ impl Change { /// What applying this change costs. pub fn cost(&self) -> Cost { match self { - Self::Version { .. } | Self::QueriesChanged | Self::ConfigChanged => Cost::Nothing, + Self::Version { .. } | Self::QueriesChanged | Self::ConfigChanged | Self::RuntimeChanged => Cost::Nothing, Self::IndexAdded(_) | Self::IndexDropped(_) | Self::IndexColumnChanged { .. } | Self::IndexUniquenessChanged { .. } | Self::IndexBackendChanged { .. } - | Self::PrimaryIndexBackendChanged { .. } => Cost::RebuildIndexes, + | Self::PrimaryIndexBackendChanged { .. } + | Self::ColumnarChanged => Cost::RebuildIndexes, Self::ColumnAdded(_) | Self::ColumnDropped(_) | Self::ColumnTypeChanged { .. } | Self::ColumnOptionalityChanged { .. } - | Self::ColumnMoved { .. } => Cost::RewriteRows, + | Self::ColumnMoved { .. } + | Self::PageSizeChanged { .. } => Cost::RewriteRows, Self::Renamed { .. } | Self::PersistenceChanged { .. } + | Self::StorageChanged { .. } | Self::PartitionKeyChanged { .. } | Self::PrimaryKeyChanged { .. } | Self::PrimaryKeyGeneratorChanged { .. } => Cost::NeedsIntent, @@ -364,6 +384,12 @@ impl Diff { to: declared.version, }); } + if stored.storage != declared.storage { + changes.push(Change::StorageChanged { + from: stored.storage, + to: declared.storage, + }); + } if stored.persist != declared.persist { changes.push(Change::PersistenceChanged { from: stored.persist, @@ -398,7 +424,29 @@ impl Diff { if stored.queries != declared.queries { changes.push(Change::QueriesChanged); } - if stored.config != declared.config { + if stored.runtime != declared.runtime { + changes.push(Change::RuntimeChanged); + } + // A changed explicit page setting is conservatively a row rewrite, + // including transitions to/from a default selected by the generator. + if stored.config.page_size != declared.config.page_size { + changes.push(Change::PageSizeChanged { + from: stored.config.page_size, + to: declared.config.page_size, + }); + } + if stored.columnar_indexes != declared.columnar_indexes + || stored.config.columnar_slot_id != declared.config.columnar_slot_id + || stored.config.columnar_chunk_rows != declared.config.columnar_chunk_rows + || stored.columns.iter().any(|column| { + declared + .column(&column.name) + .is_some_and(|after| column.columnar != after.columnar) + }) + { + changes.push(Change::ColumnarChanged); + } + if stored.config.row_derives != declared.config.row_derives { changes.push(Change::ConfigChanged); } @@ -512,6 +560,10 @@ fn describe_change(change: &Change) -> String { Change::PrimaryIndexBackendChanged { from, to } => { format!("primary index: {} -> {}", from.name(), to.name()) } + Change::StorageChanged { from, to } => format!("row storage {from:?} -> {to:?}"), + Change::PageSizeChanged { from, to } => format!("page size {from:?} -> {to:?}"), + Change::ColumnarChanged => "columnar layout or clustering changed".to_string(), + Change::RuntimeChanged => "runtime changed".to_string(), Change::QueriesChanged => "queries changed".to_string(), Change::ConfigChanged => "config changed".to_string(), } diff --git a/dsl/src/schema/emit_dsl.rs b/dsl/src/schema/emit_dsl.rs index e9c3d2b4..fe73a108 100644 --- a/dsl/src/schema/emit_dsl.rs +++ b/dsl/src/schema/emit_dsl.rs @@ -18,7 +18,7 @@ use std::fmt::Write as _; use super::{ColumnSpec, IndexSpec, OperationSpec, Schema}; -use crate::model::{GeneratorType, IndexBackend, Persistence}; +use crate::model::{GeneratorType, IndexBackend, Persistence, RuntimeBackend}; const INDENT: &str = " "; @@ -30,6 +30,13 @@ impl Schema { let _ = writeln!(out, "name: {},", self.name); let _ = writeln!(out, "version: {},", self.version); + // Only when true. `vec: false` is what every declaration written + // before this key existed meant, so writing it out would add a line to + // every emitted schema in the corpus to say nothing. + if self.storage.is_vec() { + let _ = writeln!(out, "vec: true,"); + } + match self.persist { // An omitted `persist` is not the same as `persist: false`: the // macro requires the acknowledgement before it will accept an @@ -46,6 +53,16 @@ impl Schema { if let Some(key) = &self.partition_by { let _ = writeln!(out, "partition_by: {}: {},", key.name, key.ty); + // Required beside it, so emitting one without the other produces + // text this crate's own parser refuses. + let _ = writeln!(out, "partition_max_size: {},", key.max_size); + } + + // Same rule as `using` on a column: writing the default back out would + // be correct but noisy, and an omitted `runtime` and an explicit + // `runtime: nagoya` are the same table. + if self.runtime != RuntimeBackend::default() { + let _ = writeln!(out, "runtime: {},", runtime_to_dsl(self.runtime)); } let _ = writeln!(out, "columns: {{"); @@ -62,11 +79,36 @@ impl Schema { let _ = writeln!(out, "}},"); } + if !self.columnar_indexes.is_empty() { + let _ = writeln!(out, "columnar_indexes: {{"); + for index in &self.columnar_indexes { + let _ = writeln!(out, "{INDENT}{}: {{", index.name); + let _ = writeln!(out, "{INDENT}{INDENT}cluster_by: [{}],", index.cluster_by.join(", ")); + let _ = writeln!(out, "{INDENT}}},"); + } + let _ = writeln!(out, "}},"); + } + if !self.queries.is_empty() { let _ = writeln!(out, "queries: {{"); - write_query_block(&mut out, "update", &self.queries.updates); - write_query_block(&mut out, "delete", &self.queries.deletes); - write_query_block(&mut out, "in_place", &self.queries.in_place); + write_query_block( + &mut out, + "update", + self.queries.update_runtime.as_deref(), + &self.queries.updates, + ); + write_query_block( + &mut out, + "delete", + self.queries.delete_runtime.as_deref(), + &self.queries.deletes, + ); + write_query_block( + &mut out, + "in_place", + self.queries.in_place_runtime.as_deref(), + &self.queries.in_place, + ); let _ = writeln!(out, "}},"); } @@ -75,6 +117,12 @@ impl Schema { if let Some(page_size) = self.config.page_size { let _ = writeln!(out, "{INDENT}page_size: {page_size},"); } + if let Some(slot_id) = &self.config.columnar_slot_id { + let _ = writeln!(out, "{INDENT}columnar_slot_id: {slot_id},"); + } + if let Some(chunk_rows) = self.config.columnar_chunk_rows { + let _ = writeln!(out, "{INDENT}columnar_chunk_rows: {chunk_rows},"); + } if !self.config.row_derives.is_empty() { // `row_derives` reads identifiers until it meets another config // key, so it has to be written last of the two. @@ -121,6 +169,23 @@ fn column_to_dsl(column: &ColumnSpec) -> String { out.push_str(" optional"); } + // `columnar`, with only the options that were written. A bare `columnar` + // and `columnar(chunk_rows(2))` are different declarations, and the second + // is not the first plus a default, so nothing is filled in here. + if let Some(columnar) = &column.columnar { + out.push_str(" columnar"); + let mut options = Vec::new(); + if let Some(chunk_rows) = columnar.chunk_rows { + options.push(format!("chunk_rows({chunk_rows})")); + } + if let Some(compression) = &columnar.compression { + options.push(format!("compression({compression})")); + } + if !options.is_empty() { + let _ = write!(out, "({})", options.join(", ")); + } + } + // A primary-key column always carries a backend once parsed, because the // model fills the default in. Writing the default back out would be // correct but noisy, and the point of this emitter is text a person will @@ -145,11 +210,29 @@ fn index_to_dsl(index: &IndexSpec) -> String { out } -fn write_query_block(out: &mut String, kind: &str, operations: &[OperationSpec]) { +fn runtime_to_dsl(backend: RuntimeBackend) -> String { + match backend { + // The flavor is written even when it is the default one, because this + // arm is only reached for a backend that is not the default, and a + // reader comparing two declarations should not have to know which + // flavor `nagoya` alone means. + RuntimeBackend::Nagoya(flavor) => format!("{}({})", backend.name(), flavor.name()), + RuntimeBackend::Tokio => backend.name().to_string(), + } +} + +fn write_query_block(out: &mut String, kind: &str, runtime: Option<&str>, operations: &[OperationSpec]) { if operations.is_empty() { return; } - let _ = writeln!(out, "{INDENT}{kind}: {{"); + match runtime { + Some(profile) => { + let _ = writeln!(out, "{INDENT}{kind} runtime {profile}: {{"); + } + None => { + let _ = writeln!(out, "{INDENT}{kind}: {{"); + } + } for operation in operations { let _ = writeln!( out, @@ -165,3 +248,38 @@ fn write_query_block(out: &mut String, kind: &str, operations: &[OperationSpec]) // routinely fed to a macro older than the emitter that wrote it. let _ = writeln!(out, "{INDENT}}}"); } + +#[cfg(test)] +mod storage_round_trip { + use crate::schema::Schema; + + /// `storage: vec` survives a parse and an emit. + /// + /// The emitter is fed back to the macro, so a key it drops is a key that + /// silently changes which table a regenerated declaration produces. Paged + /// is the default and is deliberately not written; vec always is. + #[test] + fn storage_vec_survives_but_paged_is_never_written() { + let declared = "name: T,\nversion: 1,\nvec: true,\ncolumns: {\n id: u64 primary_key,\n}\n"; + let schema = Schema::parse(declared).expect("valid"); + assert!(schema.storage.is_vec()); + assert!(schema.to_dsl().contains("vec: true,"), "got: {}", schema.to_dsl()); + + let paged = "name: T,\nversion: 1,\ncolumns: {\n id: u64 primary_key,\n}\n"; + let schema = Schema::parse(paged).expect("valid"); + assert!(!schema.storage.is_vec()); + assert!(!schema.to_dsl().contains("vec:"), "got: {}", schema.to_dsl()); + } + + /// And the emitted text parses back to the same schema. + #[test] + fn the_emitted_text_round_trips() { + let declared = "name: T,\nversion: 1,\nvec: true,\ncolumns: {\n id: u64 primary_key,\n value: u64,\n}\n"; + let once = Schema::parse(declared).expect("valid"); + let text = once.to_dsl(); + let twice = Schema::parse(&text).expect("the emitter writes valid text"); + assert_eq!(once.storage, twice.storage); + assert_eq!(once.persist, twice.persist); + assert_eq!(text, twice.to_dsl()); + } +} diff --git a/dsl/src/schema/emit_uml.rs b/dsl/src/schema/emit_uml.rs index 25bef40c..1f185921 100644 --- a/dsl/src/schema/emit_uml.rs +++ b/dsl/src/schema/emit_uml.rs @@ -72,10 +72,21 @@ impl Schema { let _ = writeln!(out, " }}"); if let Some(key) = &self.partition_by { + // The row count rather than the width. The width is how it is + // declared; the count is what a reader of a diagram wants, and the + // whole reason the key is required is that the shape was not + // visible without it. + let size = match crate::model::PartitionMaxSize::from_type_name(&key.max_size) { + Some(width) => match width.rows() { + Some(rows) => format!("at most {rows} rows each"), + None => "unbounded".to_string(), + }, + None => format!("at most {} rows each", key.max_size), + }; let _ = writeln!( out, - " note for {} \"partitioned by {}: {}\"", - self.name, key.name, key.ty + " note for {} \"partitioned by {}: {}, {}\"", + self.name, key.name, key.ty, size ); } } diff --git a/dsl/src/schema/mod.rs b/dsl/src/schema/mod.rs index bb33a37a..701f05e0 100644 --- a/dsl/src/schema/mod.rs +++ b/dsl/src/schema/mod.rs @@ -46,7 +46,7 @@ use proc_macro2::TokenStream; use syn::spanned::Spanned as _; -use crate::model::{Columns, GeneratorType, IndexBackend, Persistence, Queries}; +use crate::model::{Columns, GeneratorType, IndexBackend, Persistence, Queries, RuntimeBackend, Storage}; use crate::parser::Parser; mod diff; @@ -71,6 +71,11 @@ pub struct Schema { /// resolved value rather than the absence, because a consumer comparing an /// on-disk version against a declared one wants a number either way. pub version: u32, + /// What holds the rows. `serde(default)` is [`Storage::Paged`], so a + /// schema written before this field existed reads back as the table it + /// was: every declaration then was paged. + #[cfg_attr(feature = "serde", serde(default))] + pub storage: Storage, /// Whether persistence was selected, and whether it was selected at all. pub persist: Persistence, /// The routing key of a partitioned table. Not a column: it is stored once @@ -80,10 +85,54 @@ pub struct Schema { pub columns: Vec, /// Secondary indexes in declaration order. pub indexes: Vec, + /// The runtime the table is built against. Absent in the declaration means + /// [`RuntimeBackend::default`], and this stores the resolved value for the + /// same reason `version` does: a consumer asking which runtime a table + /// uses wants an answer either way. + #[cfg_attr(feature = "serde", serde(default))] + pub runtime: RuntimeBackend, /// Generated queries, sorted by name within each kind. pub queries: QueriesSpec, /// The `config` block. pub config: ConfigSpec, + /// Columnar indexes in declaration order. + /// + /// **This was parsed and then dropped.** `from_tokens` read the block into + /// the model and the `Schema` it returned never carried it, so a columnar + /// table emitted by `to_dsl` came back without its clustering, and every + /// consumer downstream of this type, including the TypeScript emitter and + /// the JSON dump, was blind to it. + /// + /// The round-trip test could not catch that: the property is + /// `parse(emit(parse(s))) == parse(s)`, which holds trivially for anything + /// this type does not model. See `columnar_survives_the_round_trip`. + #[cfg_attr(feature = "serde", serde(default))] + pub columnar_indexes: Vec, +} + +/// A column's `columnar(...)` options. +/// +/// `Some` means the column declared `columnar`, with or without options. +/// Absent options are absent rather than defaulted, so an emitted declaration +/// says what was written: `columnar` and `columnar(chunk_rows(2))` are +/// different text and the second is not the first plus a default. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ColumnarSpec { + /// `chunk_rows(n)`, when written. + pub chunk_rows: Option, + /// `compression(name)`, when it differs from the default. + pub compression: Option, +} + +/// A columnar index: `name: { cluster_by: [field, ..] }`. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ColumnarIndexSpec { + /// Index name. + pub name: String, + /// The fields it clusters by, in declaration order. + pub cluster_by: Vec, } /// A column declaration: `name: Type [primary_key] [autoincrement|custom] [optional] [using backend]`. @@ -102,6 +151,9 @@ pub struct ColumnSpec { /// The primary-key generator. Only meaningful when `primary_key` is set, /// and shared by every column of a composite key. pub generator: GeneratorType, + /// The `columnar(...)` options, when the column declared them. + #[cfg_attr(feature = "serde", serde(default))] + pub columnar: Option, /// The primary index backend. `Some` on primary-key columns, carrying the /// declared backend or the default when `using` was omitted; `None` /// elsewhere, because `using` on a non-key column is a parse error. @@ -130,6 +182,13 @@ pub struct PartitionKeySpec { pub name: String, /// Unsigned integer type. See [`crate::model::PARTITION_KEY_TYPES`]. pub ty: String, + /// The declared `partition_max_size` width, as it is written. See + /// [`crate::model::PartitionMaxSize`]. + /// + /// Not optional, because the key it belongs to is not: a stored schema + /// without it predates the key and would compare unequal to every + /// declaration, which is the honest answer rather than a defect. + pub max_size: String, } /// The `queries` block. @@ -143,6 +202,13 @@ pub struct QueriesSpec { pub deletes: Vec, /// `in_place:` operations. pub in_place: Vec, + /// The profile named by `update runtime :`, if written. Unresolved: + /// see [`crate::model::Queries::update_runtime`]. + pub update_runtime: Option, + /// The profile named by `delete runtime :`, if written. + pub delete_runtime: Option, + /// The profile named by `in_place runtime :`, if written. + pub in_place_runtime: Option, } impl QueriesSpec { @@ -174,12 +240,26 @@ pub struct ConfigSpec { pub page_size: Option, /// Extra derives placed on the generated row type. pub row_derives: Vec, + /// `columnar_slot_id`, when it differs from the default. + /// + /// Stored as the difference rather than the resolved value, because the + /// parser applies defaults and a resolved value cannot be told from a + /// written one. Emitting a default that was never written is noise; not + /// emitting a written non-default loses it. + #[cfg_attr(feature = "serde", serde(default))] + pub columnar_slot_id: Option, + /// `columnar_chunk_rows`, when it differs from the default. + #[cfg_attr(feature = "serde", serde(default))] + pub columnar_chunk_rows: Option, } impl ConfigSpec { /// Whether anything was configured. pub fn is_empty(&self) -> bool { - self.page_size.is_none() && self.row_derives.is_empty() + self.page_size.is_none() + && self.row_derives.is_empty() + && self.columnar_slot_id.is_none() + && self.columnar_chunk_rows.is_none() } } @@ -204,16 +284,20 @@ impl Schema { let name = parser.parse_name()?; let version = parser.parse_version()?.unwrap_or(1); + let storage = parser.parse_storage()?; let persist = parser.parse_persist()?; let partition_by = parser.parse_partition_by()?.map(|key| PartitionKeySpec { name: key.name.to_string(), ty: key.ty.to_string(), + max_size: key.max_size.type_name().to_string(), }); let mut columns: Option = None; let mut indexes = None; let mut queries: Option = None; let mut config = None; + let mut columnar_indexes = None; + let mut runtime = None; while let Some(ident) = parser.peek_next() { match ident.to_string().as_str() { @@ -221,24 +305,37 @@ impl Schema { "indexes" => indexes = Some(parser.parse_indexes()?), "queries" => queries = Some(parser.parse_queries()?), "config" => config = Some(parser.parse_configs()?), + "columnar_indexes" => columnar_indexes = Some(parser.parse_columnar_indexes()?), + // Free-order like the blocks around it, and unlike `persist` + // and `partition_by`, because nothing downstream of it depends + // on having been read first. + "runtime" => { + let span = ident.span(); + if runtime.is_some() { + return Err(syn::Error::new(span, crate::parser::DUPLICATE_RUNTIME)); + } + runtime = Some(parser.parse_runtime()?); + } "version" => { return Err(syn::Error::new( ident.span(), "version must be specified before columns/indexes/queries/config", )); } - "persist" | "partition_by" => { + "vec" | "persist" | "partition_by" | "partition_max_size" => { return Err(syn::Error::new( ident.span(), - "`persist` and `partition_by` are positional; the required order is: \ - name, version, persist, partition_by, then columns/indexes/queries/config", + "`vec`, `persist`, `partition_by` and `partition_max_size` are positional; the required \ + order is: name, version, vec, persist, partition_by, partition_max_size, then \ + columns/indexes/queries/config", )); } other => { return Err(syn::Error::new( ident.span(), format!( - "Unexpected token `{other}`; expected one of `columns`, `indexes`, `queries`, `config`" + "Unexpected token `{other}`; expected one of `columns`, `indexes`, `columnar_indexes`, \ + `queries`, `config`, `runtime`" ), )); } @@ -247,6 +344,9 @@ impl Schema { let mut model = columns.ok_or_else(|| syn::Error::new(parser.input.span(), "Expected a `columns` block in declaration"))?; + if let Some(columnar_indexes) = columnar_indexes { + model.columnar_indexes = columnar_indexes.indexes; + } if let Some(indexes) = indexes { model.indexes = indexes; } @@ -254,17 +354,34 @@ impl Schema { Ok(Self { name: name.to_string(), version, + storage, persist, partition_by, + runtime: runtime.unwrap_or_default(), columns: columns_from_model(&model)?, indexes: indexes_from_model(&model), queries: queries.map(queries_from_model).unwrap_or_default(), config: config - .map(|config| ConfigSpec { - page_size: config.page_size, - row_derives: config.row_derives.iter().map(ToString::to_string).collect(), + .map(|config| { + let defaults = crate::model::Config::default(); + ConfigSpec { + page_size: config.page_size, + row_derives: config.row_derives.iter().map(ToString::to_string).collect(), + columnar_slot_id: (config.columnar_slot_id != defaults.columnar_slot_id) + .then(|| config.columnar_slot_id.type_name().to_owned()), + columnar_chunk_rows: (config.columnar_chunk_rows != defaults.columnar_chunk_rows) + .then_some(config.columnar_chunk_rows), + } }) .unwrap_or_default(), + columnar_indexes: model + .columnar_indexes + .values() + .map(|index| ColumnarIndexSpec { + name: index.name.to_string(), + cluster_by: index.cluster_by.iter().map(ToString::to_string).collect(), + }) + .collect(), }) } @@ -299,6 +416,14 @@ fn columns_from_model(model: &Columns) -> syn::Result> { } else { GeneratorType::None }, + columnar: model.columnar_fields.get(name).map(|config| { + let defaults = crate::model::ColumnarFieldConfig::default(); + ColumnarSpec { + chunk_rows: config.chunk_rows, + compression: (config.compression != defaults.compression) + .then(|| config.compression.name().to_owned()), + } + }), index_backend: primary_key.then_some(model.primary_index_backend), }) }) @@ -371,6 +496,9 @@ fn queries_from_model(queries: Queries) -> QueriesSpec { } QueriesSpec { + update_runtime: queries.update_runtime.map(|profile| profile.to_string()), + delete_runtime: queries.delete_runtime.map(|profile| profile.to_string()), + in_place_runtime: queries.in_place_runtime.map(|profile| profile.to_string()), updates: convert(queries.updates), deletes: convert(queries.deletes), in_place: convert(queries.in_place), diff --git a/dsl/src/validate.rs b/dsl/src/validate.rs index 36415291..44980c1b 100644 --- a/dsl/src/validate.rs +++ b/dsl/src/validate.rs @@ -19,31 +19,42 @@ use crate::model::{Columns, IndexBackend, Persistence}; -/// data_bucket's on-disk layer seeks with its own hardcoded `PAGE_SIZE` of -/// 16384 bytes (`seek_to_page_start`, `seek_by_link`, `persist_page`), while -/// the generated table threads the user's `page_size` through its page-id and -/// length arithmetic. Any other value therefore reads and writes the wrong -/// file offsets as soon as the table persists, silently corrupting it. -/// In-memory tables never seek a file: for them `page_size` only sizes index -/// nodes and stays configurable. -const DATA_BUCKET_PAGE_SIZE: u32 = 16384; +/// Bytes of `GeneralHeader` at the front of every persisted page. A page has +/// to be larger than this or there is no room left for a row. +const GENERAL_HEADER_SIZE: u32 = 28; +/// The smallest persisted page worth allowing. Below this the header is most +/// of the page and the table spends its time on page transitions; the number +/// is a floor against obvious mistakes, not a tuned value. +/// +/// It applies to persisted tables only. An in-memory table writes no header, +/// so its `page_size` only sizes index nodes and a small one is a legitimate +/// choice rather than a mistake. +const MINIMUM_PERSISTED_PAGE_SIZE: u32 = 512; + +/// A persisted table used to be refused any page size but 16384, because +/// `data_bucket` computed every offset from a hardcoded `PAGE_SIZE` in +/// `seek_to_page_start`, `seek_by_link` and `persist_page` while the generated +/// table threaded the configured size through its page-id arithmetic. The two +/// disagreed and the file was silently corrupt. +/// +/// Those seeks take the stride as a parameter now, and the generated table +/// passes its own constant to every one of them, in the data file and the index +/// file alike. The restriction is gone. What is left is the arithmetic that +/// still has to hold. pub fn validate_page_size(config: Option<&crate::model::Config>, persistence: Persistence) -> syn::Result<()> { let Some(config) = config else { return Ok(()) }; let Some(page_size) = config.page_size else { return Ok(()); }; - if persistence.is_persisted() && page_size != DATA_BUCKET_PAGE_SIZE { - let span = config.page_size_span.unwrap_or_else(proc_macro2::Span::call_site); + let span = config.page_size_span.unwrap_or_else(proc_macro2::Span::call_site); + if persistence.is_persisted() && page_size < MINIMUM_PERSISTED_PAGE_SIZE { return Err(syn::Error::new( span, format!( - "`page_size: {page_size}` cannot be combined with `persist: true`: the on-disk \ - layer (data_bucket) hardcodes {DATA_BUCKET_PAGE_SIZE}-byte pages in every file \ - seek, so a persisted table with any other page size reads and writes the wrong \ - pages and corrupts its files. Remove `page_size` (or set it to \ - {DATA_BUCKET_PAGE_SIZE}); custom page sizes remain available for in-memory \ - tables, where they only size index nodes" + "`page_size: {page_size}` is below the {MINIMUM_PERSISTED_PAGE_SIZE}-byte \ + minimum for a persisted table. Each page on disk carries a \ + {GENERAL_HEADER_SIZE}-byte header, so a page this small is mostly header" ), )); } @@ -143,7 +154,7 @@ fn index_backends_into(columns: &Columns, persistence: Persistence, errors: &mut for index in columns.indexes.values().filter(|index| !index.is_unique) { match index.backend { IndexBackend::WorktablesIndex | IndexBackend::Arctic => {} - IndexBackend::Indexset | IndexBackend::Congee => { + IndexBackend::Indexset | IndexBackend::Congee | IndexBackend::FxHash => { errors.push(syn::Error::new( index.name.span(), format!( @@ -264,7 +275,9 @@ pub fn supported_key_types(backend: IndexBackend) -> Option<&'static [&'static s IndexBackend::Arctic => Some(&[ "String", "u8", "u16", "u32", "u64", "u128", "usize", "i8", "i16", "i32", "i64", "i128", ]), - IndexBackend::WorktablesIndex | IndexBackend::Indexset => None, + // A hash map indexes anything hashable, which every column type this + // macro accepts already is, so there is no list to check against. + IndexBackend::FxHash | IndexBackend::WorktablesIndex | IndexBackend::Indexset => None, } } @@ -290,6 +303,9 @@ pub fn all( ) -> Vec { let mut errors = Vec::new(); index_backends_into(columns, persistence, &mut errors); + if let Err(error) = validate_columnar_indexes(columns) { + errors.push(error); + } if let Err(error) = validate_page_size(config, persistence) { errors.push(error); } @@ -303,3 +319,106 @@ pub fn all( } errors } + +/// Columnar indexes must cluster by columnar fields that exist and must not +/// collide with a columnar field's own generated scan methods. +pub fn validate_columnar_indexes(columns: &Columns) -> syn::Result<()> { + for primary_key in &columns.primary_keys { + if columns.columnar_fields.contains_key(primary_key) { + return Err(syn::Error::new( + primary_key.span(), + "the primary key participates in columnar identity implicitly and must not declare `columnar`", + )); + } + } + + if !columns.columnar_indexes.is_empty() && columns.columnar_fields.is_empty() { + return Err(syn::Error::new( + proc_macro2::Span::call_site(), + "`columnar_indexes` requires at least one field declaring `columnar`", + )); + } + + for index in columns.columnar_indexes.values() { + if columns.columnar_fields.contains_key(&index.name) { + return Err(syn::Error::new( + index.name.span(), + format!( + "columnar index `{}` conflicts with a columnar field name and would generate duplicate scan methods", + index.name + ), + )); + } + for field in &index.cluster_by { + if !columns.columns_map.contains_key(field) { + return Err(syn::Error::new( + field.span(), + format!("columnar index `{}` references unknown field `{field}`", index.name), + )); + } + if !columns.columnar_fields.contains_key(field) { + return Err(syn::Error::new( + field.span(), + format!( + "columnar index `{}` requires field `{field}` to declare `columnar(...)`", + index.name + ), + )); + } + } + } + Ok(()) +} + +/// Reject paged query shapes for which no operation is generated. Vec queries +/// have a separate synchronous implementation and must not inherit these limits. +pub fn validate_query_storage( + columns: &Columns, + queries: &crate::model::Queries, + storage: crate::model::Storage, +) -> syn::Result<()> { + if storage.is_vec() { + if let Some(profile) = queries + .update_runtime + .as_ref() + .or(queries.delete_runtime.as_ref()) + .or(queries.in_place_runtime.as_ref()) + { + return Err(syn::Error::new( + profile.span(), + "vec tables are synchronous and cannot schedule a query runtime profile", + )); + } + return Ok(()); + } + for (name, op) in &queries.updates { + let by_primary = columns.primary_keys.len() == 1 && columns.primary_keys.first() == Some(&op.by); + let by_index = columns.indexes.values().any(|index| index.field == op.by); + if !by_primary && !by_index { + return Err(syn::Error::new( + op.by.span(), + format!( + "update query `{name}` requires a single-column primary key or a secondary index on `{}`", + op.by + ), + )); + } + } + for (name, op) in &queries.in_place { + if columns.primary_keys.len() != 1 || columns.primary_keys.first() != Some(&op.by) { + return Err(syn::Error::new( + op.by.span(), + format!( + "in_place query `{name}` requires selection by the single-column primary key; use an update query for an indexed predicate" + ), + )); + } + if op.columns.iter().any(|column| columns.primary_keys.contains(column)) { + return Err(syn::Error::new( + name.span(), + "in_place queries cannot mutate primary key columns; use an update query to maintain indexes", + )); + } + } + Ok(()) +} diff --git a/dsl/tests/check.rs b/dsl/tests/check.rs index f10dc2a0..2e882e94 100644 --- a/dsl/tests/check.rs +++ b/dsl/tests/check.rs @@ -59,7 +59,7 @@ fn every_broken_rule_is_reported() { persist: true, columns: { id: u64 primary_key, label: String }, indexes: { label_idx: label unique using congee }, - config: { page_size: 4096 }", + config: { page_size: 64 }", ); assert!(checked.schema.is_some()); diff --git a/dsl/tests/cli.rs b/dsl/tests/cli.rs index bfcfe6b6..2358caa0 100644 --- a/dsl/tests/cli.rs +++ b/dsl/tests/cli.rs @@ -71,10 +71,13 @@ fn scan_finds_declarations_in_a_function_body() { /// declaration". /// /// The difference is not academic and this test exists because the first -/// version of the binary got it wrong. `page_size: 4096` beside -/// `persist: true` parses perfectly: it is a well-formed declaration. The -/// macro refuses it, because the on-disk layer hardcodes 16384-byte pages and -/// any other value reads and writes the wrong file offsets. +/// version of the binary got it wrong. `page_size: 64` parses perfectly: it is +/// a well-formed declaration. The macro refuses it, because a persisted page +/// that small is mostly its own 28-byte header. +/// +/// The fixture used to be `page_size: 4096` beside `persist: true`, refused +/// while the on-disk layer hardcoded its stride. It takes the stride as a +/// parameter now, so that combination is accepted and tests nothing. /// /// A round trip built on `Schema::parse` therefore reports success for output /// that does not compile, which is precisely the mistake a second @@ -82,7 +85,7 @@ fn scan_finds_declarations_in_a_function_body() { /// to stop. #[test] fn a_declaration_the_macro_refuses_is_not_a_successful_round_trip() { - let refused = "name: T, persist: true, columns: { id: u64 primary_key }, config: { page_size: 4096 },"; + let refused = "name: T, persist: true, columns: { id: u64 primary_key }, config: { page_size: 64 },"; // It parses. That is the trap. assert!( diff --git a/dsl/tests/columnar_schema.rs b/dsl/tests/columnar_schema.rs new file mode 100644 index 00000000..8379aeed --- /dev/null +++ b/dsl/tests/columnar_schema.rs @@ -0,0 +1,89 @@ +use worktable_dsl::schema::{Change, Cost, Diff}; +use worktable_dsl::{Schema, check}; + +const SOURCE: &str = " + name: Metrics, + persist: false, + columns: { + id: u64 primary_key, + host: u64 columnar(chunk_rows(8), compression(none)), + timestamp: i64 columnar, + }, + columnar_indexes: { host_time: { cluster_by: [host, timestamp] } }, + config: { columnar_slot_id: ColumnSlotId16, columnar_chunk_rows: 32 }, +"; + +#[test] +fn columnar_survives_the_round_trip() { + let schema = Schema::parse(SOURCE).unwrap(); + assert_eq!(schema.columns.iter().filter(|c| c.columnar.is_some()).count(), 2); + assert_eq!( + schema.column("host").unwrap().columnar.as_ref().unwrap().chunk_rows, + Some(8) + ); + assert_eq!(schema.columnar_indexes[0].cluster_by, ["host", "timestamp"]); + assert_eq!(schema.config.columnar_chunk_rows, Some(32)); + assert_eq!(schema.config.columnar_slot_id.as_deref(), Some("ColumnSlotId16")); + assert_eq!(schema, Schema::parse(&schema.to_dsl()).unwrap()); + assert!(check(&schema.to_dsl()).is_acceptable()); +} + +#[test] +fn every_derived_layout_change_requires_a_rebuild() { + let stored = Schema::parse(SOURCE).unwrap(); + let mut variants = vec![stored.clone(); 4]; + variants[0].config.columnar_chunk_rows = Some(64); + variants[1].config.columnar_slot_id = Some("ColumnSlotId32".into()); + variants[2].columnar_indexes[0].cluster_by.reverse(); + variants[3].columns[1].columnar.as_mut().unwrap().chunk_rows = Some(16); + for declared in variants { + let diff = Diff::between(&stored, &declared); + assert!(diff.changes.contains(&Change::ColumnarChanged), "{}", diff.describe()); + assert_eq!(diff.cost(), Cost::RebuildIndexes); + assert!(diff.rows_are_readable()); + } +} + +#[test] +fn checker_rejects_a_cluster_key_that_is_not_columnar() { + let checked = check( + "name: Bad, persist: false, + columns: { id: u64 primary_key, value: u64 columnar, other: u64 }, + columnar_indexes: { bad: { cluster_by: [other] } }", + ); + assert!(checked.schema.is_some()); + assert!( + checked.diagnostics.iter().any(|d| d.message.contains("requires field")), + "{:?}", + checked.diagnostics + ); +} + +#[test] +fn changed_page_size_cannot_reuse_row_links() { + let stored = Schema::parse("name: Rows, columns: { id: u64 primary_key }, config: { page_size: 8192 }").unwrap(); + let declared = Schema::parse("name: Rows, columns: { id: u64 primary_key }, config: { page_size: 32768 }").unwrap(); + let diff = Diff::between(&stored, &declared); + assert_eq!(diff.cost(), Cost::RewriteRows); + assert!(!diff.rows_are_readable()); + assert!(diff.describe().contains("page size")); +} + +#[test] +fn changing_storage_requires_an_explicit_conversion() { + let stored = Schema::parse("name: Rows, columns: { id: u64 primary_key }").unwrap(); + let declared = Schema::parse("name: Rows, vec: true, columns: { id: u64 primary_key }").unwrap(); + let diff = Diff::between(&stored, &declared); + assert_eq!(diff.cost(), Cost::NeedsIntent); + assert!(!diff.rows_are_readable()); + assert!(diff.describe().contains("row storage")); +} + +#[test] +fn changing_runtime_is_reported_without_a_row_rewrite() { + let stored = Schema::parse(SOURCE).unwrap(); + let declared = Schema::parse(&format!("{SOURCE} runtime: nagoya(spread),")).unwrap(); + let diff = Diff::between(&stored, &declared); + assert_eq!(diff.changes, vec![Change::RuntimeChanged]); + assert_eq!(diff.cost(), Cost::Nothing); +} diff --git a/dsl/tests/diff.rs b/dsl/tests/diff.rs index 812acba8..b072349b 100644 --- a/dsl/tests/diff.rs +++ b/dsl/tests/diff.rs @@ -265,8 +265,23 @@ fn a_changed_partition_key_needs_a_person() { // cannot be recomputed from the row: it is only knowable from where the // row already is. let stored = parse("name: Price, columns: { id: u64 primary_key, bid: f64 }"); - let declared = - parse("name: Price, version: 2, partition_by: shard: u32, columns: { id: u64 primary_key, bid: f64 }"); + let declared = parse( + "name: Price, version: 2, partition_by: shard: u32, partition_max_size: u64, columns: { id: u64 primary_key, bid: f64 }", + ); + assert_eq!(Diff::between(&stored, &declared).cost(), Cost::NeedsIntent); +} + +#[test] +fn a_changed_partition_size_needs_a_person() { + // The width is not decoration: it selects which table is generated per + // partition, and narrowing it caps a partition that was uncapped. Neither + // is recoverable from stored rows, so it lands where a changed key does. + let stored = parse( + "name: Price, partition_by: shard: u32, partition_max_size: u64, columns: { id: u64 primary_key, bid: f64 }", + ); + let declared = parse( + "name: Price, version: 2, partition_by: shard: u32, partition_max_size: u8, columns: { id: u64 primary_key, bid: f64 }", + ); assert_eq!(Diff::between(&stored, &declared).cost(), Cost::NeedsIntent); } diff --git a/dsl/tests/query_storage.rs b/dsl/tests/query_storage.rs new file mode 100644 index 00000000..48fa48b6 --- /dev/null +++ b/dsl/tests/query_storage.rs @@ -0,0 +1,29 @@ +use worktable_dsl::check::check; + +#[test] +fn paged_mutation_shapes_fail_before_emission() { + for query in [ + "update: { Change(value) by value }", + "in_place: { Change(value) by value }", + "in_place: { Change(id) by id }", + ] { + let checked = check(&format!( + "name: T, columns: {{ id: u64 primary_key, value: u64 }}, queries: {{ {query} }}" + )); + assert!(!checked.diagnostics.is_empty(), "{query} should be rejected"); + } +} +#[test] +fn a_vec_query_cannot_silently_ignore_a_runtime_profile() { + let checked = check( + "name: T, vec: true, columns: { id: u64 primary_key, value: u64 }, queries: { update runtime scheduled: { Change(value) by id } }", + ); + assert!(checked.diagnostics.iter().any(|d| d.message.contains("synchronous"))); +} +#[test] +fn supported_paged_index_updates_remain_valid() { + let checked = check( + "name: T, columns: { id: u64 primary_key, value: u64, amount: u64 }, indexes: { value_idx: value }, queries: { update: { Change(amount) by value } }", + ); + assert!(checked.diagnostics.is_empty(), "{:?}", checked.diagnostics); +} diff --git a/dsl/tests/round_trip.rs b/dsl/tests/round_trip.rs index 78fffdaa..6fc0d934 100644 --- a/dsl/tests/round_trip.rs +++ b/dsl/tests/round_trip.rs @@ -20,11 +20,20 @@ use std::path::{Path, PathBuf}; use worktable_dsl::{Schema, declarations_in_source}; +/// `tests/ui` is a corpus of declarations the macro must **refuse**, so the +/// scanner has to skip it. Reading it would assert the opposite of what those +/// files are for, and the failure would read as a parser bug rather than as the +/// harness finding exactly what it was built to find. +const NOT_A_CORPUS: &str = "ui"; + fn rust_files(root: &Path, out: &mut Vec) { let Ok(entries) = fs::read_dir(root) else { return }; for entry in entries.flatten() { let path = entry.path(); if path.is_dir() { + if path.file_name().is_some_and(|name| name == NOT_A_CORPUS) { + continue; + } rust_files(&path, out); } else if path.extension().is_some_and(|extension| extension == "rs") { out.push(path); @@ -113,3 +122,55 @@ fn reading_the_same_declaration_twice_gives_the_same_schema() { assert_eq!(first, second); assert_eq!(first.to_dsl(), second.to_dsl()); } + +/// Every top-level block survives the emitter. +/// +/// **The corpus round trip above cannot catch a dropped block.** Its property +/// is `parse(emit(parse(s))) == parse(s)`, stated on `Schema`, so anything +/// `Schema` does not model is dropped symmetrically and compares equal. That is +/// not hypothetical: `columnar_indexes` was parsed into the model, discarded +/// when the `Schema` was built, never emitted, and the corpus test reported +/// success on the very declarations that use it. +/// +/// So this states the property against the **text** instead. One minimal +/// declaration per block, and the block's keyword has to come back out. It is +/// coarse on purpose: a check that understood the contents would be the same +/// code as the emitter and would agree with it for the same reasons. +#[test] +fn every_top_level_block_survives_the_emitter() { + // Each case is the smallest declaration that legally uses its block. + let cases: [(&str, &str); 6] = [ + ("columns", "name: A, columns: { id: u64 primary_key }"), + ( + "indexes", + "name: A, columns: { id: u64 primary_key, x: u64 }, indexes: { x_idx: x }", + ), + ( + "columnar_indexes", + "name: A, columns: { id: u64 primary_key, a: u32 columnar, b: i64 columnar }, \ + columnar_indexes: { ab: { cluster_by: [a, b], }, }", + ), + ( + "queries", + "name: A, columns: { id: u64 primary_key, x: u64 }, queries: { update: { X(x) by id } }", + ), + ( + "config", + "name: A, columns: { id: u64 primary_key }, config: { page_size: 16384, }", + ), + ("runtime", "name: A, runtime: tokio, columns: { id: u64 primary_key }"), + ]; + + for (block, source) in cases { + let schema = Schema::parse(source).unwrap_or_else(|error| panic!("`{block}` case does not parse: {error}")); + let emitted = schema.to_dsl(); + assert!( + emitted.contains(block), + "the emitter dropped `{block}`. It parsed, so the loss is between the model and the \ + text, which is where `columnar_indexes` was lost.\nsource:{source}\nemitted:\n{emitted}" + ); + let reparsed = + Schema::parse(&emitted).unwrap_or_else(|error| panic!("`{block}` case does not re-parse: {error}")); + assert_eq!(schema, reparsed, "`{block}` case changed across a round trip"); + } +} diff --git a/dsl/tests/schema.rs b/dsl/tests/schema.rs index fb0ea36a..aeeaa9fc 100644 --- a/dsl/tests/schema.rs +++ b/dsl/tests/schema.rs @@ -107,6 +107,7 @@ fn a_schema_survives_a_trip_through_serde() { version: 4, persist: true, partition_by: shard: u32, + partition_max_size: u64, columns: { id: u64 primary_key autoincrement using congee, payload: String optional, diff --git a/dsl/tests/uml.rs b/dsl/tests/uml.rs index b584e3ad..2989df3e 100644 --- a/dsl/tests/uml.rs +++ b/dsl/tests/uml.rs @@ -71,11 +71,12 @@ fn mermaid_puts_the_partition_key_in_a_note_not_a_column() { " name: Price, partition_by: symbol_id: u16, + partition_max_size: u64, columns: { exchange_id: u8 primary_key, bid: f64 } ", ); let diagram = schema.to_mermaid(); - assert!(diagram.contains("note for Price \"partitioned by symbol_id: u16\"")); + assert!(diagram.contains("note for Price \"partitioned by symbol_id: u16, unbounded\"")); assert!(!diagram.contains("symbol_id : u16")); } diff --git a/examples/guide_check.rs b/examples/guide_check.rs new file mode 100644 index 00000000..b853013a --- /dev/null +++ b/examples/guide_check.rs @@ -0,0 +1,98 @@ +//! Executable examples for the callsites in `docs/wt-user-guide.typ`. +//! If the guide drifts from the API, this stops compiling. +use worktable::prelude::*; +use worktable::worktable; + +worktable! ( + name: Order, + columns: { + id: u64 primary_key autoincrement, + symbol: String, + quantity: u64, + }, + indexes: { + symbol_idx: symbol, + }, + queries: { + update: { QuantityById(quantity) by id, } + } +); + +worktable!( + name: Reading, + columns: { + id: u64 primary_key, + host_id: u64 columnar, + timestamp: u64 columnar, + }, + columnar_indexes: { host_time: { cluster_by: [host_id, timestamp], } } +); + +worktable!( + name: Snapshot, + vec: true, + columns: { id: u64 primary_key using fxhash, quantity: u64, } +); + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let table = OrderWorkTable::default(); + table + .insert(OrderRow { + id: table.get_next_pk().into(), + symbol: "ETH".into(), + quantity: 3, + }) + .await?; + let found = table.select_by_symbol("ETH".into()).execute()?; + assert_eq!(found.len(), 1); + table + .insert_many(vec![OrderRow { + id: 100, + symbol: "BTC".into(), + quantity: 5, + }]) + .await?; + table + .update_quantity_by_id(QuantityByIdQuery { quantity: 7 }, 100) + .await?; + assert_eq!(table.select(100).unwrap().quantity, 7); + assert_eq!(table.select_all().limit(1).execute()?.len(), 1); + assert_eq!(table.row_count(), 2); + assert!(table.used_bytes() > 0); + let _ = table.system_info(); + table.delete_many(vec![100u64]).await?; + let vacuum = table.vacuum_with_pacing(VacuumPacing { + batch_pages: 1, + ..Default::default() + }); + vacuum.vacuum().await?; + assert_eq!(vacuum.diagnostics().completions, 1); + + let readings = ReadingWorkTable::default(); + readings + .insert(ReadingRow { + id: 1, + host_id: 7, + timestamp: 1000, + }) + .await?; + let refs = readings.columnar_select_host_time(7, 1000)?; + assert_eq!(readings.columnar_project_timestamp(&refs)?[0].1, 1000); + assert_eq!(readings.columnar_scan_host_id()?.len(), 1); + assert_eq!(readings.columnar_scan_host_time()?.len(), 1); + assert_eq!(readings.columnar_slots_in_use(), 1); + readings.rebuild_columnar()?; + assert_eq!(readings.columnar_project_timestamp(&refs)?.len(), 1); + readings.delete(1).await?; + assert!(readings.columnar_project_timestamp(&refs)?.is_empty()); + + let mut snapshot = SnapshotWorkTable::with_capacity(4); + snapshot.insert(SnapshotRow { id: 1, quantity: 3 }).unwrap(); + assert_eq!(snapshot.select(&1).unwrap().quantity, 3); + snapshot.delete(&1).expect("existing row"); + assert_eq!(snapshot.ghost_count(), 1); + snapshot.compact(); + assert!(snapshot.is_empty()); + Ok(()) +} diff --git a/examples/system_info_render.rs b/examples/system_info_render.rs new file mode 100644 index 00000000..a6e23aa2 --- /dev/null +++ b/examples/system_info_render.rs @@ -0,0 +1,24 @@ +use worktable::prelude::*; +use worktable::worktable; + +worktable! ( + name: Shown, + columns: { id: u64 primary_key autoincrement, symbol: String, qty: u64 }, + indexes: { symbol_idx: symbol, qty_idx: qty } +); + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let table = ShownWorkTable::default(); + for i in 0..5_000u64 { + table + .insert(ShownRow { + id: table.get_next_pk().into(), + symbol: format!("SYM{}", i % 97), + qty: i, + }) + .await?; + } + print!("{}", table.system_info()); + Ok(()) +} diff --git a/scripts/build-guides.sh b/scripts/build-guides.sh new file mode 100755 index 00000000..21b68908 --- /dev/null +++ b/scripts/build-guides.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Canonical Typst sources; PDFs are local build artifacts. +set -eu +ROOT=$(cd "$(dirname "$0")/.." && pwd) +cd "$ROOT" +typst compile docs/wt-user-guide.typ docs/wt-user-guide.pdf +typst compile docs/why-worktables.typ docs/why-worktables.pdf diff --git a/scripts/check-no-std.sh b/scripts/check-no-std.sh new file mode 100644 index 00000000..8fd4e9c8 --- /dev/null +++ b/scripts/check-no-std.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Check a supported OS target with Rust std deliberately absent from its sysroot. +set -eu +root=$(pwd) +target=${NO_STD_TARGET:-$(rustc -vV | awk '/^host:/ {print $2}')} +libdir=$(rustc --print target-libdir --target "$target") +scratch="$root/target/no-std-sysroot/$target" +mkdir -p "$scratch/lib/rustlib/$target/lib" +for library in "$libdir"/*; do + name=$(basename "$library") + case "$name" in + libstd-*|libstd_detect-*|libtest-*|libproc_macro-*|librustc_std_workspace_std-*|std-*|std_detect-*|test-*|proc_macro-*) + rm -f "$scratch/lib/rustlib/$target/lib/$name" + continue ;; + esac + ln -sf "$library" "$scratch/lib/rustlib/$target/lib/$name" +done +printf '%s\n' 'pub fn forbidden() { std::mem::drop(1u8); }' > "$scratch/negative.rs" +if rustc --crate-type lib --emit metadata --target "$target" --sysroot "$scratch" "$scratch/negative.rs" -o "$scratch/negative.rmeta" > "$scratch/negative.log" 2>&1; then + echo 'std unexpectedly available in negative control' >&2 + exit 1 +fi +if ! grep -q "can't find crate for .*std" "$scratch/negative.log"; then + cat "$scratch/negative.log" >&2 + exit 1 +fi +printf '%s\n' '#![no_std]' 'extern crate alloc;' 'pub fn allowed(v: alloc::vec::Vec) -> usize { v.len() }' > "$scratch/positive.rs" +rustc --crate-type lib --emit metadata --target "$target" --sysroot "$scratch" "$scratch/positive.rs" -o "$scratch/positive.rmeta" +# Explicit --target keeps proc macros and build scripts on the ordinary host sysroot. +CARGO_ENCODED_RUSTFLAGS=$(printf '%s\037%s' --sysroot "$scratch") +export CARGO_ENCODED_RUSTFLAGS +cargo check --target "$target" "$@" diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index a667366b..2976e045 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -57,6 +57,20 @@ echo "=== build and test (all-features) ===" run cargo build --workspace --all-targets --all-features run cargo test --workspace --all-targets --all-features +echo "=== cell-lock concurrency models ===" +run env "RUSTFLAGS=--cfg wt_loom" CARGO_TARGET_DIR=target/cell-lock-loom cargo test --release --lib cell_lock_models + +echo "=== library without default features ===" +run sh scripts/check-no-std.sh -p worktable --lib --no-default-features +run sh scripts/check-no-std.sh --manifest-path tests/nostd-consumer/Cargo.toml +run cargo test --manifest-path tests/nostd-consumer/Cargo.toml +run rustup target add x86_64-pc-windows-gnu +run env NO_STD_TARGET=x86_64-pc-windows-gnu CARGO_TARGET_DIR=target/no-std-cross sh scripts/check-no-std.sh -p worktable --lib --no-default-features +for search in wti-predictable-search wti-hybrid-search wti-std-search; do + run sh scripts/check-no-std.sh -p worktable --lib --no-default-features --features "$search,logical-index-persistence,versioned-row-publication,runtime-backends" +done +run cargo clippy -p worktable --lib --no-default-features -- -D warnings + echo "=== clippy (default) ===" run cargo clippy --workspace --all-targets -- -D warnings diff --git a/src/atomic_key_table.rs b/src/atomic_key_table.rs new file mode 100644 index 00000000..7272618a --- /dev/null +++ b/src/atomic_key_table.rs @@ -0,0 +1,380 @@ +//! A fixed-capacity table whose rows are claimed without blocking and updated +//! in place. +//! +//! # What this is for, and why the other two storages cannot do it +//! +//! A `storage: vec` table takes `&mut self` to insert, because it pushes onto a +//! `Vec` and a push can reallocate, which moves every row. That is the right +//! shape for a single writer and unusable from several threads at once. A +//! `storage: paged` table is usable from several threads and buys that with an +//! archived row, links into pages, a row-level lock map and change-data-capture. +//! +//! The case that needs neither is a **counter table**: many writers, a small set +//! of keys that stabilises almost immediately, and an update that is a +//! read-modify-write on the row rather than a replacement of it. Performance +//! measurement is the archetype, sixteen workers recording timings against a +//! handful of named sites. +//! +//! # How it avoids both a lock and a reallocation +//! +//! Capacity is fixed at construction and every value is built then, so **no row +//! ever moves** and a `&V` handed out stays valid for the life of the table. A +//! key is claimed with one `compare_exchange`; after that, finding it is a plain +//! load. The value is updated through `&V`, so `V` supplies its own interior +//! mutability, and this module takes no position on what a row contains. +//! +//! The load-before-claim order is the point. Claiming with a `compare_exchange` +//! on every lookup takes the cache line exclusively even when nothing changes, +//! so writers contending for a key they all already own would serialise on it. +//! A relaxed load is a shared read. +//! +//! # There is no row snapshot, deliberately +//! +//! A row is `&V` and `V` supplies its own interior mutability, so a reader that +//! wants two fields reads two atomics and there is no instant at which it held +//! both. A count of 10 beside a total of 900 can be observed even though no +//! writer ever left the row in that state. +//! +//! That is accepted rather than fixed. The alternatives are a sequence lock or +//! a lock per row, and both put back the contended cache line this type exists +//! to avoid: the whole point of claiming a slot once and then never touching +//! the key again is that a hot row is a shared read. +//! +//! **If you need two values to agree, pack them into one atomic.** Two `u32` +//! counters in an `AtomicU64` are updated with one `fetch_add` of +//! `1 << 32 | delta` and read with one load, and they are then exactly as +//! consistent as each other. That is the supported answer, and it is enough for +//! the case this exists for: a count and a total. +//! +//! # What it does not do +//! +//! No removal, no resize, and no iteration order beyond slot order. A full table +//! refuses rather than growing, and [`AtomicKeyTable::len`] says how many slots +//! are taken so a caller can see it coming. +//! +//! Ported from `worktable-vec`, which this supersedes. That crate is deprecated +//! and this was the last thing in it that lived nowhere else. + +use alloc::vec::Vec; +use core::sync::atomic::{AtomicUsize, Ordering}; + +// A 64-bit `usize`, and it refuses rather than assuming one. `GOLDEN` below is +// a 64-bit constant, and truncating it to 32 bits leaves an even number, which +// is not invertible and quietly collapses keys onto the same slot. Nothing here +// is built or tested for a narrower target, so the honest answer is to say so +// at compile time instead of carrying a second constant nobody exercises. +#[cfg(not(target_pointer_width = "64"))] +compile_error!("`AtomicKeyTable` requires a 64-bit target"); + +/// Scatter a key across the table. +/// +/// # Why not the low bits, and why not a modulo +/// +/// The first version of this in `worktable-vec` shifted the key right by four +/// and took it modulo the capacity, which is wrong twice. The shift assumed a +/// pointer key, whose low bits are alignment zeros; handed small integers it +/// maps every key under sixteen to slot zero, and a measured lookup over +/// sixty-four sequential keys walked a probe chain 9.3x slower than a linear +/// scan of the same rows. The modulo is an integer division on the hottest path. +/// +/// Fibonacci hashing fixes the first: multiplying by the golden ratio spreads +/// any input across the whole word, and taking the **high** bits reads that +/// spread. A power-of-two capacity fixes the second: the index is then a mask. +#[inline(always)] +const fn scatter(key: usize, shift: u32, mask: usize) -> usize { + // 2^64 / phi, odd so the multiply is invertible and no input is lost. + const GOLDEN: usize = 0x9E37_79B9_7F4A_7C15u64 as usize; + (key.wrapping_mul(GOLDEN) >> shift) & mask +} + +/// Open-addressed slots with linear probing, sized once and never resized. +#[derive(Debug)] +pub struct AtomicKeyTable { + keys: Vec, + values: Vec, + /// Capacity is a power of two, so the index is a mask rather than a division. + mask: usize, + shift: u32, +} + +impl AtomicKeyTable { + /// A table with at least `capacity` slots, every value built now. + /// + /// Rounded up to a power of two so the slot index is a mask rather than a + /// division. Size it generously: this is open addressed with linear + /// probing, so a table much past half full costs a long probe on every miss. + #[must_use] + pub fn with_capacity(capacity: usize) -> Self { + let slots = capacity.max(1).next_power_of_two(); + let mut keys = Vec::with_capacity(slots); + let mut values = Vec::with_capacity(slots); + for _ in 0..slots { + keys.push(AtomicUsize::new(0)); + values.push(V::default()); + } + Self { + keys, + values, + mask: slots - 1, + shift: usize::BITS - slots.trailing_zeros(), + } + } +} + +impl AtomicKeyTable { + /// The row for this key, claiming a slot if it has none yet. + /// + /// It returns the existing row or creates one, and never replaces what is + /// there. The row is then updated through `&V`, which is where this differs + /// from a paged `upsert`: the value carries its own interior mutability + /// rather than being written back whole. + /// + /// `None` means the table is full. Zero is the empty sentinel and is + /// rejected rather than silently colliding with an unclaimed slot. + pub fn upsert(&self, key: usize) -> Option<&V> { + if key == 0 || self.keys.is_empty() { + return None; + } + let capacity = self.keys.len(); + let mut at = scatter(key, self.shift, self.mask); + for _ in 0..capacity { + match self.keys[at].load(Ordering::Acquire) { + existing if existing == key => return Some(&self.values[at]), + 0 => match self.keys[at].compare_exchange(0, key, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => return Some(&self.values[at]), + Err(taken) if taken == key => return Some(&self.values[at]), + Err(_) => at = (at + 1) & self.mask, + }, + _ => at = (at + 1) & self.mask, + } + } + None + } + + /// The row for this key, or `None` if no row has been created for it. + /// Never creates one. + pub fn select(&self, key: usize) -> Option<&V> { + if key == 0 || self.keys.is_empty() { + return None; + } + let capacity = self.keys.len(); + let mut at = scatter(key, self.shift, self.mask); + for _ in 0..capacity { + match self.keys[at].load(Ordering::Acquire) { + existing if existing == key => return Some(&self.values[at]), + 0 => return None, + _ => at = (at + 1) & self.mask, + } + } + None + } + + /// Every claimed row, in slot order. + pub fn iter(&self) -> impl Iterator { + self.keys + .iter() + .zip(self.values.iter()) + .filter_map(|(k, v)| match k.load(Ordering::Acquire) { + 0 => None, + key => Some((key, v)), + }) + } + + /// How many rows the table holds. + /// + /// A walk of every slot, not a counter. Claiming is a `compare_exchange` on + /// one slot and nothing else, and a shared counter beside it would put back + /// the contended line the design exists to avoid. + #[must_use] + pub fn len(&self) -> usize { + self.iter().count() + } + + /// Whether any row has been created. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// How many rows the table can hold. Fixed at construction. + #[must_use] + pub fn capacity(&self) -> usize { + self.keys.len() + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::{AtomicU64, Ordering}; + + use super::*; + + #[derive(Default)] + struct Counter(AtomicU64); + + #[test] + fn a_claimed_row_is_found_by_a_plain_load_and_never_reclaimed() { + let table: AtomicKeyTable = AtomicKeyTable::with_capacity(64); + let first = table.upsert(7).expect("capacity"); + first.0.fetch_add(1, Ordering::Relaxed); + let again = table.upsert(7).expect("already claimed"); + again.0.fetch_add(1, Ordering::Relaxed); + assert_eq!(again.0.load(Ordering::Relaxed), 2, "the second call found the same row"); + assert_eq!(table.len(), 1, "one key claimed one slot"); + } + + #[test] + fn zero_is_the_empty_sentinel_and_is_refused_rather_than_colliding() { + let table: AtomicKeyTable = AtomicKeyTable::with_capacity(8); + assert!(table.upsert(0).is_none(), "zero would be indistinguishable from empty"); + assert!(table.select(0).is_none()); + assert_eq!(table.len(), 0); + } + + #[test] + fn a_full_table_refuses_rather_than_growing() { + let table: AtomicKeyTable = AtomicKeyTable::with_capacity(4); + for key in 1..=4 { + assert!(table.upsert(key).is_some(), "slot {key} fits"); + } + assert_eq!(table.len(), 4); + assert!(table.upsert(5).is_none(), "the fifth has nowhere to go"); + assert!(table.upsert(3).is_some(), "a claimed key is still reachable when full"); + } + + #[test] + fn select_never_creates_a_row() { + let table: AtomicKeyTable = AtomicKeyTable::with_capacity(8); + assert!(table.select(9).is_none()); + assert_eq!(table.len(), 0, "select must not take a slot"); + table.upsert(9).expect("capacity"); + assert!(table.select(9).is_some()); + } + + #[test] + fn every_claimed_row_is_iterated_and_no_empty_one_is() { + let table: AtomicKeyTable = AtomicKeyTable::with_capacity(32); + for key in [11usize, 22, 33] { + table + .upsert(key) + .expect("capacity") + .0 + .store(key as u64, Ordering::Relaxed); + } + let mut seen: Vec<(usize, u64)> = table.iter().map(|(k, v)| (k, v.0.load(Ordering::Relaxed))).collect(); + seen.sort_unstable(); + assert_eq!(seen, alloc::vec![(11usize, 11u64), (22, 22), (33, 33)]); + } + + /// Small sequential keys must not all land in one slot. + /// + /// The regression `scatter` exists for. The first version of this, in + /// `worktable-vec`, shifted the key right by four and took it modulo the + /// capacity. The shift assumes a pointer key whose low bits are alignment + /// zeros; handed small integers it maps **every key under sixteen onto slot + /// zero**, and a measured lookup over sixty-four sequential keys ran 9.3x + /// slower than a linear scan of the same rows. + /// + /// Asserted on the slot distribution rather than through the public API, + /// because the public API cannot tell the difference: every key is findable + /// either way, and what breaks is only how far each lookup walks. This + /// module's own test can see the private index, so it checks the thing that + /// actually went wrong. + #[test] + fn small_sequential_keys_land_on_distinct_slots() { + let table: AtomicKeyTable = AtomicKeyTable::with_capacity(256); + + let mut slots: Vec = (1..=64usize).map(|key| scatter(key, table.shift, table.mask)).collect(); + slots.sort_unstable(); + slots.dedup(); + + // 64 keys into 256 slots: by the birthday bound a good scatter leaves + // roughly 57 distinct, and the broken one leaves exactly 1. Anything + // above half is unambiguously the former. + assert!( + slots.len() > 32, + "64 sequential keys landed on only {} distinct slots of 256; this is the \ + low-bits regression", + slots.len() + ); + + // And the keys the caller would actually use still all resolve. + for key in 1..=64usize { + table.upsert(key).expect("capacity"); + } + assert_eq!(table.len(), 64); + for key in 1..=64usize { + assert!(table.select(key).is_some(), "key {key} went missing"); + } + for key in 65..=128usize { + assert!(table.select(key).is_none(), "key {key} was never claimed"); + } + } + + #[test] + fn concurrent_writers_agree_on_one_row_per_key() { + extern crate std; + let table: AtomicKeyTable = AtomicKeyTable::with_capacity(512); + let shared = &table; + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(move || { + for round in 0..1_000usize { + let key = (round % 16) + 1; + shared.upsert(key).expect("capacity").0.fetch_add(1, Ordering::Relaxed); + } + }); + } + }); + assert_eq!( + table.len(), + 16, + "sixteen keys, sixteen slots, whatever the interleaving" + ); + let total: u64 = table.iter().map(|(_, v)| v.0.load(Ordering::Relaxed)).sum(); + assert_eq!(total, 8 * 1_000, "no update was lost and none was double counted"); + } + + /// The documented way to make two values agree: pack them into one atomic. + /// + /// There is no row snapshot and there will not be one, so this is the + /// supported answer and it is worth having a worked example of it in the + /// tests rather than only in prose. + #[test] + fn two_values_packed_into_one_atomic_stay_consistent() { + extern crate std; + + /// Count in the high 32 bits, total in the low 32. + #[derive(Default)] + struct CountAndTotal(AtomicU64); + + impl CountAndTotal { + fn record(&self, value: u32) { + self.0.fetch_add((1u64 << 32) | u64::from(value), Ordering::Relaxed); + } + + /// One load, so the pair is exactly as consistent as each other. + fn read(&self) -> (u32, u32) { + let packed = self.0.load(Ordering::Relaxed); + ((packed >> 32) as u32, packed as u32) + } + } + + let table: AtomicKeyTable = AtomicKeyTable::with_capacity(64); + let shared = &table; + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(move || { + for _ in 0..500 { + shared.upsert(1).expect("capacity").record(3); + } + }); + } + }); + + let (count, total) = table.select(1).expect("claimed").read(); + assert_eq!(count, 4_000); + assert_eq!(total, 12_000); + assert_eq!(total, count * 3, "the pair was never observed disagreeing"); + } +} diff --git a/src/columnar.rs b/src/columnar.rs new file mode 100644 index 00000000..742c8dc9 --- /dev/null +++ b/src/columnar.rs @@ -0,0 +1,338 @@ +use alloc::collections::{BTreeMap, BTreeSet}; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; +use core::sync::atomic::{AtomicU64, Ordering}; + +use crate::mem_stat::MemStat; + +/// Compact position used by generated columnar storage. +/// +/// This is supplemental metadata, never a replacement for a WorkTable primary +/// key. Slot IDs are not sort keys and are not durable identities. +pub trait ColumnSlotId: Copy + Debug + Eq + Ord + Hash + Send + Sync + MemStat + 'static { + const BITS: u8; + + fn try_from_position(position: u64) -> Option; + fn position(self) -> u64; + + fn slot(self) -> usize { + usize::try_from(self.position()).expect("column slot ID exceeds this target's address space") + } +} + +macro_rules! column_slot_id { + ($name:ident, $inner:ty, $bits:literal) => { + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] + pub struct $name($inner); + + impl ColumnSlotId for $name { + const BITS: u8 = $bits; + + fn try_from_position(position: u64) -> Option { + <$inner>::try_from(position).ok().map(Self) + } + + fn position(self) -> u64 { + self.0 as u64 + } + } + + impl MemStat for $name { + fn heap_size(&self) -> usize { + 0 + } + + fn used_size(&self) -> usize { + 0 + } + } + }; +} + +column_slot_id!(ColumnSlotId8, u8, 8); +column_slot_id!(ColumnSlotId16, u16, 16); +column_slot_id!(ColumnSlotId32, u32, 32); +column_slot_id!(ColumnSlotId64, u64, 64); + +static NEXT_COLUMNAR_INCARNATION: AtomicU64 = AtomicU64::new(1); + +/// Returns a process-local table incarnation used to invalidate retained +/// columnar references when a table is rebuilt or reopened. +#[doc(hidden)] +pub fn next_columnar_incarnation() -> u64 { + NEXT_COLUMNAR_INCARNATION + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) + .expect("columnar table incarnation space is exhausted") +} + +/// Identity carried by generated columnar query results. +/// +/// The primary key remains authoritative. The slot, generation, and table +/// incarnation are private validation metadata and are deliberately not +/// serializable or exposed as ordering keys. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ColumnarRowRef { + primary_key: PrimaryKey, + slot_id: SlotId, + generation: u64, + incarnation: u64, +} + +impl ColumnarRowRef { + /// Returns the authoritative WorkTable primary key. + pub fn primary_key(&self) -> &PrimaryKey { + &self.primary_key + } + + /// Constructor used by generated WorkTable code. + #[doc(hidden)] + pub fn __new(primary_key: PrimaryKey, slot_id: SlotId, generation: u64, incarnation: u64) -> Self { + Self { + primary_key, + slot_id, + generation, + incarnation, + } + } + + #[doc(hidden)] + pub fn __slot_id(&self) -> SlotId + where + SlotId: Copy, + { + self.slot_id + } + + #[doc(hidden)] + pub fn __generation(&self) -> u64 { + self.generation + } + + #[doc(hidden)] + pub fn __incarnation(&self) -> u64 { + self.incarnation + } +} + +impl MemStat for ColumnarRowRef { + fn heap_size(&self) -> usize { + self.primary_key.heap_size() + self.slot_id.heap_size() + } + + fn used_size(&self) -> usize { + self.primary_key.used_size() + self.slot_id.used_size() + } +} + +/// Compression used by a generated columnar field. +/// +/// Mutable chunks are currently unencoded; unsupported policies are rejected +/// by the macro instead of being accepted as inert configuration. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum ColumnCompression { + #[default] + None, +} + +impl ColumnCompression { + pub fn is_encoded(self) -> bool { + false + } +} + +impl MemStat for ColumnCompression { + fn heap_size(&self) -> usize { + 0 + } + + fn used_size(&self) -> usize { + 0 + } +} + +/// Chunked storage for one generated columnar field. +#[derive(Debug)] +pub struct ColumnarColumn { + chunk_rows: usize, + compression: ColumnCompression, + chunks: Vec>>, +} + +impl ColumnarColumn { + pub fn new(chunk_rows: usize, compression: ColumnCompression) -> Self { + assert!(chunk_rows > 0, "columnar chunks cannot be empty"); + Self { + chunk_rows, + compression, + chunks: Vec::new(), + } + } + + pub fn chunk_rows(&self) -> usize { + self.chunk_rows + } + + pub fn compression(&self) -> ColumnCompression { + self.compression + } + + pub fn set(&mut self, slot_id: SlotId, value: T) { + let row = slot_id.slot(); + let chunk_index = row / self.chunk_rows; + let offset = row % self.chunk_rows; + while self.chunks.len() <= chunk_index { + self.chunks.push(Vec::new()); + } + let chunk = &mut self.chunks[chunk_index]; + if chunk.len() <= offset { + chunk.resize_with(offset + 1, || None); + } + chunk[offset] = Some(value); + } + + pub fn remove(&mut self, slot_id: SlotId) -> Option { + let row = slot_id.slot(); + self.chunks + .get_mut(row / self.chunk_rows) + .and_then(|chunk| chunk.get_mut(row % self.chunk_rows)) + .and_then(Option::take) + } + + pub fn get(&self, slot_id: SlotId) -> Option<&T> { + let row = slot_id.slot(); + self.chunks + .get(row / self.chunk_rows) + .and_then(|chunk| chunk.get(row % self.chunk_rows)) + .and_then(Option::as_ref) + } + + pub fn iter(&self) -> impl Iterator { + let chunk_rows = self.chunk_rows; + self.chunks.iter().enumerate().flat_map(move |(chunk_index, chunk)| { + chunk.iter().enumerate().filter_map(move |(offset, value)| { + value.as_ref().map(|value| { + let position = (chunk_index * chunk_rows + offset) as u64; + let slot_id = SlotId::try_from_position(position) + .expect("stored column position fits its configured column slot ID"); + (slot_id, value) + }) + }) + }) + } +} + +impl MemStat for ColumnarColumn { + fn heap_size(&self) -> usize { + self.chunks.heap_size() + } + + fn used_size(&self) -> usize { + self.chunks.used_size() + } +} + +/// Ordered metadata for one generated `columnar_indexes` declaration. +#[derive(Debug)] +pub struct ClusteredColumnarIndex { + rows: BTreeMap>, +} + +impl Default for ClusteredColumnarIndex { + fn default() -> Self { + Self { rows: BTreeMap::new() } + } +} + +impl ClusteredColumnarIndex { + pub fn insert(&mut self, key: K, slot_id: SlotId) { + self.rows.entry(key).or_default().insert(slot_id); + } + + pub fn remove(&mut self, key: &K, slot_id: SlotId) { + let remove_key = self.rows.get_mut(key).is_some_and(|rows| { + rows.remove(&slot_id); + rows.is_empty() + }); + if remove_key { + self.rows.remove(key); + } + } + + pub fn exact(&self, key: &K) -> Vec { + self.rows + .get(key) + .map(|rows| rows.iter().copied().collect()) + .unwrap_or_default() + } + + pub fn ordered_slot_ids(&self) -> Vec { + self.rows.values().flat_map(|rows| rows.iter().copied()).collect() + } +} + +impl MemStat for ClusteredColumnarIndex { + fn heap_size(&self) -> usize { + self.rows.heap_size() + } + + fn used_size(&self) -> usize { + self.rows.used_size() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chunks_are_addressed_by_configured_slot_id() { + let mut column = ColumnarColumn::new(2, ColumnCompression::None); + column.set(ColumnSlotId16(3), 30); + column.set(ColumnSlotId16(0), 10); + + assert_eq!(column.chunk_rows(), 2); + assert_eq!(column.get(ColumnSlotId16(3)), Some(&30)); + assert_eq!( + column + .iter::() + .map(|(id, value)| (id.0, *value)) + .collect::>(), + [(0, 10), (3, 30)] + ); + + assert_eq!(column.remove(ColumnSlotId16(0)), Some(10)); + assert!(column.get(ColumnSlotId16(0)).is_none()); + } + + #[test] + fn clustered_index_preserves_key_order() { + let mut index = ClusteredColumnarIndex::default(); + index.insert((2, 1), ColumnSlotId8(1)); + index.insert((1, 9), ColumnSlotId8(2)); + index.insert((1, 9), ColumnSlotId8(0)); + + assert_eq!(index.exact(&(1, 9)), [ColumnSlotId8(0), ColumnSlotId8(2)]); + assert_eq!( + index.ordered_slot_ids(), + [ColumnSlotId8(0), ColumnSlotId8(2), ColumnSlotId8(1)] + ); + } + + #[test] + fn widths_have_expected_capacity_boundaries() { + assert_eq!(ColumnSlotId8::try_from_position(255), Some(ColumnSlotId8(255))); + assert_eq!(ColumnSlotId8::try_from_position(256), None); + assert_eq!(ColumnSlotId16::try_from_position(65_535), Some(ColumnSlotId16(65_535))); + assert_eq!(ColumnSlotId16::try_from_position(65_536), None); + assert_eq!( + ColumnSlotId32::try_from_position(u32::MAX as u64), + Some(ColumnSlotId32(u32::MAX)) + ); + assert_eq!(ColumnSlotId32::try_from_position(u32::MAX as u64 + 1), None); + assert_eq!( + ColumnSlotId64::try_from_position(u64::MAX), + Some(ColumnSlotId64(u64::MAX)) + ); + } +} diff --git a/src/features/database_s3.rs b/src/features/database_s3.rs new file mode 100644 index 00000000..118ebc08 --- /dev/null +++ b/src/features/database_s3.rs @@ -0,0 +1,599 @@ +//! Database-wide S3 persistence through DataBucket generations. + +use alloc::{format, string::String, vec::Vec}; +use core::fmt::{Debug, Formatter}; +use core::hash::Hash; +use core::marker::PhantomData; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{Read as _, Seek as _, SeekFrom, Write as _}; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use data_bucket::storage::{ + CatalogMutation, CatalogName, CatalogRecord, PageAddress, PageKind, SystemIndexRecord, TableId, +}; + +use crate::persistence::operation::{BatchOperation, Operation}; +use crate::persistence::{ + DiskConfig, DiskPersistenceEngine, PersistenceConfig, PersistenceEngine, SpaceDataOps, SpaceIndexOps, + SpaceSecondaryIndexOps, +}; +use crate::prelude::{PrimaryKeyGeneratorState, TablePrimaryKey, WT_DATA_EXTENSION, WT_INDEX_EXTENSION}; +use crate::{S3Database, TableSecondaryIndexEventsOps}; + +const INDEX_CHUNK_BYTES: usize = data_bucket::PAGE_SIZE; + +#[derive(Clone)] +pub struct DatabaseS3DiskConfig { + pub disk: DiskConfig, + pub database: S3Database, +} + +impl Debug for DatabaseS3DiskConfig { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("DatabaseS3DiskConfig") + .field("disk", &self.disk) + .field("domain", &self.database.id()) + .finish() + } +} + +impl PersistenceConfig for DatabaseS3DiskConfig { + fn table_path(&self) -> &str { + self.disk.table_path() + } + + fn version(&self) -> u32 { + self.disk.version() + } +} + +pub struct DatabaseS3PersistenceEngine< + SpaceData, + SpacePrimaryIndex, + SpaceSecondaryIndexes, + PrimaryKey, + SecondaryIndexEvents, + AvailableIndexes, + PrimaryKeyGenState = <::Generator as PrimaryKeyGeneratorState>::State, +> where + PrimaryKey: TablePrimaryKey, + ::Generator: PrimaryKeyGeneratorState, +{ + inner: DiskPersistenceEngine< + SpaceData, + SpacePrimaryIndex, + SpaceSecondaryIndexes, + PrimaryKey, + SecondaryIndexEvents, + AvailableIndexes, + PrimaryKeyGenState, + >, + config: DatabaseS3DiskConfig, + table_id: TableId, + marker: PhantomData<(PrimaryKey, SecondaryIndexEvents, AvailableIndexes, PrimaryKeyGenState)>, +} + +impl< + SpaceData, + SpacePrimaryIndex, + SpaceSecondaryIndexes, + PrimaryKey, + SecondaryIndexEvents, + AvailableIndexes, + PrimaryKeyGenState, +> + DatabaseS3PersistenceEngine< + SpaceData, + SpacePrimaryIndex, + SpaceSecondaryIndexes, + PrimaryKey, + SecondaryIndexEvents, + AvailableIndexes, + PrimaryKeyGenState, + > +where + PrimaryKey: Clone + Debug + Ord + TablePrimaryKey + Send + Sync, + ::Generator: PrimaryKeyGeneratorState, + SpaceData: SpaceDataOps + Send + Sync, + SpacePrimaryIndex: SpaceIndexOps + Send + Sync, + SpaceSecondaryIndexes: SpaceSecondaryIndexOps + Send + Sync, + SecondaryIndexEvents: Clone + Debug + Default + TableSecondaryIndexEventsOps + Send + Sync, + PrimaryKeyGenState: Clone + Debug + Send + Sync, + AvailableIndexes: Clone + Copy + Debug + Eq + Hash + Send + Sync, +{ + fn table_name(config: &DatabaseS3DiskConfig) -> eyre::Result<&str> { + Path::new(config.disk.table_path()) + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| eyre::eyre!("invalid table path")) + } + + fn restore_from_database(config: &DatabaseS3DiskConfig, table_id: TableId) -> eyre::Result<()> { + let catalog = config.database.catalog(); + let pages = catalog + .system_pages() + .into_iter() + .filter(|page| page.table_id == table_id) + .collect::>(); + if pages.is_empty() { + return Ok(()); + } + let table = catalog + .system_tables() + .into_iter() + .find(|table| table.table_id == table_id) + .ok_or_else(|| eyre::eyre!("system catalog has pages for an unknown table"))?; + let indexes = catalog + .system_indexes() + .into_iter() + .filter(|index| index.table_id == table_id) + .map(|index| (index.space_id, index)) + .collect::>(); + let table_path = Path::new(config.disk.table_path()); + let stage = staging_path(table_path, "domain-stage")?; + remove_path_if_exists(&stage)?; + std::fs::create_dir_all(&stage)?; + + let restore = (|| { + let mut lengths = BTreeMap::::new(); + for page in pages { + let relative = if page.space_id == table.data_space_id { + PathBuf::from(WT_DATA_EXTENSION) + } else { + let index = indexes + .get(&page.space_id) + .ok_or_else(|| eyre::eyre!("system catalog page has no owning index"))?; + if index.primary { + PathBuf::from(format!("primary{WT_INDEX_EXTENSION}")) + } else { + PathBuf::from(format!("{}{WT_INDEX_EXTENSION}", index.name.as_str())) + } + }; + let address = PageAddress { + domain: config.database.id(), + table_id, + space_id: page.space_id, + page_id: page.page_id, + page_kind: page.page_kind, + }; + let image = config + .database + .read_page(address)? + .ok_or_else(|| eyre::eyre!("system catalog page object is missing"))?; + let offset = if page.space_id == table.data_space_id { + if image.len() != table.page_stride as usize { + return Err(eyre::eyre!("remote data page length does not match the table stride")); + } + let header = data_bucket::inspect_page_image_header(&image)?; + if header.page_id != page.page_id || header.space_id != page.space_id { + return Err(eyre::eyre!( + "remote data page identity does not match the system catalog" + )); + } + u64::from(table.page_stride).checked_mul(usize::from(page.page_id) as u64) + } else { + if image.is_empty() || image.len() > INDEX_CHUNK_BYTES { + return Err(eyre::eyre!("remote index chunk has an invalid length")); + } + (INDEX_CHUNK_BYTES as u64).checked_mul(usize::from(page.page_id) as u64) + } + .ok_or_else(|| eyre::eyre!("remote page offset overflow"))?; + let path = stage.join(relative); + let mut file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(&path)?; + file.seek(SeekFrom::Start(offset))?; + file.write_all(&image)?; + lengths + .entry(path) + .and_modify(|length| *length = (*length).max(offset + image.len() as u64)) + .or_insert(offset + image.len() as u64); + } + for (path, length) in lengths { + let file = std::fs::OpenOptions::new().write(true).open(path)?; + file.set_len(length)?; + file.sync_all()?; + } + Ok::<(), eyre::Report>(()) + })(); + if let Err(error) = restore { + let _ = std::fs::remove_dir_all(&stage); + return Err(error); + } + publish_stage(table_path, &stage) + } + + fn sync_to_database(&self) -> eyre::Result<()> { + let scan = scan_table( + Path::new(self.config.disk.table_path()), + SpaceData::PAGE_STRIDE, + self.config.database.id(), + self.table_id, + )?; + let catalog = self.config.database.catalog(); + let current_pages = catalog + .system_pages() + .into_iter() + .filter(|page| page.table_id == self.table_id) + .map(|page| (CatalogRecord::Page(page.clone()).key(), page)) + .collect::>(); + let scanned_keys = scan.pages.keys().copied().collect::>(); + let mut generation = self.config.database.begin_generation()?; + let mut changed = false; + for (key, page) in &scan.pages { + if current_pages + .get(key) + .is_some_and(|current| current.checksum == page.hash) + { + continue; + } + generation.put_page(page.address, page.image.clone(), 0, 0); + changed = true; + } + for (key, page) in ¤t_pages { + if !scanned_keys.contains(key) { + generation.delete_page(PageAddress { + domain: self.config.database.id(), + table_id: self.table_id, + space_id: page.space_id, + page_id: page.page_id, + page_kind: page.page_kind, + }); + changed = true; + } + } + + let mut table = catalog + .system_tables() + .into_iter() + .find(|table| table.table_id == self.table_id) + .ok_or_else(|| eyre::eyre!("registered table is missing from the system catalog"))?; + if table.data_space_id != scan.data_space_id || table.page_stride != SpaceData::PAGE_STRIDE { + table.data_space_id = scan.data_space_id; + table.page_stride = SpaceData::PAGE_STRIDE; + generation.update_catalog(CatalogMutation::Upsert(CatalogRecord::Table(table))); + changed = true; + } + + let current_indexes = catalog + .system_indexes() + .into_iter() + .filter(|index| index.table_id == self.table_id) + .collect::>(); + let mut next_index_id = current_indexes + .iter() + .map(|index| index.index_id) + .max() + .unwrap_or(0) + .saturating_add(1); + for scanned in &scan.indexes { + let existing = current_indexes + .iter() + .find(|index| index.name == scanned.name || index.space_id == scanned.space_id); + let mut index = SystemIndexRecord { + table_id: self.table_id, + index_id: existing.map_or_else( + || { + let id = next_index_id; + next_index_id = next_index_id.saturating_add(1); + id + }, + |index| index.index_id, + ), + space_id: scanned.space_id, + primary: scanned.primary, + name: scanned.name.clone(), + entries: existing.map_or(0, |index| index.entries), + generation: existing.map_or(self.config.database.generation() + 1, |index| index.generation), + }; + if existing != Some(&index) { + index.generation = self.config.database.generation() + 1; + generation.update_catalog(CatalogMutation::Upsert(CatalogRecord::Index(index))); + changed = true; + } + } + for index in current_indexes { + if !scan.indexes.iter().any(|scanned| scanned.space_id == index.space_id) { + generation.update_catalog(CatalogMutation::Delete(CatalogRecord::Index(index).key())); + changed = true; + } + } + if changed { + self.config.database.commit_generation(generation.finish())?; + } + Ok(()) + } +} + +impl< + SpaceData, + SpacePrimaryIndex, + SpaceSecondaryIndexes, + PrimaryKey, + SecondaryIndexEvents, + AvailableIndexes, + PrimaryKeyGenState, +> PersistenceEngine + for DatabaseS3PersistenceEngine< + SpaceData, + SpacePrimaryIndex, + SpaceSecondaryIndexes, + PrimaryKey, + SecondaryIndexEvents, + AvailableIndexes, + PrimaryKeyGenState, + > +where + PrimaryKey: Clone + Debug + Ord + TablePrimaryKey + Send + Sync, + ::Generator: PrimaryKeyGeneratorState, + SpaceData: SpaceDataOps + Send + Sync, + SpacePrimaryIndex: SpaceIndexOps + Send + Sync, + SpaceSecondaryIndexes: SpaceSecondaryIndexOps + Send + Sync, + SecondaryIndexEvents: Clone + Debug + Default + TableSecondaryIndexEventsOps + Send + Sync, + PrimaryKeyGenState: Clone + Debug + Send + Sync, + AvailableIndexes: Clone + Copy + Debug + Eq + Hash + Send + Sync, +{ + type Config = DatabaseS3DiskConfig; + + async fn new(config: Self::Config) -> eyre::Result { + let table_id = config.database.register_table_with_stride( + Self::table_name(&config)?, + config.disk.version(), + SpaceData::PAGE_STRIDE, + )?; + Self::restore_from_database(&config, table_id)?; + let inner = DiskPersistenceEngine::new(config.disk.clone()).await?; + Ok(Self { + inner, + config, + table_id, + marker: PhantomData, + }) + } + + async fn apply_operation( + &mut self, + operation: Operation, + ) -> eyre::Result<()> { + self.inner.apply_operation(operation).await?; + self.sync_to_database() + } + + async fn apply_batch_operation( + &mut self, + operation: BatchOperation, + ) -> eyre::Result<()> { + self.inner.apply_batch_operation(operation).await?; + self.sync_to_database() + } + + async fn reclaim_data_pages(&mut self, page_ids: Vec) -> eyre::Result<()> { + self.inner.reclaim_data_pages(page_ids).await?; + self.sync_to_database() + } + + async fn ensure_schema( + &mut self, + row_schema: Vec<(String, String)>, + primary_key_fields: Vec, + secondary_index_types: Vec<(String, String)>, + ) -> eyre::Result<()> { + self.inner + .ensure_schema(row_schema, primary_key_fields, secondary_index_types) + .await + } + + async fn validate_schema( + &mut self, + row_schema: Vec<(String, String)>, + primary_key_fields: Vec, + secondary_index_types: Vec<(String, String)>, + ) -> eyre::Result<()> { + self.inner + .validate_schema(row_schema, primary_key_fields, secondary_index_types) + .await + } + + fn config(&self) -> &Self::Config { + &self.config + } +} + +struct ScannedPage { + address: PageAddress, + image: Vec, + hash: [u8; 32], +} + +struct ScannedIndex { + name: CatalogName, + space_id: data_bucket::SpaceId, + primary: bool, +} + +struct TableScan { + data_space_id: data_bucket::SpaceId, + pages: BTreeMap<[u8; 32], ScannedPage>, + indexes: Vec, +} + +fn scan_table( + root: &Path, + stride: u32, + domain: data_bucket::storage::StorageDomainId, + table_id: TableId, +) -> eyre::Result { + let mut pages = BTreeMap::new(); + let mut indexes = Vec::new(); + let mut data_space_id = data_bucket::SpaceId(0); + if !root.exists() { + return Ok(TableScan { + data_space_id, + pages, + indexes, + }); + } + let mut paths = std::fs::read_dir(root)? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>()?; + paths.sort(); + for path in paths { + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + let is_data = name == WT_DATA_EXTENSION; + let is_index = name.ends_with(WT_INDEX_EXTENSION); + if !is_data && !is_index { + continue; + } + if is_data { + let length = std::fs::metadata(&path)?.len(); + if length % u64::from(stride) != 0 { + return Err(eyre::eyre!("data file length is not a whole number of pages")); + } + let mut file = std::fs::File::open(&path)?; + for _ in 0..length / u64::from(stride) { + let mut image = vec![0; stride as usize]; + file.read_exact(&mut image)?; + let header = data_bucket::inspect_page_image_header(&image)?; + if data_space_id.0 == 0 { + data_space_id = header.space_id; + } else if data_space_id != header.space_id { + return Err(eyre::eyre!("one data file contains multiple space identifiers")); + } + let page_kind = if header.page_type == data_bucket::PageType::Data { + PageKind::Data + } else { + PageKind::Metadata + }; + insert_scanned_page( + &mut pages, + PageAddress { + domain, + table_id, + space_id: header.space_id, + page_id: header.page_id, + page_kind, + }, + image, + )?; + } + } else { + let primary = name == format!("primary{WT_INDEX_EXTENSION}"); + let logical_name = if primary { + "primary" + } else { + name.strip_suffix(WT_INDEX_EXTENSION) + .ok_or_else(|| eyre::eyre!("invalid index file name"))? + }; + let space_id = index_space_id(name); + if space_id == data_space_id || indexes.iter().any(|index| index.space_id == space_id) { + return Err(eyre::eyre!("generated index storage identifier collision")); + } + indexes.push(ScannedIndex { + name: CatalogName::new(logical_name)?, + space_id, + primary, + }); + let page_kind = if primary { + PageKind::PrimaryIndex + } else { + PageKind::SecondaryIndex + }; + let mut file = std::fs::File::open(&path)?; + let mut page_id = 0_u32; + loop { + let mut image = vec![0; INDEX_CHUNK_BYTES]; + let read = file.read(&mut image)?; + if read == 0 { + break; + } + image.truncate(read); + insert_scanned_page( + &mut pages, + PageAddress { + domain, + table_id, + space_id, + page_id: page_id.into(), + page_kind, + }, + image, + )?; + page_id = page_id + .checked_add(1) + .ok_or_else(|| eyre::eyre!("index file has too many chunks"))?; + } + } + } + Ok(TableScan { + data_space_id, + pages, + indexes, + }) +} + +fn index_space_id(name: &str) -> data_bucket::SpaceId { + let hash = blake3::hash(name.as_bytes()); + let bytes = hash.as_bytes(); + let mut id = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + if id == 0 { + id = 1; + } + data_bucket::SpaceId(id) +} + +fn insert_scanned_page( + pages: &mut BTreeMap<[u8; 32], ScannedPage>, + address: PageAddress, + image: Vec, +) -> eyre::Result<()> { + let key = CatalogRecord::page_key(address); + let hash = *blake3::hash(&image).as_bytes(); + if pages.insert(key, ScannedPage { address, image, hash }).is_some() { + return Err(eyre::eyre!("duplicate logical page in table files")); + } + Ok(()) +} + +fn staging_path(table_path: &Path, label: &str) -> eyre::Result { + let parent = table_path.parent().unwrap_or_else(|| Path::new(".")); + let name = table_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| eyre::eyre!("invalid table path"))?; + let nonce = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos(); + Ok(parent.join(format!(".{name}.{label}-{}-{nonce}", std::process::id()))) +} + +fn remove_path_if_exists(path: &Path) -> eyre::Result<()> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(path)?, + Ok(_) => std::fs::remove_file(path)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + Ok(()) +} + +fn publish_stage(table_path: &Path, stage: &Path) -> eyre::Result<()> { + if let Some(parent) = table_path.parent() { + std::fs::create_dir_all(parent)?; + } + if !table_path.exists() { + std::fs::rename(stage, table_path)?; + return Ok(()); + } + let backup = staging_path(table_path, "domain-backup")?; + std::fs::rename(table_path, &backup)?; + if let Err(error) = std::fs::rename(stage, table_path) { + std::fs::rename(&backup, table_path)?; + return Err(error.into()); + } + std::fs::remove_dir_all(backup)?; + Ok(()) +} diff --git a/src/features/mod.rs b/src/features/mod.rs index 182dcba3..93838398 100644 --- a/src/features/mod.rs +++ b/src/features/mod.rs @@ -1,5 +1,10 @@ #[cfg(feature = "s3-support")] +pub mod database_s3; +#[cfg(feature = "s3-support")] pub mod s3_support; +#[cfg(feature = "s3-support")] +pub use database_s3::{DatabaseS3DiskConfig, DatabaseS3PersistenceEngine}; + #[cfg(feature = "s3-support")] pub use s3_support::*; diff --git a/src/features/s3_support.rs b/src/features/s3_support.rs index 6a93ced7..3615ce98 100644 --- a/src/features/s3_support.rs +++ b/src/features/s3_support.rs @@ -1,11 +1,15 @@ -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; -use std::path::Path; -use std::time::Duration; +use alloc::{format, string::String, string::ToString, vec::Vec}; +use core::fmt::{Debug, Write as _}; +use core::hash::Hash; +use core::marker::PhantomData; +use core::time::Duration; +use std::collections::{HashMap, HashSet}; +use std::io::{BufReader, Read as _, Seek as _, SeekFrom, Write as _}; +use std::path::{Component, Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; -use reqwest::Client; use rusty_s3::{Bucket, Credentials, S3Action, UrlStyle}; +use ureq::Agent; use url::Url; use walkdir::WalkDir; @@ -17,6 +21,17 @@ use crate::persistence::{ }; use crate::prelude::{PrimaryKeyGeneratorState, TablePrimaryKey, WT_DATA_EXTENSION, WT_INDEX_EXTENSION}; +const MANIFEST_FILE: &str = "manifest.v1"; +const MANIFEST_MAGIC_V1: &[u8; 8] = b"WTS3M001"; +const MANIFEST_MAGIC_V2: &[u8; 8] = b"WTS3M002"; +/// The throughput target for a full upload or a run of adjacent dirty pages. +/// It is deliberately a target rather than a minimum: one changed DataBucket +/// page is published as one page-sized immutable segment. +const SEGMENT_TARGET: usize = 4 * 1024 * 1024; +const CHANGE_BLOCK_SIZE: usize = data_bucket::PAGE_SIZE; +const MAX_MANIFEST_FILES: usize = 16_384; +const MAX_MANIFEST_EXTENTS: usize = 4_194_304; + #[derive(Debug, Clone)] pub struct S3Config { pub bucket_name: String, @@ -39,7 +54,321 @@ impl PersistenceConfig for S3DiskConfig { } fn version(&self) -> u32 { - todo!() + self.disk.version() + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct SegmentExtent { + file_offset: u64, + length: u32, + segment_offset: u32, + segment_length: u32, + hash: [u8; 32], +} + +impl SegmentExtent { + fn file_end(&self) -> eyre::Result { + self.file_offset + .checked_add(u64::from(self.length)) + .ok_or_else(|| eyre::eyre!("S3 manifest extent offset overflow")) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct ManifestFile { + path: String, + length: u64, + extents: Vec, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +struct TableManifest { + files: Vec, +} + +impl TableManifest { + fn encode(&self) -> eyre::Result> { + let file_count = u32::try_from(self.files.len()).map_err(|_| eyre::eyre!("too many S3 manifest files"))?; + if self.files.len() > MAX_MANIFEST_FILES { + return Err(eyre::eyre!("too many S3 manifest files")); + } + let total_extents = self.files.iter().try_fold(0_usize, |total, file| { + total + .checked_add(file.extents.len()) + .ok_or_else(|| eyre::eyre!("S3 manifest extent count overflow")) + })?; + if total_extents > MAX_MANIFEST_EXTENTS { + return Err(eyre::eyre!("too many extents in S3 manifest")); + } + let mut bytes = Vec::new(); + bytes.extend_from_slice(MANIFEST_MAGIC_V2); + bytes.extend_from_slice(&file_count.to_le_bytes()); + + for file in &self.files { + validate_manifest_file(file)?; + validate_relative_path(&file.path)?; + let path = file.path.as_bytes(); + let path_len = u16::try_from(path.len()).map_err(|_| eyre::eyre!("S3 manifest path is too long"))?; + let extent_count = + u32::try_from(file.extents.len()).map_err(|_| eyre::eyre!("too many extents in S3 manifest"))?; + bytes.extend_from_slice(&path_len.to_le_bytes()); + bytes.extend_from_slice(path); + bytes.extend_from_slice(&file.length.to_le_bytes()); + bytes.extend_from_slice(&extent_count.to_le_bytes()); + for extent in &file.extents { + bytes.extend_from_slice(&extent.file_offset.to_le_bytes()); + bytes.extend_from_slice(&extent.length.to_le_bytes()); + bytes.extend_from_slice(&extent.segment_offset.to_le_bytes()); + bytes.extend_from_slice(&extent.segment_length.to_le_bytes()); + bytes.extend_from_slice(&extent.hash); + } + } + + let checksum = blake3::hash(&bytes); + bytes.extend_from_slice(checksum.as_bytes()); + Ok(bytes) + } + + fn decode(bytes: &[u8]) -> eyre::Result { + if bytes.len() < MANIFEST_MAGIC_V2.len() + 4 + 32 { + return Err(eyre::eyre!("S3 manifest is truncated")); + } + let (payload, checksum) = bytes.split_at(bytes.len() - 32); + if blake3::hash(payload).as_bytes() != checksum { + return Err(eyre::eyre!("S3 manifest checksum mismatch")); + } + + let magic = payload + .get(..MANIFEST_MAGIC_V2.len()) + .ok_or_else(|| eyre::eyre!("S3 manifest is truncated"))?; + if magic == MANIFEST_MAGIC_V1 { + return Self::decode_v1(payload); + } + if magic != MANIFEST_MAGIC_V2 { + return Err(eyre::eyre!("unsupported S3 manifest format")); + } + + let mut reader = ManifestReader::new(payload); + reader.take(MANIFEST_MAGIC_V2.len())?; + let file_count = reader.u32()? as usize; + if file_count > MAX_MANIFEST_FILES { + return Err(eyre::eyre!("S3 manifest contains too many files")); + } + + let mut files = Vec::with_capacity(file_count); + let mut previous_path: Option = None; + let mut total_extents = 0_usize; + for _ in 0..file_count { + let path_len = reader.u16()? as usize; + if path_len == 0 { + return Err(eyre::eyre!("S3 manifest contains an empty path")); + } + let path = core::str::from_utf8(reader.take(path_len)?)?.to_string(); + validate_relative_path(&path)?; + if previous_path.as_ref().is_some_and(|previous| previous >= &path) { + return Err(eyre::eyre!("S3 manifest file paths are not strictly sorted")); + } + previous_path = Some(path.clone()); + + let length = reader.u64()?; + let extent_count = reader.u32()? as usize; + total_extents = total_extents + .checked_add(extent_count) + .ok_or_else(|| eyre::eyre!("S3 manifest extent count overflow"))?; + if total_extents > MAX_MANIFEST_EXTENTS { + return Err(eyre::eyre!("S3 manifest contains too many extents")); + } + + let mut extents = Vec::with_capacity(extent_count); + for _ in 0..extent_count { + let file_offset = reader.u64()?; + let extent_length = reader.u32()?; + let segment_offset = reader.u32()?; + let segment_length = reader.u32()?; + let mut hash = [0_u8; 32]; + let hash_length = hash.len(); + hash.copy_from_slice(reader.take(hash_length)?); + extents.push(SegmentExtent { + file_offset, + length: extent_length, + segment_offset, + segment_length, + hash, + }); + } + let file = ManifestFile { path, length, extents }; + validate_manifest_file(&file)?; + files.push(file); + } + + if !reader.is_empty() { + return Err(eyre::eyre!("S3 manifest has trailing data")); + } + Ok(Self { files }) + } + + fn decode_v1(payload: &[u8]) -> eyre::Result { + let mut reader = ManifestReader::new(payload); + if reader.take(MANIFEST_MAGIC_V1.len())? != MANIFEST_MAGIC_V1 { + return Err(eyre::eyre!("unsupported S3 manifest format")); + } + let file_count = reader.u32()? as usize; + if file_count > MAX_MANIFEST_FILES { + return Err(eyre::eyre!("S3 manifest contains too many files")); + } + + let mut files = Vec::with_capacity(file_count); + let mut previous_path: Option = None; + let mut total_chunks = 0_usize; + for _ in 0..file_count { + let path_len = reader.u16()? as usize; + if path_len == 0 { + return Err(eyre::eyre!("S3 manifest contains an empty path")); + } + let path = core::str::from_utf8(reader.take(path_len)?)?.to_string(); + validate_relative_path(&path)?; + if previous_path.as_ref().is_some_and(|previous| previous >= &path) { + return Err(eyre::eyre!("S3 manifest file paths are not strictly sorted")); + } + previous_path = Some(path.clone()); + + let length = reader.u64()?; + let chunk_count = reader.u32()? as usize; + total_chunks = total_chunks + .checked_add(chunk_count) + .ok_or_else(|| eyre::eyre!("S3 manifest chunk count overflow"))?; + if total_chunks > MAX_MANIFEST_EXTENTS { + return Err(eyre::eyre!("S3 manifest contains too many chunks")); + } + let expected_chunks = if length == 0 { + 0 + } else { + usize::try_from(length.div_ceil(SEGMENT_TARGET as u64))? + }; + if chunk_count != expected_chunks { + return Err(eyre::eyre!("S3 manifest chunk count does not match file length")); + } + + let mut extents = Vec::with_capacity(chunk_count); + let mut file_offset = 0_u64; + for index in 0..chunk_count { + let chunk_length = reader.u32()?; + if chunk_length == 0 || chunk_length as usize > SEGMENT_TARGET { + return Err(eyre::eyre!("S3 manifest contains an invalid chunk length")); + } + if index + 1 != chunk_count && chunk_length as usize != SEGMENT_TARGET { + return Err(eyre::eyre!("S3 manifest contains a short interior chunk")); + } + let mut hash = [0_u8; 32]; + let hash_length = hash.len(); + hash.copy_from_slice(reader.take(hash_length)?); + extents.push(SegmentExtent { + file_offset, + length: chunk_length, + segment_offset: 0, + segment_length: chunk_length, + hash, + }); + file_offset = file_offset + .checked_add(u64::from(chunk_length)) + .ok_or_else(|| eyre::eyre!("S3 manifest file length overflow"))?; + } + let file = ManifestFile { path, length, extents }; + validate_manifest_file(&file)?; + files.push(file); + } + if !reader.is_empty() { + return Err(eyre::eyre!("S3 manifest has trailing data")); + } + Ok(Self { files }) + } + + fn committed_segments(&self) -> HashSet<[u8; 32]> { + self.files + .iter() + .flat_map(|file| file.extents.iter().map(|extent| extent.hash)) + .collect() + } + + fn file(&self, path: &str) -> Option<&ManifestFile> { + self.files + .binary_search_by(|file| file.path.as_str().cmp(path)) + .ok() + .map(|index| &self.files[index]) + } +} + +fn validate_manifest_file(file: &ManifestFile) -> eyre::Result<()> { + let mut described_length = 0_u64; + for extent in &file.extents { + if extent.file_offset != described_length { + return Err(eyre::eyre!("S3 manifest extents do not cover the file contiguously")); + } + if extent.length == 0 || extent.segment_length == 0 || extent.segment_length as usize > SEGMENT_TARGET { + return Err(eyre::eyre!("S3 manifest contains an invalid extent length")); + } + let segment_end = extent + .segment_offset + .checked_add(extent.length) + .ok_or_else(|| eyre::eyre!("S3 manifest segment offset overflow"))?; + if segment_end > extent.segment_length { + return Err(eyre::eyre!("S3 manifest extent exceeds its segment")); + } + described_length = extent.file_end()?; + } + if described_length != file.length { + return Err(eyre::eyre!("S3 manifest extents do not cover the file length")); + } + Ok(()) +} + +struct ManifestReader<'a> { + bytes: &'a [u8], + position: usize, +} + +impl<'a> ManifestReader<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, position: 0 } + } + + fn take(&mut self, length: usize) -> eyre::Result<&'a [u8]> { + let end = self + .position + .checked_add(length) + .ok_or_else(|| eyre::eyre!("S3 manifest offset overflow"))?; + let value = self + .bytes + .get(self.position..end) + .ok_or_else(|| eyre::eyre!("S3 manifest is truncated"))?; + self.position = end; + Ok(value) + } + + fn u16(&mut self) -> eyre::Result { + let mut bytes = [0_u8; 2]; + let length = bytes.len(); + bytes.copy_from_slice(self.take(length)?); + Ok(u16::from_le_bytes(bytes)) + } + + fn u32(&mut self) -> eyre::Result { + let mut bytes = [0_u8; 4]; + let length = bytes.len(); + bytes.copy_from_slice(self.take(length)?); + Ok(u32::from_le_bytes(bytes)) + } + + fn u64(&mut self) -> eyre::Result { + let mut bytes = [0_u8; 8]; + let length = bytes.len(); + bytes.copy_from_slice(self.take(length)?); + Ok(u64::from_le_bytes(bytes)) + } + + fn is_empty(&self) -> bool { + self.position == self.bytes.len() } } @@ -68,7 +397,9 @@ pub struct S3SyncDiskPersistenceEngine< config: S3DiskConfig, bucket: Bucket, credentials: Credentials, - client: Client, + client: Agent, + committed_manifest: Option, + committed_blocks: HashMap>, phantom: PhantomData<(PrimaryKey, SecondaryIndexEvents, PrimaryKeyGenState, AvailableIndexes)>, } @@ -100,125 +431,596 @@ where PrimaryKeyGenState: Clone + Debug + Send + Sync, AvailableIndexes: Clone + Copy + Debug + Eq + Hash + Send + Sync, { - fn create_bucket(config: &S3Config) -> eyre::Result<(Bucket, Credentials, Client)> { + fn create_bucket(config: &S3Config) -> eyre::Result<(Bucket, Credentials, Agent)> { let credentials = Credentials::new(&config.access_key, &config.secret_key); let endpoint: Url = config.endpoint.parse()?; let region = config.region.clone().unwrap_or_else(|| "auto".to_string()); let bucket = Bucket::new(endpoint, UrlStyle::Path, config.bucket_name.clone(), region)?; - let client = Client::builder().timeout(Duration::from_secs(30)).build()?; + // Blocking, like every other I/O call in this crate. The persistence + // engine owns its thread, so a request that blocks it is the right + // execution shape and does not require a Tokio reactor. + let client = ureq::AgentBuilder::new().timeout(Duration::from_secs(30)).build(); Ok((bucket, credentials, client)) } - async fn sync_to_s3(&self) -> eyre::Result<()> { - let table_path = self.config.disk.table_path(); - let table_path = Path::new(table_path); - let prefix = self.config.s3.prefix.as_deref().unwrap_or(""); + fn table_name(config: &S3DiskConfig) -> eyre::Result<&str> { + Path::new(config.disk.table_path()) + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| eyre::eyre!("invalid table path")) + } + + fn full_s3_path(prefix: &str, s3_path: &str, table_name: &str) -> String { + let prefix = prefix.trim_end_matches('/'); + let path = s3_path.trim_start_matches('/'); + if prefix.is_empty() { + format!("{table_name}/{path}") + } else { + format!("{prefix}/{table_name}/{path}") + } + } + + fn object_key(&self, path: &str) -> eyre::Result { + Ok(Self::full_s3_path( + self.config.s3.prefix.as_deref().unwrap_or(""), + path, + Self::table_name(&self.config)?, + )) + } + + fn chunk_path(hash: &[u8; 32]) -> String { + let mut hex = String::with_capacity(64); + for byte in hash { + write!(&mut hex, "{byte:02x}").expect("writing to String cannot fail"); + } + format!("chunks/{hex}") + } + fn get_object_optional( + bucket: &Bucket, + credentials: &Credentials, + client: &Agent, + key: &str, + ) -> eyre::Result>> { + let action = bucket.get_object(Some(credentials), key); + let url = action.sign(Duration::from_secs(3600)); + let response = match client.get(url.as_str()).call() { + Ok(response) => response, + Err(ureq::Error::Status(404, _)) => return Ok(None), + Err(error) => return Err(error.into()), + }; + let mut bytes = Vec::new(); + response.into_reader().read_to_end(&mut bytes)?; + Ok(Some(bytes)) + } + + fn put_object_verified(&self, key: &str, bytes: &[u8]) -> eyre::Result<()> { + let action = self.bucket.put_object(Some(&self.credentials), key); + let url = action.sign(Duration::from_secs(3600)); + match self.client.put(url.as_str()).send_bytes(bytes) { + Ok(_) => Ok(()), + Err(put_error) => { + // A connection can fail after the object service committed the + // PUT. Resolve that ambiguity before reporting failure; the + // caller must never repeat a local database mutation merely to + // discover that its manifest was already published. + let stored = Self::get_object_optional(&self.bucket, &self.credentials, &self.client, key)?; + if stored.as_deref() == Some(bytes) { + Ok(()) + } else { + Err(put_error.into()) + } + } + } + } + + async fn sync_to_s3(&mut self) -> eyre::Result<()> { + let table_path = Path::new(self.config.disk.table_path()); if !table_path.exists() { return Ok(()); } - for entry in WalkDir::new(table_path) + let mut local_files = WalkDir::new(table_path) .into_iter() - .filter_map(|e| e.ok()) - .filter(|e| e.file_type().is_file()) - { - let local_path = entry.path(); - let relative = local_path.strip_prefix(table_path).unwrap_or(local_path); - let table_name = table_path - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| eyre::eyre!("Invalid table path"))?; - let s3_key = Self::full_s3_path(prefix, &relative.to_string_lossy(), table_name); - - tracing::debug!(local_path = %local_path.display(), s3_key = %s3_key, "Uploading file to S3"); - - let content = tokio::fs::read(local_path).await?; - - let action = self.bucket.put_object(Some(&self.credentials), &s3_key); - let url = action.sign(Duration::from_secs(3600)); + .collect::, _>>()? + .into_iter() + .filter(|entry| entry.file_type().is_file()) + .filter(|entry| is_table_file(entry.path())) + .map(|entry| { + let path = entry.path().to_path_buf(); + let relative = canonical_relative_path(table_path, &path)?; + Ok((relative, path)) + }) + .collect::>>()?; + local_files.sort_by(|left, right| left.0.cmp(&right.0)); + + let committed_segments = self + .committed_manifest + .as_ref() + .map_or_else(HashSet::new, TableManifest::committed_segments); + let mut uploaded_segments = HashSet::new(); + let mut files = Vec::with_capacity(local_files.len()); + let mut next_blocks = HashMap::with_capacity(local_files.len()); + + for (relative, local_path) in local_files { + let blocks = describe_file_blocks(&local_path)?; + let file_length = blocks.last().map_or(0, LocalBlock::end); + let previous_file = self + .committed_manifest + .as_ref() + .and_then(|manifest| manifest.file(&relative)); + let previous_hashes = self.committed_blocks.get(&relative); + let mut extents = match previous_file { + Some(file) => trim_extents(&file.extents, file_length)?, + None => Vec::new(), + }; + + let dirty = blocks + .iter() + .enumerate() + .map(|(index, block)| { + previous_file.is_none() + || previous_hashes + .and_then(|hashes| hashes.get(index)) + .is_none_or(|hash| hash != &block.hash) + }) + .collect::>(); + + let mut index = 0; + while index < blocks.len() { + if !dirty[index] { + index += 1; + continue; + } + let first = index; + let mut segment_length = blocks[index].length as usize; + index += 1; + while index < blocks.len() + && dirty[index] + && segment_length + blocks[index].length as usize <= SEGMENT_TARGET + { + segment_length += blocks[index].length as usize; + index += 1; + } - self.client.put(url).body(content).send().await?.error_for_status()?; + let file_offset = blocks[first].offset; + let bytes = read_file_range(&local_path, file_offset, segment_length)?; + let hash = *blake3::hash(&bytes).as_bytes(); + if !committed_segments.contains(&hash) && uploaded_segments.insert(hash) { + let key = self.object_key(&Self::chunk_path(&hash))?; + self.put_object_verified(&key, &bytes)?; + } + let extent = SegmentExtent { + file_offset, + length: u32::try_from(segment_length)?, + segment_offset: 0, + segment_length: u32::try_from(segment_length)?, + hash, + }; + extents = overlay_extent(&extents, extent)?; + } + + let file = ManifestFile { + path: relative, + length: file_length, + extents, + }; + validate_manifest_file(&file)?; + next_blocks.insert(file.path.clone(), blocks.into_iter().map(|block| block.hash).collect()); + files.push(file); } - tracing::debug!("S3 sync complete"); + let manifest = TableManifest { files }; + let manifest_bytes = manifest.encode()?; + let manifest_key = self.object_key(MANIFEST_FILE)?; + self.put_object_verified(&manifest_key, &manifest_bytes)?; + self.committed_manifest = Some(manifest); + self.committed_blocks = next_blocks; + + tracing::debug!(new_segments = uploaded_segments.len(), "S3 table manifest committed"); Ok(()) } - fn full_s3_path(prefix: &str, s3_path: &str, table_name: &str) -> String { - let prefix = prefix.trim_end_matches('/'); - let path = s3_path.trim_start_matches('/'); - if prefix.is_empty() { - format!("{}/{}", table_name, path) + async fn sync_from_s3( + bucket: &Bucket, + credentials: &Credentials, + client: &Agent, + config: &S3DiskConfig, + ) -> eyre::Result> { + let table_name = Self::table_name(config)?; + let prefix = config.s3.prefix.as_deref().unwrap_or(""); + let manifest_key = Self::full_s3_path(prefix, MANIFEST_FILE, table_name); + + if let Some(bytes) = Self::get_object_optional(bucket, credentials, client, &manifest_key)? { + let manifest = TableManifest::decode(&bytes)?; + Self::restore_manifest(bucket, credentials, client, config, &manifest).await?; + tracing::info!(table_name, "S3 table manifest restored"); + return Ok(Some(manifest)); + } + + if Self::restore_legacy_objects(bucket, credentials, client, config).await? { + tracing::info!( + table_name, + "legacy S3 table objects restored; next write will publish a manifest" + ); } else { - format!("{}/{}/{}", prefix, table_name, path) + tracing::debug!(table_name, "no committed table objects found in S3"); } + Ok(None) } - async fn sync_from_s3( + async fn restore_manifest( bucket: &Bucket, credentials: &Credentials, - client: &Client, + client: &Agent, config: &S3DiskConfig, + manifest: &TableManifest, ) -> eyre::Result<()> { + let table_path = Path::new(config.disk.table_path()); + let stage = staging_path(table_path, "stage")?; + remove_path_if_exists(&stage)?; + std::fs::create_dir_all(&stage)?; + + let prefix = config.s3.prefix.as_deref().unwrap_or(""); + let table_name = Self::table_name(config)?; + let restore_result = async { + for file in &manifest.files { + let local_path = stage.join(&file.path); + if let Some(parent) = local_path.parent() { + std::fs::create_dir_all(parent)?; + } + let mut restored_file = std::fs::File::create(&local_path)?; + restored_file.set_len(file.length)?; + let mut by_segment: HashMap<[u8; 32], (u32, Vec<&SegmentExtent>)> = HashMap::new(); + for extent in &file.extents { + let entry = by_segment + .entry(extent.hash) + .or_insert_with(|| (extent.segment_length, Vec::new())); + if entry.0 != extent.segment_length { + return Err(eyre::eyre!("S3 manifest gives one segment conflicting lengths")); + } + entry.1.push(extent); + } + for (hash, (segment_length, extents)) in by_segment { + let key = Self::full_s3_path(prefix, &Self::chunk_path(&hash), table_name); + let bytes = Self::get_object_optional(bucket, credentials, client, &key)? + .ok_or_else(|| eyre::eyre!("S3 manifest references missing segment {key}"))?; + if bytes.len() != segment_length as usize || blake3::hash(&bytes).as_bytes() != &hash { + return Err(eyre::eyre!("S3 segment failed length or hash validation: {key}")); + } + for extent in extents { + let from = extent.segment_offset as usize; + let to = from + .checked_add(extent.length as usize) + .ok_or_else(|| eyre::eyre!("S3 manifest segment slice overflow"))?; + restored_file.seek(SeekFrom::Start(extent.file_offset))?; + restored_file.write_all(&bytes[from..to])?; + } + } + restored_file.flush()?; + } + Ok::<(), eyre::Report>(()) + } + .await; + + if let Err(error) = restore_result { + let _ = std::fs::remove_dir_all(&stage); + return Err(error); + } + publish_staged_table(table_path, &stage) + } + + async fn restore_legacy_objects( + bucket: &Bucket, + credentials: &Credentials, + client: &Agent, + config: &S3DiskConfig, + ) -> eyre::Result { use rusty_s3::actions::ListObjectsV2; - let table_path = config.disk.table_path(); - let table_path = Path::new(table_path); + let table_path = Path::new(config.disk.table_path()); + let table_name = Self::table_name(config)?; let prefix = config.s3.prefix.as_deref().unwrap_or(""); + let table_root = Self::full_s3_path(prefix, "", table_name); + let mut continuation = None; + let mut objects = Vec::new(); - let table_name = table_path - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| eyre::eyre!("Invalid table path"))?; + loop { + let mut action = bucket.list_objects_v2(Some(credentials)); + action.with_prefix(&table_root); + if let Some(token) = continuation.as_deref() { + action.with_continuation_token(token); + } + let url = action.sign(Duration::from_secs(3600)); + let response = client.get(url.as_str()).call()?; + let parsed = ListObjectsV2::parse_response(&response.into_string()?)?; + for object in parsed.contents { + let Some(relative) = object.key.strip_prefix(&table_root) else { + continue; + }; + if !is_table_name(relative) || validate_relative_path(relative).is_err() { + continue; + } + let bytes = Self::get_object_optional(bucket, credentials, client, &object.key)? + .ok_or_else(|| eyre::eyre!("listed S3 object disappeared: {}", object.key))?; + objects.push((relative.to_string(), bytes)); + } + continuation = parsed.next_continuation_token; + if continuation.is_none() { + break; + } + } + + if objects.is_empty() { + return Ok(false); + } + objects.sort_by(|left, right| left.0.cmp(&right.0)); + let stage = staging_path(table_path, "legacy-stage")?; + remove_path_if_exists(&stage)?; + std::fs::create_dir_all(&stage)?; + for (relative, bytes) in objects { + crate::fsx::write(stage.join(relative), bytes).await?; + } + publish_staged_table(table_path, &stage)?; + Ok(true) + } +} - let s3_path = Self::full_s3_path(prefix, "", table_name); +fn is_table_name(name: &str) -> bool { + name.ends_with(WT_DATA_EXTENSION) || name.ends_with(WT_INDEX_EXTENSION) +} - let mut action = bucket.list_objects_v2(Some(credentials)); - action.with_prefix(&s3_path); - action.with_delimiter("/"); - let url = action.sign(Duration::from_secs(3600)); +#[derive(Clone, Debug)] +struct LocalBlock { + offset: u64, + length: u32, + hash: [u8; 32], +} - let response = client.get(url).send().await?.error_for_status()?; +impl LocalBlock { + fn end(&self) -> u64 { + self.offset + u64::from(self.length) + } +} - let text = response.text().await?; - let parsed = ListObjectsV2::parse_response(&text)?; +fn describe_file_blocks(path: &Path) -> eyre::Result> { + let mut file = BufReader::with_capacity(SEGMENT_TARGET, std::fs::File::open(path)?); + let mut buffer = vec![0_u8; CHANGE_BLOCK_SIZE]; + let mut blocks = Vec::new(); + let mut offset = 0_u64; + loop { + let length = read_buffer(&mut file, &mut buffer)?; + if length == 0 { + break; + } + blocks.push(LocalBlock { + offset, + length: u32::try_from(length)?, + hash: *blake3::hash(&buffer[..length]).as_bytes(), + }); + offset = offset + .checked_add(u64::try_from(length)?) + .ok_or_else(|| eyre::eyre!("local table file length overflow"))?; + } + Ok(blocks) +} - if parsed.contents.is_empty() { - tracing::debug!(s3_prefix = %s3_path, "No objects found in S3"); - return Ok(()); +fn describe_table_blocks(table_path: &Path) -> eyre::Result>> { + if !table_path.exists() { + return Ok(HashMap::new()); + } + let mut result = HashMap::new(); + for entry in WalkDir::new(table_path) { + let entry = entry?; + if !entry.file_type().is_file() || !is_table_file(entry.path()) { + continue; } + let relative = canonical_relative_path(table_path, entry.path())?; + let hashes = describe_file_blocks(entry.path())? + .into_iter() + .map(|block| block.hash) + .collect(); + result.insert(relative, hashes); + } + Ok(result) +} - tokio::fs::create_dir_all(table_path).await?; +fn read_file_range(path: &Path, offset: u64, length: usize) -> eyre::Result> { + let mut file = std::fs::File::open(path)?; + file.seek(SeekFrom::Start(offset))?; + let mut bytes = vec![0_u8; length]; + file.read_exact(&mut bytes)?; + Ok(bytes) +} - for obj in parsed.contents { - let s3_key = &obj.key; +fn read_buffer(file: &mut impl std::io::Read, buffer: &mut [u8]) -> std::io::Result { + let mut length = 0; + while length < buffer.len() { + let read = file.read(&mut buffer[length..])?; + if read == 0 { + break; + } + length += read; + } + Ok(length) +} + +fn trim_extents(extents: &[SegmentExtent], length: u64) -> eyre::Result> { + let mut trimmed = Vec::new(); + for extent in extents { + if extent.file_offset >= length { + break; + } + let keep = extent.file_end()?.min(length) - extent.file_offset; + let mut extent = extent.clone(); + extent.length = u32::try_from(keep)?; + trimmed.push(extent); + } + Ok(trimmed) +} - let filename = s3_key.rsplit('/').next().unwrap_or(s3_key); +fn overlay_extent(extents: &[SegmentExtent], replacement: SegmentExtent) -> eyre::Result> { + let start = replacement.file_offset; + let end = replacement.file_end()?; + let mut result = Vec::with_capacity(extents.len() + 2); + let mut inserted = false; - if !filename.ends_with(WT_DATA_EXTENSION) && !filename.ends_with(WT_INDEX_EXTENSION) { - tracing::debug!(s3_key = %s3_key, "Skipping non-table file"); - continue; + for extent in extents { + let extent_end = extent.file_end()?; + if extent_end <= start { + result.push(extent.clone()); + continue; + } + if extent.file_offset >= end { + if !inserted { + result.push(replacement.clone()); + inserted = true; } + result.push(extent.clone()); + continue; + } - let local_path = table_path.join(filename); + if extent.file_offset < start { + let mut left = extent.clone(); + left.length = u32::try_from(start - extent.file_offset)?; + result.push(left); + } + if !inserted { + result.push(replacement.clone()); + inserted = true; + } + if extent_end > end { + let skipped = u32::try_from(end - extent.file_offset)?; + let mut right = extent.clone(); + right.file_offset = end; + right.length = u32::try_from(extent_end - end)?; + right.segment_offset = right + .segment_offset + .checked_add(skipped) + .ok_or_else(|| eyre::eyre!("S3 manifest segment offset overflow"))?; + result.push(right); + } + } + if !inserted { + result.push(replacement); + } + merge_adjacent_extents(result) +} - tracing::debug!(s3_key = %s3_key, local_path = %local_path.display(), "Downloading file from S3"); +fn merge_adjacent_extents(extents: Vec) -> eyre::Result> { + let mut merged: Vec = Vec::with_capacity(extents.len()); + for extent in extents { + if let Some(previous) = merged.last_mut() { + let contiguous_file = previous.file_end()? == extent.file_offset; + let contiguous_segment = previous + .segment_offset + .checked_add(previous.length) + .is_some_and(|offset| offset == extent.segment_offset); + if contiguous_file + && contiguous_segment + && previous.segment_length == extent.segment_length + && previous.hash == extent.hash + { + previous.length = previous + .length + .checked_add(extent.length) + .ok_or_else(|| eyre::eyre!("S3 manifest extent length overflow"))?; + continue; + } + } + merged.push(extent); + } + Ok(merged) +} - let action = bucket.get_object(Some(credentials), s3_key); - let url = action.sign(Duration::from_secs(3600)); +fn is_table_file(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(is_table_name) +} - let response = client.get(url).send().await?.error_for_status()?; +fn validate_relative_path(path: &str) -> eyre::Result<()> { + if path.is_empty() || path.chars().any(|character| matches!(character, '\\' | ':' | '\0')) { + return Err(eyre::eyre!("invalid S3 manifest path")); + } + let parsed = Path::new(path); + if parsed + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + Ok(()) + } else { + Err(eyre::eyre!("unsafe S3 manifest path: {path}")) + } +} - let content = response.bytes().await?; - tokio::fs::write(&local_path, content).await?; +fn canonical_relative_path(root: &Path, path: &Path) -> eyre::Result { + let relative = path.strip_prefix(root)?; + let mut value = String::new(); + for component in relative.components() { + let Component::Normal(component) = component else { + return Err(eyre::eyre!("unsafe local table path: {}", path.display())); + }; + let component = component + .to_str() + .ok_or_else(|| eyre::eyre!("table path is not valid UTF-8: {}", path.display()))?; + if !value.is_empty() { + value.push('/'); } + value.push_str(component); + } + validate_relative_path(&value)?; + Ok(value) +} - tracing::info!(table_name = %table_name, "S3 download sync complete"); - Ok(()) +fn staging_path(table_path: &Path, label: &str) -> eyre::Result { + let parent = table_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let name = table_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| eyre::eyre!("invalid table path"))?; + let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos(); + Ok(parent.join(format!(".{name}.s3-{label}-{}-{timestamp}", std::process::id()))) +} + +fn remove_path_if_exists(path: &Path) -> eyre::Result<()> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(path)?, + Ok(_) => std::fs::remove_file(path)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + Ok(()) +} + +fn publish_staged_table(table_path: &Path, stage: &Path) -> eyre::Result<()> { + if let Some(parent) = table_path.parent().filter(|parent| !parent.as_os_str().is_empty()) { + std::fs::create_dir_all(parent)?; + } + if !table_path.exists() { + std::fs::rename(stage, table_path)?; + return Ok(()); } + + let backup = staging_path(table_path, "backup")?; + std::fs::rename(table_path, &backup)?; + if let Err(error) = std::fs::rename(stage, table_path) { + if let Err(rollback_error) = std::fs::rename(&backup, table_path) { + return Err(eyre::eyre!( + "failed to install restored S3 table ({error}) and failed to restore local table ({rollback_error})" + )); + } + return Err(error.into()); + } + if let Err(error) = std::fs::remove_dir_all(&backup) { + tracing::warn!(path = %backup.display(), error = %error, "restored table but could not remove backup directory"); + } + Ok(()) } impl< @@ -256,12 +1058,11 @@ where Self: Sized, { let (bucket, credentials, client) = Self::create_bucket(&config.s3)?; - - if let Err(e) = Self::sync_from_s3(&bucket, &credentials, &client, &config).await { - tracing::warn!(error = %e, "Failed to sync from S3, continuing with local files"); - } - + // If a manifest exists, failure is fatal: continuing with local files + // could publish stale state over a newer committed remote generation. + let committed_manifest = Self::sync_from_s3(&bucket, &credentials, &client, &config).await?; let inner = DiskPersistenceEngine::new(config.disk.clone()).await?; + let committed_blocks = describe_table_blocks(Path::new(config.disk.table_path()))?; Ok(Self { inner, @@ -269,6 +1070,8 @@ where bucket, credentials, client, + committed_manifest, + committed_blocks, phantom: PhantomData, }) } @@ -278,8 +1081,7 @@ where op: Operation, ) -> eyre::Result<()> { self.inner.apply_operation(op).await?; - self.sync_to_s3().await?; - Ok(()) + self.sync_to_s3().await } async fn apply_batch_operation( @@ -287,11 +1089,147 @@ where batch_op: BatchOperation, ) -> eyre::Result<()> { self.inner.apply_batch_operation(batch_op).await?; - self.sync_to_s3().await?; - Ok(()) + self.sync_to_s3().await } fn config(&self) -> &Self::Config { &self.config } } + +#[cfg(test)] +mod tests { + use super::*; + + fn extent(file_offset: u64, bytes: &[u8]) -> SegmentExtent { + SegmentExtent { + file_offset, + length: bytes.len() as u32, + segment_offset: 0, + segment_length: bytes.len() as u32, + hash: *blake3::hash(bytes).as_bytes(), + } + } + + #[test] + fn manifest_round_trip_is_deterministic() { + let manifest = TableManifest { + files: vec![ + ManifestFile { + path: ".wt.data".to_string(), + length: 3, + extents: vec![extent(0, b"abc")], + }, + ManifestFile { + path: "primary.wt.idx".to_string(), + length: 0, + extents: Vec::new(), + }, + ], + }; + let encoded = manifest.encode().unwrap(); + assert_eq!(TableManifest::decode(&encoded).unwrap(), manifest); + assert_eq!(manifest.encode().unwrap(), encoded); + } + + #[test] + fn manifest_rejects_corruption_and_unsafe_paths() { + let manifest = TableManifest { + files: vec![ManifestFile { + path: ".wt.data".to_string(), + length: 3, + extents: vec![extent(0, b"abc")], + }], + }; + let mut encoded = manifest.encode().unwrap(); + encoded[12] ^= 1; + assert!(TableManifest::decode(&encoded).is_err()); + + let unsafe_manifest = TableManifest { + files: vec![ManifestFile { + path: "../outside.wt.data".to_string(), + length: 0, + extents: Vec::new(), + }], + }; + assert!(unsafe_manifest.encode().is_err()); + } + + #[test] + fn manifest_requires_exact_extent_coverage() { + let manifest = TableManifest { + files: vec![ManifestFile { + path: ".wt.data".to_string(), + length: 4, + extents: vec![extent(0, b"abc")], + }], + }; + assert!(manifest.encode().is_err()); + } + + #[test] + fn manifest_requires_strictly_sorted_unique_paths() { + let file = |path: &str| ManifestFile { + path: path.to_string(), + length: 0, + extents: Vec::new(), + }; + let unsorted = TableManifest { + files: vec![file("primary.wt.idx"), file(".wt.data")], + } + .encode() + .unwrap(); + assert!(TableManifest::decode(&unsorted).is_err()); + + let duplicate = TableManifest { + files: vec![file(".wt.data"), file(".wt.data")], + } + .encode() + .unwrap(); + assert!(TableManifest::decode(&duplicate).is_err()); + } + + #[test] + fn a_page_change_splits_a_large_segment_without_reuploading_it() { + let original = vec![7_u8; SEGMENT_TARGET]; + let original_extent = extent(0, &original); + let changed_page = vec![9_u8; CHANGE_BLOCK_SIZE]; + let replacement = extent(CHANGE_BLOCK_SIZE as u64, &changed_page); + + let extents = overlay_extent(core::slice::from_ref(&original_extent), replacement.clone()).unwrap(); + assert_eq!(extents.len(), 3); + assert_eq!(extents[0].length as usize, CHANGE_BLOCK_SIZE); + assert_eq!(extents[1], replacement); + assert_eq!(extents[2].file_offset, (2 * CHANGE_BLOCK_SIZE) as u64); + assert_eq!(extents[2].segment_offset, (2 * CHANGE_BLOCK_SIZE) as u32); + assert_eq!(extents[2].file_end().unwrap(), SEGMENT_TARGET as u64); + assert_eq!(extents[0].hash, original_extent.hash); + assert_eq!(extents[2].hash, original_extent.hash); + } + + #[test] + fn legacy_manifest_decodes_as_segment_extents() { + let contents = [vec![1_u8; SEGMENT_TARGET], vec![2_u8; 19]]; + let mut payload = Vec::new(); + payload.extend_from_slice(MANIFEST_MAGIC_V1); + payload.extend_from_slice(&1_u32.to_le_bytes()); + let path = b".wt.data"; + payload.extend_from_slice(&(path.len() as u16).to_le_bytes()); + payload.extend_from_slice(path); + payload.extend_from_slice(&((SEGMENT_TARGET + 19) as u64).to_le_bytes()); + payload.extend_from_slice(&2_u32.to_le_bytes()); + for bytes in &contents { + payload.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + payload.extend_from_slice(blake3::hash(bytes).as_bytes()); + } + let checksum = blake3::hash(&payload); + payload.extend_from_slice(checksum.as_bytes()); + + let manifest = TableManifest::decode(&payload).unwrap(); + let file = &manifest.files[0]; + assert_eq!(file.extents.len(), 2); + assert_eq!(file.extents[0].length as usize, SEGMENT_TARGET); + assert_eq!(file.extents[1].file_offset, SEGMENT_TARGET as u64); + assert!(manifest.encode().unwrap().starts_with(MANIFEST_MAGIC_V2)); + } +} diff --git a/src/fsx.rs b/src/fsx.rs new file mode 100644 index 00000000..36ecaaea --- /dev/null +++ b/src/fsx.rs @@ -0,0 +1,62 @@ +//! The filesystem, without a runtime attached to it. +//! +//! A thin shim over [`nagoya::io`], which is where these operations live now. +//! They were here first, privately, and `data_bucket` could not name them at +//! all, so the two halves of one storage engine disagreed about what a file +//! was. The whole crate still goes through this module, so swapping backends +//! is still swapping one file. +//! +//! # Why the calls block +//! +//! Neither `tokio::fs` nor `async-fs` performs asynchronous file I/O: both hand +//! a blocking `std::fs` call to a thread pool, and what that buys is not +//! occupying a runtime worker rather than any actual overlap. It is not free. +//! Measured on this crate's scattered-update path, `tokio::fs` ran 12,316 rows +//! per second against 74,728 for the same code on blocking `std::fs`, and cold +//! reopen is 2.8x faster on an Arctic index without it. +//! +//! Because the calls block, the persistence engine should own a thread rather +//! than share a runtime's worker pool. It was never waiting on the disk through +//! a runtime anyway: the path measures 89 voluntary context switches across +//! 25,000 inserts. + +/// A file this crate reads and writes. +pub type File = nagoya::io::HostFile; + +/// Open a snapshot for reading without requesting write permission. +pub async fn open_read_only(path: impl AsRef) -> Result { + std::fs::File::open(path).map(File::new).map_err(Into::into) +} + +pub use nagoya::io::{ + Error, SeekFrom, append, create, create_dir_all, open, open_or_create, read, remove_dir_all, remove_file, rename, + write, +}; + +/// How long the file at `path` is, without opening it. +/// +/// Named for what it does. `nagoya::io::metadata` returns the length rather +/// than a metadata handle, because a length is all anything here ever wanted. +pub use nagoya::io::metadata; + +/// The operations no I/O trait carries, as free functions. +/// +/// They are methods on [`nagoya::io::File`]; these wrappers exist so call sites +/// read the same as they did when this module owned the implementation, and so +/// that a backend swap stays a change to one file. +pub async fn sync_all(file: &mut File) -> Result<(), Error> { + nagoya::io::File::sync_all(file).await +} + +pub async fn sync_data(file: &mut File) -> Result<(), Error> { + nagoya::io::File::sync_data(file).await +} + +pub async fn set_len(file: &mut File, length: u64) -> Result<(), Error> { + nagoya::io::File::set_length(file, length).await +} + +/// How long an already-open file is. +pub async fn file_metadata(file: &mut File) -> Result { + nagoya::io::File::length(file).await +} diff --git a/src/in_memory/data.rs b/src/in_memory/data.rs index 7b15d55e..c526dea0 100644 --- a/src/in_memory/data.rs +++ b/src/in_memory/data.rs @@ -1,8 +1,13 @@ -use std::cell::UnsafeCell; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::ops::{Deref, DerefMut}; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use alloc::vec::Vec; +use core::cell::UnsafeCell; +use core::fmt::Debug; +use core::marker::PhantomData; +use core::ops::{Deref, DerefMut}; +#[cfg(not(wt_loom))] +use core::sync::atomic::AtomicU32 as CellState; +use core::sync::atomic::{AtomicU32, Ordering}; +#[cfg(wt_loom)] +use loom::sync::atomic::AtomicU32 as CellState; use data_bucket::page::INNER_PAGE_SIZE; use data_bucket::page::PageId; @@ -23,122 +28,89 @@ use rkyv::{ use crate::in_memory::ArchivedRowWrapper; use crate::prelude::Link; -const CELL_LOCK_SLOTS: usize = 64; -const CELL_KEY_MASK: u64 = u32::MAX as u64; -const CELL_READER_ONE: u64 = 1 << 32; -const CELL_READER_MASK: u64 = ((1_u64 << 31) - 1) << 32; -const CELL_WRITER: u64 = 1 << 63; +const CELL_LOCK_SLOTS: usize = 256; +const CELL_READER_MASK: u32 = (1_u32 << 31) - 1; +const CELL_WRITER: u32 = 1 << 31; #[derive(Debug)] struct CellLocks { - slots: [AtomicU64; CELL_LOCK_SLOTS], + slots: [CellState; CELL_LOCK_SLOTS], } impl Default for CellLocks { fn default() -> Self { Self { - slots: std::array::from_fn(|_| AtomicU64::new(0)), + slots: core::array::from_fn(|_| CellState::new(0)), } } } impl CellLocks { #[inline] - fn key(link: Link) -> Result { - u64::from(link.offset) - .checked_add(1) - .filter(|key| *key <= CELL_KEY_MASK) - .ok_or(ExecutionError::InvalidLink) - } - - #[inline] - fn start(key: u64) -> usize { - (key.wrapping_mul(0x9e37_79b9) as usize) & (CELL_LOCK_SLOTS - 1) + fn start(link: Link) -> usize { + // Record starts are aligned to the archived row shape, so their low + // bits alone are a poor stripe selector. Mix all offset bits before + // taking the power-of-two table index. + let mut key = link.offset; + key ^= key >> 16; + key = key.wrapping_mul(0x7feb_352d); + key ^= key >> 15; + key = key.wrapping_mul(0x846c_a68b); + key ^= key >> 16; + key as usize & (CELL_LOCK_SLOTS - 1) } #[inline] fn wait(spins: &mut u32) { + #[cfg(wt_loom)] + { + let _ = spins; + loom::thread::yield_now(); + } + #[cfg(not(wt_loom))] if *spins < 64 { - std::hint::spin_loop(); + core::hint::spin_loop(); *spins += 1; } else { - std::thread::yield_now(); + crate::util::yield_now(); } } fn read(&self, link: Link) -> Result, ExecutionError> { - let key = Self::key(link)?; - let start = Self::start(key); + let state = &self.slots[Self::start(link)]; let mut spins = 0; - 'retry: loop { - for distance in 0..CELL_LOCK_SLOTS { - let state = &self.slots[(start + distance) & (CELL_LOCK_SLOTS - 1)]; - let current = state.load(Ordering::Acquire); - let current_key = current & CELL_KEY_MASK; - if current_key == key { - if current & CELL_WRITER != 0 || current & CELL_READER_MASK == CELL_READER_MASK { - Self::wait(&mut spins); - continue 'retry; - } - if state - .compare_exchange_weak(current, current + CELL_READER_ONE, Ordering::Acquire, Ordering::Relaxed) - .is_ok() - { - return Ok(CellReadGuard { state }); - } - continue 'retry; - } - if current == 0 { - if state - .compare_exchange_weak(0, key | CELL_READER_ONE, Ordering::Acquire, Ordering::Relaxed) - .is_ok() - { - return Ok(CellReadGuard { state }); - } - continue 'retry; - } + loop { + let current = state.load(Ordering::Acquire); + if current & CELL_WRITER == 0 + && current & CELL_READER_MASK != CELL_READER_MASK + && state + .compare_exchange_weak(current, current + 1, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return Ok(CellReadGuard { state }); } Self::wait(&mut spins); } } fn write(&self, link: Link) -> Result, ExecutionError> { - let key = Self::key(link)?; - let start = Self::start(key); + let state = &self.slots[Self::start(link)]; let mut spins = 0; - 'retry: loop { - for distance in 0..CELL_LOCK_SLOTS { - let state = &self.slots[(start + distance) & (CELL_LOCK_SLOTS - 1)]; - let current = state.load(Ordering::Acquire); - let current_key = current & CELL_KEY_MASK; - if current_key == key { - if current & CELL_WRITER != 0 { - Self::wait(&mut spins); - continue 'retry; - } - if state - .compare_exchange_weak(current, current | CELL_WRITER, Ordering::AcqRel, Ordering::Relaxed) - .is_err() - { - continue 'retry; - } - while state.load(Ordering::Acquire) & CELL_READER_MASK != 0 { - Self::wait(&mut spins); - } - return Ok(CellWriteGuard { state }); - } - if current == 0 { - if state - .compare_exchange_weak(0, key | CELL_WRITER, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { - return Ok(CellWriteGuard { state }); - } - continue 'retry; - } + loop { + let current = state.load(Ordering::Acquire); + if current & CELL_WRITER == 0 + && state + .compare_exchange_weak(current, current | CELL_WRITER, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + break; } Self::wait(&mut spins); } + while state.load(Ordering::Acquire) != CELL_WRITER { + Self::wait(&mut spins); + } + Ok(CellWriteGuard { state }) } fn reset(&self) { @@ -150,26 +122,20 @@ impl CellLocks { /// Shared access to one exact archived cell. pub(crate) struct CellReadGuard<'a> { - state: &'a AtomicU64, + state: &'a CellState, } impl Drop for CellReadGuard<'_> { #[inline] fn drop(&mut self) { - let previous = self.state.fetch_sub(CELL_READER_ONE, Ordering::Release); + let previous = self.state.fetch_sub(1, Ordering::Release); debug_assert_ne!(previous & CELL_READER_MASK, 0, "cell reader count underflow"); - let remaining = previous - CELL_READER_ONE; - if remaining & (CELL_READER_MASK | CELL_WRITER) == 0 { - let _ = self - .state - .compare_exchange(remaining, 0, Ordering::Release, Ordering::Relaxed); - } } } /// Exclusive access to one exact archived cell. pub(crate) struct CellWriteGuard<'a> { - state: &'a AtomicU64, + state: &'a CellState, } impl Drop for CellWriteGuard<'_> { @@ -222,9 +188,11 @@ pub struct Data { #[rkyv(with = Skip)] pub(crate) access: parking_lot::RwLock<()>, - /// Runtime-only exact-cell reader/writer coordination. The fixed table is - /// outside the archived row image, so lock state can never reach disk and - /// the beta.17 wrapper layout remains unchanged. + /// Runtime-only striped reader/writer coordination for archived cells. A + /// hash collision may conservatively make unrelated writes wait, while + /// every read/write pair for one offset always uses the same stripe. The + /// table is outside the archived row image, so lock state never reaches + /// disk and the beta.17 wrapper layout remains unchanged. #[rkyv(with = Skip)] cell_locks: CellLocks, @@ -305,6 +273,18 @@ impl Data { } } + /// Keep append allocation out of ranges owned by the restored free list. + pub(crate) fn reserve_restored_range(&self, link: Link) -> Result<(), ExecutionError> { + let end = (link.offset as usize) + .checked_add(link.length as usize) + .ok_or(ExecutionError::InvalidLink)?; + if link.page_id != self.id || link.length == 0 || end > DATA_LENGTH { + return Err(ExecutionError::InvalidLink); + } + self.free_offset.fetch_max(end as u32, Ordering::Release); + Ok(()) + } + pub fn set_page_id(&mut self, id: PageId) { self.id = id; } @@ -497,7 +477,7 @@ impl Data { // Use ptr::copy for overlapping memory regions (safe for shifting left) // When moving left (dst_offset < src_offset), this works correctly unsafe { - std::ptr::copy( + core::ptr::copy( inner_data.as_ptr().add(src_offset), inner_data.as_mut_ptr().add(dst_offset), length, @@ -567,10 +547,12 @@ impl Data { .map_err(|_| ExecutionError::LiveCellCountUnderflow) } + #[cfg(feature = "std")] pub(crate) fn has_live_cells(&self) -> bool { self.live_cells.load(Ordering::Acquire) != 0 } + #[cfg(feature = "std")] pub(crate) fn live_cell_count(&self) -> u32 { self.live_cells.load(Ordering::Acquire) } @@ -603,10 +585,11 @@ pub enum ExecutionError { LiveCellCountUnderflow, } -#[cfg(test)] +#[cfg(all(test, not(wt_loom)))] mod tests { - use std::sync::atomic::Ordering; - use std::sync::{Arc, mpsc}; + use alloc::sync::Arc; + use core::sync::atomic::Ordering; + use std::sync::mpsc; use std::thread; use rkyv::{Archive, Deserialize, Serialize}; @@ -622,6 +605,41 @@ mod tests { b: u64, } + #[test] + fn colliding_rows_keep_using_one_stable_stripe() { + let locks = super::CellLocks::default(); + let first = Link { + page_id: 1.into(), + offset: 7, + length: 16, + }; + let second = Link { offset: 14, ..first }; + assert_eq!(super::CellLocks::start(first), super::CellLocks::start(second)); + let preceding = locks.read(first).unwrap(); + let existing = locks.read(second).unwrap(); + drop(preceding); + let joining = locks.read(second).unwrap(); + assert!( + core::ptr::eq(existing.state, joining.state), + "readers of one row must share the state that excludes its writer" + ); + } + + #[test] + fn mixed_offsets_do_not_collapse_into_one_low_bit_stripe() { + let locks = super::CellLocks::default(); + let first = Link { + page_id: 1.into(), + offset: 0, + length: 16, + }; + let second = Link { offset: 64, ..first }; + assert_ne!(super::CellLocks::start(first), super::CellLocks::start(second)); + let first = locks.write(first).unwrap(); + let second = locks.write(second).unwrap(); + assert!(!core::ptr::eq(first.state, second.state)); + } + #[test] fn data_page_length_valid() { let data = Data::<()>::new(1.into()); @@ -1002,3 +1020,107 @@ mod tests { assert_eq!(retrieved, row3); } } + +#[cfg(all(test, wt_loom))] +mod cell_lock_models { + use super::{CellLocks, Link}; + use loom::{cell::UnsafeCell, sync::Arc, thread}; + + struct Protected { + locks: CellLocks, + value: UnsafeCell<(u64, u64)>, + } + + // Every access to value below holds the same row's read or write guard. + unsafe impl Sync for Protected {} + + #[test] + fn colliding_offsets_cannot_split_readers_from_a_writer() { + let mut model = loom::model::Builder::new(); + model.preemption_bound = Some(2); + model.max_branches = 10_000; + model.check(|| { + let protected = Arc::new(Protected { + locks: CellLocks::default(), + value: UnsafeCell::new((0, 0)), + }); + let first = Link { + page_id: 1.into(), + offset: 7, + length: 16, + }; + let second = Link { offset: 14, ..first }; + assert_eq!(CellLocks::start(first), CellLocks::start(second)); + let preceding = protected.locks.read(first).unwrap(); + let existing = protected.locks.read(second).unwrap(); + drop(preceding); + let reader = { + let protected = protected.clone(); + thread::spawn(move || { + let _guard = protected.locks.read(second).unwrap(); + protected.value.with(|value| unsafe { + let a = (*value).0; + thread::yield_now(); + assert_eq!(a, (*value).1); + }); + }) + }; + drop(existing); + { + let _guard = protected.locks.write(second).unwrap(); + protected.value.with_mut(|value| unsafe { + (*value).0 = 1; + thread::yield_now(); + (*value).1 = 1; + }); + } + reader.join().unwrap(); + }); + } + + #[test] + fn readers_and_writers_never_overlap_and_publish_complete_rows() { + let mut model = loom::model::Builder::new(); + model.preemption_bound = Some(2); + model.max_branches = 10_000; + model.check(|| { + let protected = Arc::new(Protected { + locks: CellLocks::default(), + value: UnsafeCell::new((0, 0)), + }); + let link = Link { + page_id: 1.into(), + offset: 64, + length: 16, + }; + let mut handles = Vec::new(); + for writer in [false, true] { + let protected = protected.clone(); + handles.push(thread::spawn(move || { + if writer { + let _guard = protected.locks.write(link).unwrap(); + protected.value.with_mut(|value| unsafe { + (*value).0 += 1; + thread::yield_now(); + (*value).1 += 1; + }); + } else { + let _guard = protected.locks.read(link).unwrap(); + protected.value.with(|value| unsafe { + let first = (*value).0; + thread::yield_now(); + assert_eq!(first, (*value).1); + }); + } + })); + } + for handle in handles { + handle.join().unwrap(); + } + let _guard = protected.locks.read(link).unwrap(); + protected.value.with(|value| unsafe { + assert_eq!(*value, (1, 1)); + }); + }); + } +} diff --git a/src/in_memory/empty_link_registry.rs b/src/in_memory/empty_link_registry.rs index 825c7ccc..7fa62fe5 100644 --- a/src/in_memory/empty_link_registry.rs +++ b/src/in_memory/empty_link_registry.rs @@ -1,13 +1,14 @@ -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use data_bucket::Link; use data_bucket::page::PageId; use derive_more::Into; use indexset::concurrent::multimap::BTreeMultiMap; use indexset::concurrent::set::BTreeSet; +use nagoya::sync::{Notify, OwnedRwLockReadGuard}; use parking_lot::FairMutex; -use tokio::sync::{Notify, OwnedRwLockReadGuard}; use crate::in_memory::DATA_INNER_LENGTH; @@ -65,13 +66,13 @@ impl IndexOrdLink { } impl PartialOrd for IndexOrdLink { - fn partial_cmp(&self, other: &Self) -> Option { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for IndexOrdLink { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { self.absolute_index().cmp(&other.absolute_index()) } } @@ -109,7 +110,7 @@ pub struct EmptyLinkRegistry { /// completes; vacuum takes the write side, so it cannot start reclaiming /// while any popped link is still being written through, and no new link /// can be popped while vacuum runs. - vacuum_lock: Arc>, + vacuum_lock: Arc>, /// How many times a caller has asked this registry for reclaimable space. /// @@ -138,7 +139,7 @@ pub struct EmptyLinkRegistry { /// reclamation, and the most valuable. A scattered single-row delete says /// nothing about where to look, so [`Self::push`] does not record /// anything and only [`Self::push_many`] does. - targeted_pages: FairMutex>, + targeted_pages: FairMutex>, } /// A [`Link`] popped from the registry, together with the read guard that @@ -343,7 +344,7 @@ impl EmptyLinkRegistry { return None; } - let guard = self.vacuum_lock.clone().try_read_owned().ok()?; + let guard = self.vacuum_lock.clone().try_read_owned()?; let _g = self.op_lock.lock(); @@ -375,7 +376,7 @@ impl EmptyLinkRegistry { /// Takes the vacuum (write) side of the exclusion: waits until every /// popped link's read guard is dropped, and blocks new pops while held. - pub async fn lock_vacuum(&self) -> tokio::sync::RwLockWriteGuard<'_, ()> { + pub async fn lock_vacuum(&self) -> nagoya::sync::RwLockWriteGuard<'_, ()> { self.vacuum_lock.write().await } @@ -418,11 +419,11 @@ impl EmptyLinkRegistry { /// ranged delete emptied part of, and it is where a sweep should look /// first. fn note_coalesced_pages(&self, links: &[Link], runs: &[IndexOrdLink]) { - let mut links_per_page: std::collections::BTreeMap = Default::default(); + let mut links_per_page: alloc::collections::BTreeMap = Default::default(); for link in links { *links_per_page.entry(link.page_id).or_default() += 1; } - let mut runs_per_page: std::collections::BTreeMap = Default::default(); + let mut runs_per_page: alloc::collections::BTreeMap = Default::default(); for run in runs { *runs_per_page.entry(run.0.page_id).or_default() += 1; } @@ -443,8 +444,8 @@ impl EmptyLinkRegistry { /// Draining rather than reading: a sweep that has taken them is /// responsible for them, and leaving them would make every later sweep /// re-prioritise pages that are already compact. - pub fn take_targeted_pages(&self) -> std::collections::BTreeSet { - std::mem::take(&mut *self.targeted_pages.lock()) + pub fn take_targeted_pages(&self) -> alloc::collections::BTreeSet { + core::mem::take(&mut *self.targeted_pages.lock()) } /// Wakes a parked vacuum when freeing crossed the configured threshold. diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index f118dff5..8eb198ea 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -1,6 +1,15 @@ +use alloc::collections::VecDeque; +use alloc::sync::Arc; +use alloc::{boxed::Box, vec::Vec}; +#[cfg(feature = "std")] use arc_swap::ArcSwap; +use core::fmt::Debug; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; +use core::sync::atomic::{AtomicU64, Ordering}; use data_bucket::page::PageId; use derive_more::{Display, Error, From}; +use hashbrown::HashSet; use parking_lot::Mutex; use parking_lot::RwLock; #[cfg(feature = "perf_measurements")] @@ -12,14 +21,6 @@ use rkyv::{ ser::{Serializer, allocator::ArenaHandle, sharing::Share}, util::AlignedVec, }; -use std::collections::{HashSet, VecDeque}; -use std::marker::PhantomData; -use std::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; -use std::{ - fmt::Debug, - sync::Arc, - sync::atomic::{AtomicU64, Ordering}, -}; use crate::in_memory::empty_link_registry::EmptyLinkRegistry; use crate::prelude::ArchivedRowWrapper; @@ -36,8 +37,41 @@ fn page_id_mapper(page_id: usize) -> usize { page_id - 1usize } +// Snapshot ownership without arc-swap's std thread-local bookkeeping. +// Clone under the lock, then release it before running any reader callback. +#[cfg(not(feature = "std"))] +#[derive(Debug)] +struct ArcSwap(RwLock>); + +#[cfg(not(feature = "std"))] +impl ArcSwap { + fn from_pointee(value: T) -> Self { + Self(RwLock::new(Arc::new(value))) + } + fn load(&self) -> Arc { + self.0.read().clone() + } + fn load_full(&self) -> Arc { + self.load() + } + fn store(&self, value: Arc) { + *self.0.write() = value; + } +} + const PAGE_DIRECTORY_CHUNK_SIZE: usize = 64; -const PAGE_DIRECTORY_ROOTS: usize = 64; +/// Roots in the page directory, so its reach is `ROOTS * CHUNK_SIZE` pages. +/// +/// **This was 64, which reached 4,096 pages: 64 MiB at the default page size.** +/// Past that, `publish` returns early and every page access falls back to an +/// `ArcSwap` snapshot of the owning list. A table of 4 KiB rows, which fit +/// three to a page, crosses it after twelve thousand rows. +/// +/// Raising it did not measurably change insert cost on that fixture - the +/// copy-on-write page list dominated, and still does at these sizes - so this +/// is a ceiling being moved rather than a cost being removed. 1,024 roots +/// reach 65,536 pages, or 1 GiB, for an 8 KiB array of pointers. +const PAGE_DIRECTORY_ROOTS: usize = 1024; const GHOSTED: u8 = 1 << 0; const DELETED: u8 = 1 << 1; const VACUUMED: u8 = 1 << 2; @@ -67,6 +101,104 @@ const RECLAIM_BATCH_LIMIT: usize = 256; /// gain. const RECLAIM_BACKLOG_TRIGGER: usize = RECLAIM_BATCH_LIMIT; +/// Pages per chunk of [`PageList`]. +/// +/// The list is copy-on-write, so an append copies whatever the writer has to +/// replace. Chunking bounds that at one chunk plus the (much shorter) spine, +/// instead of the whole list. +const PAGE_LIST_CHUNK: usize = 256; + +/// Owns every page, and appends one without copying the ones already there. +/// +/// **This was a `Vec` behind an `ArcSwap`, and appending cloned all of it.** +/// Every existing page is an `Arc`, so the clone was one atomic increment per +/// page, each touching a separately allocated page header: a cache miss apiece. +/// Appending page N cost O(N) and filling a table cost O(N^2). It only showed +/// up with large rows, because those are what make pages plentiful - at 4 KiB a +/// row, three rows to a page, per-row insert cost grew tenfold over twenty +/// thousand rows while a 256-byte row stayed flat. +/// +/// Hosted readers take an `ArcSwap` snapshot. Without std, snapshot acquisition +/// briefly locks the owning Arc; visits run after releasing that lock. +#[derive(Debug)] +struct PageList { + chunks: ArcSwap>>>>, +} + +impl PageList { + fn from_pages(pages: Vec>) -> Self { + let chunks = pages + .chunks(PAGE_LIST_CHUNK) + .map(|chunk| Arc::new(chunk.to_vec())) + .collect::>(); + Self { + chunks: ArcSwap::from_pointee(chunks), + } + } + + /// Append a page. Copies the last chunk, or starts a new one, plus the + /// spine of chunk pointers. + fn push(&self, page: Arc) { + let chunks = self.chunks.load_full(); + let mut next = (*chunks).clone(); + match next.last() { + // Every chunk but the last is full, so only the last can take one. + Some(last) if last.len() < PAGE_LIST_CHUNK => { + let mut grown = (**last).clone(); + grown.push(page); + *next.last_mut().expect("the branch matched on it") = Arc::new(grown); + } + _ => { + let mut chunk = Vec::with_capacity(PAGE_LIST_CHUNK); + chunk.push(page); + next.push(Arc::new(chunk)); + } + } + self.chunks.store(Arc::new(next)); + } + + fn len(&self) -> usize { + let chunks = self.chunks.load(); + match chunks.last() { + None => 0, + // Full but for the last, so its length is the only remainder. + Some(last) => (chunks.len() - 1) * PAGE_LIST_CHUNK + last.len(), + } + } + + fn get(&self, index: usize) -> Option> { + let chunks = self.chunks.load(); + chunks + .get(index / PAGE_LIST_CHUNK)? + .get(index % PAGE_LIST_CHUNK) + .cloned() + } + + /// Run `visit` against the page at `index`, borrowing it rather than + /// handing back an owned `Arc`. + /// + /// **`get` costs an atomic increment and the matching decrement on drop.** + /// A read that only needs the page for the length of one call pays both for + /// nothing, and it is measurable: routing the link-based read through `get` + /// moved a delete from 665 to 751 ns, while a select by primary key - which + /// goes through the page directory and never touches this - did not move at + /// all. + fn with_page(&self, index: usize, visit: impl FnOnce(&T) -> R) -> Option { + let chunks = self.chunks.load(); + let page = chunks.get(index / PAGE_LIST_CHUNK)?.get(index % PAGE_LIST_CHUNK)?; + Some(visit(page)) + } + + fn for_each(&self, mut visit: impl FnMut(&Arc)) { + let chunks = self.chunks.load(); + for chunk in chunks.iter() { + for page in chunk.iter() { + visit(page); + } + } + } +} + #[derive(Debug)] struct PageDirectoryChunk { pages: [AtomicPtr; PAGE_DIRECTORY_CHUNK_SIZE], @@ -75,7 +207,7 @@ struct PageDirectoryChunk { impl PageDirectoryChunk { fn new() -> Self { Self { - pages: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + pages: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), } } } @@ -95,7 +227,7 @@ struct PageDirectory { impl PageDirectory { fn new(pages: &[Arc]) -> Self { let directory = Self { - roots: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + roots: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), chunks: Mutex::new(Vec::new()), }; for (index, page) in pages.iter().enumerate() { @@ -115,7 +247,7 @@ impl PageDirectory { chunk = root.load(Ordering::Acquire); if chunk.is_null() { chunks.push(Box::new(PageDirectoryChunk::new())); - chunk = std::ptr::from_ref::>( + chunk = core::ptr::from_ref::>( chunks.last().expect("the chunk was just appended").as_ref(), ) .cast_mut(); @@ -261,7 +393,7 @@ where /// Immutable page-directory snapshots. Reads load one snapshot without a /// shared read-modify-write; rare growth copies and swaps the short vector. - pages: ArcSwap::WrappedRow, DATA_LENGTH>>>>, + pages: PageList::WrappedRow, DATA_LENGTH>>, /// Stable pointers for point access without ArcSwap's shared snapshot /// accounting. The corresponding `Arc`s remain owned by `pages`. page_directory: PageDirectory::WrappedRow, DATA_LENGTH>>, @@ -303,11 +435,11 @@ where return Ok(page); } - let page = { - let pages = self.pages.load(); - pages.get(index).map(Arc::as_ptr) - } - .ok_or(ExecutionError::PageNotFound(page_id))?; + let page = self + .pages + .get(index) + .map(|page| Arc::as_ptr(&page)) + .ok_or(ExecutionError::PageNotFound(page_id))?; // SAFETY: as above, the current directory retains this allocation and // all future directory snapshots clone its Arc. @@ -358,7 +490,7 @@ where /// thread can later collect it; it executes only after every reader /// pinned right now has unpinned. fn retire(&self, item: Retired) { - self.retire_many(std::iter::once(item)); + self.retire_many(core::iter::once(item)); } /// Queue several retired items behind one grace marker. @@ -578,7 +710,7 @@ where queued_page_retirements: AtomicUsize::new(0), // We are starting ID's from `1` because `0`'s page in file is info page. page_directory: PageDirectory::new(&pages), - pages: ArcSwap::from_pointee(pages), + pages: PageList::from_pages(pages), pages_write: Mutex::new(()), empty_links: EmptyLinkRegistry::::default(), empty_pages: Default::default(), @@ -602,7 +734,7 @@ where pending_retirements: AtomicUsize::new(0), queued_page_retirements: AtomicUsize::new(0), page_directory, - pages: ArcSwap::from_pointee(vec), + pages: PageList::from_pages(vec), pages_write: Mutex::new(()), empty_links: EmptyLinkRegistry::default(), empty_pages: Default::default(), @@ -743,18 +875,8 @@ where let _write = self.pages_write.lock(); if tried_page == page_id_mapper(self.current_page_id.load(Ordering::Acquire) as usize) { let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; - let pages = self.pages.load_full(); - let mut next = (*pages).clone(); let page = Arc::new(Data::new(index.into())); - next.push(page.clone()); - debug_assert_eq!(next.len(), pages.len() + 1); - debug_assert!( - next[..pages.len()] - .iter() - .zip(pages.iter()) - .all(|(new, old)| Arc::ptr_eq(new, old)) - ); - self.pages.store(Arc::new(next)); + self.pages.push(page.clone()); self.publish_page(&page); self.current_page_id.store(index, Ordering::Release); } @@ -771,9 +893,11 @@ where }; if let Some(page_id) = page_id { - let pages = self.pages.load(); let index = page_id_mapper(page_id.into()); - let page = pages[index].clone(); + let page = self + .pages + .get(index) + .expect("an empty page id names a page that was allocated"); { let _page_guard = page.access.write(); page.reset(); @@ -785,17 +909,7 @@ where let _write = self.pages_write.lock(); let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; let page = Arc::new(Data::new(index.into())); - let pages = self.pages.load_full(); - let mut next = (*pages).clone(); - next.push(page.clone()); - debug_assert_eq!(next.len(), pages.len() + 1); - debug_assert!( - next[..pages.len()] - .iter() - .zip(pages.iter()) - .all(|(new, old)| Arc::ptr_eq(new, old)) - ); - self.pages.store(Arc::new(next)); + self.pages.push(page.clone()); self.publish_page(&page); page @@ -842,22 +956,24 @@ where + Deserialize<::WrappedRow, HighDeserializer> + for<'a> rkyv::bytecheck::CheckBytes>, { - let pages = self.pages.load(); let page_id: usize = link.page_id.into(); let page_index = page_id .checked_sub(1) .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let page = pages - .get(page_index) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let wrapped = page.get_row_checked(link).map_err(ExecutionError::DataPageError)?; - if wrapped.is_ghosted() { - return Err(ExecutionError::Ghosted); - } - if wrapped.is_deleted() { - return Err(ExecutionError::Deleted); - } - Ok(wrapped.get_inner()) + // Borrowed rather than cloned: this is a read path and an `Arc` bump + // here showed up as a 13% slower delete. + self.pages + .with_page(page_index, |page| { + let wrapped = page.get_row_checked(link).map_err(ExecutionError::DataPageError)?; + if wrapped.is_ghosted() { + return Err(ExecutionError::Ghosted); + } + if wrapped.is_deleted() { + return Err(ExecutionError::Deleted); + } + Ok(wrapped.get_inner()) + }) + .ok_or(ExecutionError::PageNotFound(link.page_id))? } pub fn select_non_vacuumed(&self, link: Link) -> Result @@ -1119,9 +1235,7 @@ where } pub fn get_page(&self, page_id: PageId) -> Option::WrappedRow, DATA_LENGTH>>> { - let pages = self.pages.load(); - let page = pages.get(page_id_mapper(page_id.into()))?; - Some(page.clone()) + self.pages.get(page_id_mapper(page_id.into())) } /// Registers an already-indexed cell while rebuilding runtime metadata @@ -1138,16 +1252,19 @@ where Ok(()) } + #[cfg(feature = "std")] pub(crate) fn page_has_cells(&self, page_id: PageId) -> Result { let page = self.page_ref(page_id)?; Ok(page.has_live_cells()) } + #[cfg(feature = "std")] pub(crate) fn page_live_cell_count(&self, page_id: PageId) -> Result { let page = self.page_ref(page_id)?; Ok(page.live_cell_count()) } + #[cfg(feature = "std")] pub(crate) fn set_loaded_row_count(&self, count: usize) -> Result<(), ExecutionError> { let count = u64::try_from(count).map_err(|_| ExecutionError::RowCountOverflow)?; self.row_count.store(count, Ordering::Release); @@ -1156,6 +1273,7 @@ where /// Completes the vacuum's source-side accounting after every index has /// been swung to the destination link. + #[cfg(feature = "std")] pub(crate) fn remove_moved_cell(&self, link: Link) -> Result<(), ExecutionError> { self.remove_cell(link) } @@ -1170,11 +1288,10 @@ where /// Approximate under concurrency: a failing `save_row`'s transient /// reservation may be counted before its rollback. Metrics only. pub fn used_bytes(&self) -> u64 { - let pages = self.pages.load(); - pages - .iter() - .map(|p| u64::from(p.free_offset.load(Ordering::Relaxed))) - .sum() + let mut total = 0u64; + self.pages + .for_each(|page| total += u64::from(page.free_offset.load(Ordering::Relaxed))); + total } /// Copies a row to another page without exposing either mutable byte @@ -1188,6 +1305,7 @@ where /// concurrent low-level mutation may access either physical row while the /// move is in progress. After success, the caller must swing every index /// reference to the returned link before retiring `from_link`. + #[cfg(feature = "std")] pub(crate) unsafe fn move_row_for_vacuum( &self, from_link: Link, @@ -1230,7 +1348,7 @@ where } pub fn get_page_count(&self) -> usize { - self.pages.load().len() + self.pages.len() } pub fn get_empty_links(&self) -> Vec { @@ -1253,12 +1371,12 @@ where /// figure without it cannot be checked, because a sweep that never runs /// looks exactly like a sweep that is free. pub fn allocated_pages(&self) -> usize { - self.pages.load().len() + self.pages.len() } /// Heap bytes reserved by the fixed-size data-page allocations. pub fn allocated_bytes(&self) -> usize { - self.pages.load().len() * std::mem::size_of::::WrappedRow, DATA_LENGTH>>() + self.pages.len() * core::mem::size_of::::WrappedRow, DATA_LENGTH>>() } /// Pages allocated but currently on the empty list, so reusable without @@ -1281,14 +1399,15 @@ where &self.empty_links } - pub fn with_empty_links(mut self, links: Vec) -> Self { + pub fn with_empty_links(mut self, links: Vec) -> Result { let registry = EmptyLinkRegistry::default(); for l in links { + self.page_ref(l.page_id)?.reserve_restored_range(l)?; registry.push(l) } self.empty_links = registry; - self + Ok(self) } pub fn current_page_id(&self) -> PageId { @@ -1302,6 +1421,7 @@ where /// current page serves as the sweep's first destination. Concurrent /// inserts are safe: the insert path rechecks `current_page_id` under the /// page barrier before writing and retries if the target changed. + #[cfg(feature = "std")] pub(crate) fn rotate_current_for_vacuum(&self, page_id: PageId) { debug_assert!( self.get_page(page_id).is_some(), @@ -1348,12 +1468,12 @@ impl ExecutionError { #[cfg(test)] mod tests { - use std::collections::HashSet; - use std::sync::Arc; - use std::sync::atomic::Ordering; + use alloc::sync::Arc; + use core::sync::atomic::Ordering; + use core::time::Duration; + use hashbrown::HashSet; use std::sync::mpsc; use std::thread; - use std::time::Duration; use std::time::Instant; use parking_lot::RwLock; @@ -1627,7 +1747,7 @@ mod tests { impl Drop for RemoteReader { fn drop(&mut self) { let (disconnected, _rx) = mpsc::channel(); - let _ = std::mem::replace(&mut self.commands, disconnected); + let _ = core::mem::replace(&mut self.commands, disconnected); if let Some(thread) = self.thread.take() { thread.join().unwrap(); } diff --git a/src/in_memory/row.rs b/src/in_memory/row.rs index 5b38803b..38066fff 100644 --- a/src/in_memory/row.rs +++ b/src/in_memory/row.rs @@ -1,5 +1,5 @@ +use core::fmt::Debug; use rkyv::Archive; -use std::fmt::Debug; pub trait PublicationSafe: Send + Sync + 'static {} diff --git a/src/index/arctic.rs b/src/index/arctic.rs index 42336285..ea0b820c 100644 --- a/src/index/arctic.rs +++ b/src/index/arctic.rs @@ -1,10 +1,11 @@ //! Arctic adapter for memory-only unique WorkTable indexes. -use std::borrow::Borrow; -use std::fmt::{self, Debug}; -use std::marker::PhantomData; -use std::ops::{Bound, RangeBounds}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use alloc::{string::String, vec::Vec}; +use core::borrow::Borrow; +use core::fmt::{self, Debug}; +use core::marker::PhantomData; +use core::ops::{Bound, RangeBounds}; +use core::sync::atomic::{AtomicUsize, Ordering}; use arctic::{ConcurrentMap, Key, Order}; @@ -183,15 +184,25 @@ impl ArcticValue for u64 { } } +/// An offset or length exceeds Arctic's inline link representation. +#[derive(Debug)] +pub struct ArcticLinkError(pub data_bucket::Link); + +impl core::fmt::Display for ArcticLinkError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "link cannot be represented by Arctic: page {:?}, offset {}, length {}", + self.0.page_id, self.0.offset, self.0.length + ) + } +} +impl core::error::Error for ArcticLinkError {} + #[doc(hidden)] -pub fn validate_arctic_link(link: data_bucket::Link) -> eyre::Result<()> { +pub fn validate_arctic_link(link: data_bucket::Link) -> Result<(), ArcticLinkError> { if link.offset > u32::from(u16::MAX) || link.length > u32::from(u16::MAX) { - eyre::bail!( - "link cannot be represented by Arctic: page {:?}, offset {}, length {}", - link.page_id, - link.offset, - link.length, - ); + return Err(ArcticLinkError(link)); } Ok(()) } @@ -327,6 +338,7 @@ where self.inner.allocated_node_bytes() } + #[cfg(feature = "std")] pub(crate) fn export_topology( &mut self, mut encode: impl FnMut(&V) -> T, @@ -337,6 +349,7 @@ where self.inner.export_topology(|value| encode(&V::from_arctic(*value))) } + #[cfg(feature = "std")] pub(crate) fn from_topology( topology: arctic::topology::Topology, mut decode: impl FnMut(T) -> V, @@ -481,8 +494,9 @@ where #[cfg(test)] mod tests { - use std::ops::Bound; - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use core::ops::Bound; + use std::sync::Barrier; use super::{ArcticIndex, UniqueIndex}; diff --git a/src/index/arctic_multi.rs b/src/index/arctic_multi.rs index 744eba81..f9fc346c 100644 --- a/src/index/arctic_multi.rs +++ b/src/index/arctic_multi.rs @@ -45,10 +45,11 @@ //! the dead entry and retries with a fresh slot, and the SMR guard it holds //! keeps the memory valid throughout. -use std::borrow::Borrow; -use std::fmt::{self, Debug}; -use std::ops::{Bound, ControlFlow, RangeBounds}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use alloc::{boxed::Box, vec::Vec}; +use core::borrow::Borrow; +use core::fmt::{self, Debug}; +use core::ops::{Bound, ControlFlow, RangeBounds}; +use core::sync::atomic::{AtomicUsize, Ordering}; use arctic::{ConcurrentMap, Key as ArcticNativeKey, Order}; use parking_lot::RwLock; @@ -190,7 +191,7 @@ where /// Returns every `(key, value)` pair stored under `key`, in insertion /// order, as a stable snapshot. An unknown key yields an empty iterator. - pub fn get(&self, key: &K) -> std::vec::IntoIter<(K, V)> { + pub fn get(&self, key: &K) -> alloc::vec::IntoIter<(K, V)> { let raw = key.to_arctic(); let Some(slot) = self.inner.get(raw.borrow()) else { return Vec::new().into_iter(); @@ -222,7 +223,7 @@ where let mut slots = 0; while let Some((_, slot)) = entries.lend() { let links = slot.read(); - slots += std::mem::size_of::>>() + links.links.capacity() * std::mem::size_of::(); + slots += core::mem::size_of::>>() + links.links.capacity() * core::mem::size_of::(); } self.inner.allocated_node_bytes() + slots } @@ -287,8 +288,9 @@ where #[cfg(test)] mod tests { - use std::ops::Bound; - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use core::ops::Bound; + use std::sync::Barrier; use super::ArcticMultiIndex; diff --git a/src/index/available_index.rs b/src/index/available_index.rs index 32b18991..a9b3e4ac 100644 --- a/src/index/available_index.rs +++ b/src/index/available_index.rs @@ -1,3 +1,4 @@ +use alloc::{string::String, string::ToString}; pub trait AvailableIndex { fn to_string_value(&self) -> String; } diff --git a/src/index/congee.rs b/src/index/congee.rs index e564c754..42ba8a6d 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -1,9 +1,10 @@ //! Congee adapter for memory-only unique WorkTable indexes. -use std::fmt::{self, Debug}; -use std::ops::{Bound, RangeBounds}; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::{self, Debug}; +use core::ops::{Bound, RangeBounds}; +use core::sync::atomic::{AtomicUsize, Ordering}; use congee::{CongeeRaw, DefaultAllocator}; use parking_lot::Mutex; @@ -53,7 +54,7 @@ pub struct CongeeIndex { // serialize mutations until the backend offers the required visibility. mutation: Mutex<()>, len: AtomicUsize, - marker: std::marker::PhantomData<(K, V)>, + marker: core::marker::PhantomData<(K, V)>, } impl Debug for CongeeIndex { @@ -79,7 +80,7 @@ where inner: CongeeRaw::new_with_drainer(DefaultAllocator {}, drainer), mutation: Mutex::new(()), len: AtomicUsize::new(0), - marker: std::marker::PhantomData, + marker: core::marker::PhantomData, } } } @@ -113,7 +114,7 @@ where // SAFETY: callers guarantee that `pointer` was produced by // `Arc::into_raw(...).expose_provenance()` for the same `V` and still // owns one strong reference. - unsafe { Arc::from_raw(std::ptr::with_exposed_provenance(pointer)) } + unsafe { Arc::from_raw(core::ptr::with_exposed_provenance(pointer)) } } #[inline] @@ -180,12 +181,13 @@ where .map(|(key, pointer)| { // SAFETY: the pinned epoch keeps every returned tree-owned // pointer alive until its value has been cloned. - let value = unsafe { &*std::ptr::with_exposed_provenance::(pointer) }; + let value = unsafe { &*core::ptr::with_exposed_provenance::(pointer) }; (K::from_congee(key), value.clone()) }) .collect() } + #[cfg(feature = "std")] pub(crate) fn export_topology( &mut self, mut encode: impl FnMut(&V) -> T, @@ -193,10 +195,11 @@ where self.inner.export_topology(|pointer| { // SAFETY: every raw payload is a live tree-owned `Arc` pointer, // and the exclusive borrow prevents removal while it is cloned. - unsafe { encode(&*std::ptr::with_exposed_provenance::(pointer)) } + unsafe { encode(&*core::ptr::with_exposed_provenance::(pointer)) } }) } + #[cfg(feature = "std")] pub(crate) fn from_topology( topology: congee::topology::Topology, mut decode: impl FnMut(T) -> V, @@ -217,7 +220,7 @@ where inner, mutation: Mutex::new(()), len: AtomicUsize::new(len), - marker: std::marker::PhantomData, + marker: core::marker::PhantomData, }) } } @@ -238,7 +241,7 @@ where let pointer = self.inner.get(&key.into_congee(), &guard)?; // SAFETY: the epoch guard keeps the tree-owned `Arc` alive for the // duration of `read`, and the pointer originated from `Arc::into_raw`. - let value = unsafe { &*std::ptr::with_exposed_provenance::(pointer) }; + let value = unsafe { &*core::ptr::with_exposed_provenance::(pointer) }; Some(read(value)) } @@ -334,8 +337,9 @@ where #[cfg(test)] mod tests { - use std::ops::Bound; - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use core::ops::Bound; + use std::sync::Barrier; use super::{CongeeIndex, UniqueIndex}; diff --git a/src/index/mod.rs b/src/index/mod.rs index c36970cd..af864837 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -11,7 +11,9 @@ mod table_secondary_index; mod unique; mod unsized_node; -pub use arctic::{ArcticEntry, ArcticIndex, ArcticKey, ArcticStringKey, ArcticValue, validate_arctic_link}; +pub use arctic::{ + ArcticEntry, ArcticIndex, ArcticKey, ArcticLinkError, ArcticStringKey, ArcticValue, validate_arctic_link, +}; pub use arctic_multi::ArcticMultiIndex; pub use available_index::AvailableIndex; pub use congee::{CongeeIndex, CongeeKey}; @@ -23,13 +25,15 @@ pub use persistent_art::{ }; pub use persistent_wti::PersistentWtiIndex; pub use primary_index::PrimaryIndex; -pub use table_index::{ - TableIndex, TableIndexCdc, convert_change_events, convert_multi_change_events, convert_upstream_change_events, -}; +#[cfg(feature = "vanilla-index")] +pub use table_index::convert_upstream_change_events; +pub use table_index::{TableIndex, TableIndexCdc, convert_change_events, convert_multi_change_events}; pub use table_secondary_index::{ IndexError, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, }; -pub use unique::{UniqueIndex, UpstreamIndexMap, UpstreamIndexPair}; +pub use unique::UniqueIndex; +#[cfg(feature = "vanilla-index")] +pub use unique::{UpstreamIndexMap, UpstreamIndexPair}; pub use unsized_node::UnsizedNode; #[derive(Clone, Debug)] diff --git a/src/index/persistent_art.rs b/src/index/persistent_art.rs index eda5a690..2f73d88b 100644 --- a/src/index/persistent_art.rs +++ b/src/index/persistent_art.rs @@ -6,12 +6,13 @@ //! different stripes remain concurrent because their Set/Remove records //! commute during recovery. -use std::array; -use std::collections::hash_map::DefaultHasher; -use std::fmt::{self, Debug}; -use std::hash::{Hash, Hasher}; -use std::ops::RangeBounds; -use std::sync::atomic::{AtomicU64, Ordering}; +use alloc::vec::Vec; +use core::array; +use core::fmt::{self, Debug}; +use core::hash::{Hash, Hasher}; +use core::ops::RangeBounds; +use core::sync::atomic::{AtomicU64, Ordering}; +use rustc_hash::FxHasher as DefaultHasher; use data_bucket::Link; use indexset::cdc::change::{ChangeEvent, Id}; @@ -49,7 +50,7 @@ where K: ArcticKey, V: Clone + Debug + PartialEq + Send + Sync + 'static, { - pub fn get(&self, key: &K) -> std::vec::IntoIter<(K, V)> { + pub fn get(&self, key: &K) -> alloc::vec::IntoIter<(K, V)> { self.inner.get(key) } @@ -128,7 +129,7 @@ impl PersistentArtIndex { } fn mutation_stripe(&self, key: &K) -> &Mutex<()> { - let mut hasher = DefaultHasher::new(); + let mut hasher = DefaultHasher::default(); key.hash(&mut hasher); &self.mutation_stripes[hasher.finish() as usize % MUTATION_STRIPES] } @@ -318,7 +319,8 @@ where #[cfg(test)] mod tests { - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use std::sync::Barrier; use super::*; diff --git a/src/index/persistent_wti.rs b/src/index/persistent_wti.rs index 8c05edde..b8e629d8 100644 --- a/src/index/persistent_wti.rs +++ b/src/index/persistent_wti.rs @@ -9,11 +9,12 @@ //! not claims about the live WTI node position or maximum; the shadow validates //! that marker and derives the real structural metadata itself. -use std::array; -use std::fmt::{self, Debug}; -use std::hash::{Hash, Hasher}; -use std::ops::RangeBounds; -use std::sync::atomic::{AtomicU64, Ordering}; +use alloc::vec::Vec; +use core::array; +use core::fmt::{self, Debug}; +use core::hash::{Hash, Hasher}; +use core::ops::RangeBounds; +use core::sync::atomic::{AtomicU64, Ordering}; use data_bucket::Link; use indexset::cdc::change::{ChangeEvent, Id}; @@ -285,7 +286,7 @@ where #[cfg(test)] mod tests { - use std::collections::HashSet; + use hashbrown::HashSet; use data_bucket::page::PageId; diff --git a/src/index/primary_index.rs b/src/index/primary_index.rs index d8883498..49969579 100644 --- a/src/index/primary_index.rs +++ b/src/index/primary_index.rs @@ -1,8 +1,9 @@ //! Primary-key to row-location index. -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; diff --git a/src/index/table_index/cdc.rs b/src/index/table_index/cdc.rs index 0cb4a588..e8658a06 100644 --- a/src/index/table_index/cdc.rs +++ b/src/index/table_index/cdc.rs @@ -1,18 +1,26 @@ -use std::fmt::Debug; -use std::hash::Hash; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::concurrent::map::BTreeMap as VanillaIndexMap; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; -use crate::index::table_index::util::{convert_change_events, convert_upstream_change_events}; +#[cfg(feature = "vanilla-index")] +use crate::UpstreamIndexMap; +use crate::index::table_index::util::convert_change_events; +#[cfg(feature = "vanilla-index")] +use crate::index::table_index::util::convert_upstream_change_events; use crate::util::OffsetEqLink; -use crate::{ArcticIndex, ArcticKey, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, UniqueIndex, UpstreamIndexMap}; +use crate::{ArcticIndex, ArcticKey, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, UniqueIndex}; pub trait TableIndexCdc { fn insert_cdc(&self, value: T, link: Link) -> (Option, Vec>>); @@ -74,6 +82,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl TableIndexCdc for UpstreamIndexMap, Node> where T: Debug + Eq + Hash + Clone + Send + Ord, diff --git a/src/index/table_index/mod.rs b/src/index/table_index/mod.rs index a405b5d3..8a40d6ed 100644 --- a/src/index/table_index/mod.rs +++ b/src/index/table_index/mod.rs @@ -1,24 +1,32 @@ -use std::fmt::Debug; -use std::hash::Hash; +use core::fmt::Debug; +use core::hash::Hash; use data_bucket::Link; use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; +#[cfg(feature = "vanilla-index")] +use crate::UpstreamIndexMap; use crate::util::OffsetEqLink; +#[allow(unused_imports)] +use crate::{}; use crate::{ ArcticIndex, ArcticKey, ArcticMultiIndex, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, PersistentArcticIndex, - PersistentArcticMultiIndex, PersistentCongeeIndex, PersistentWtiIndex, UniqueIndex, UpstreamIndexMap, + PersistentArcticMultiIndex, PersistentCongeeIndex, PersistentWtiIndex, UniqueIndex, }; mod cdc; pub mod util; pub use cdc::TableIndexCdc; -pub use util::{convert_change_events, convert_multi_change_events, convert_upstream_change_events}; +#[cfg(feature = "vanilla-index")] +pub use util::convert_upstream_change_events; +pub use util::{convert_change_events, convert_multi_change_events}; pub trait TableIndex { fn insert(&self, value: T, link: Link) -> Option; @@ -114,6 +122,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl TableIndex for UpstreamIndexMap where T: Debug + Eq + Hash + Clone + Send + Ord, diff --git a/src/index/table_index/util.rs b/src/index/table_index/util.rs index 4ffc57cb..e056db39 100644 --- a/src/index/table_index/util.rs +++ b/src/index/table_index/util.rs @@ -1,7 +1,10 @@ +use alloc::vec::Vec; use indexset::cdc::change::ChangeEvent; use indexset::core::multipair::MultiPair; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::cdc::change::ChangeEvent as VanillaChangeEvent; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; pub fn convert_change_event(ev: ChangeEvent>) -> ChangeEvent> @@ -142,6 +145,7 @@ where /// Normalizes upstream IndexSet CDC events into WorkTablesIndex's event type, /// which remains the stable persistence boundary used by DataBucket. +#[cfg(feature = "vanilla-index")] pub fn convert_upstream_change_events( evs: Vec>>, ) -> Vec>> @@ -193,6 +197,7 @@ where .collect() } +#[cfg(feature = "vanilla-index")] fn upstream_pair(pair: VanillaPair) -> Pair where L1: Into, diff --git a/src/index/table_secondary_index/cdc.rs b/src/index/table_secondary_index/cdc.rs index 804014d8..0b7914da 100644 --- a/src/index/table_secondary_index/cdc.rs +++ b/src/index/table_secondary_index/cdc.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use alloc::vec::Vec; +use hashbrown::HashMap; use data_bucket::Link; diff --git a/src/index/table_secondary_index/index_events.rs b/src/index/table_secondary_index/index_events.rs index 9183da9b..acdf0a9f 100644 --- a/src/index/table_secondary_index/index_events.rs +++ b/src/index/table_secondary_index/index_events.rs @@ -1,6 +1,6 @@ use crate::prelude::IndexChangeEventId; +use hashbrown::HashMap; use indexset::cdc::change; -use std::collections::HashMap; pub trait TableSecondaryIndexEventsOps { fn extend(&mut self, another: Self) diff --git a/src/index/table_secondary_index/info.rs b/src/index/table_secondary_index/info.rs index 569a4953..6da1dfdc 100644 --- a/src/index/table_secondary_index/info.rs +++ b/src/index/table_secondary_index/info.rs @@ -1,4 +1,5 @@ use crate::prelude::IndexInfo; +use alloc::vec::Vec; pub trait TableSecondaryIndexInfo { fn index_info(&self) -> Vec; diff --git a/src/index/table_secondary_index/mod.rs b/src/index/table_secondary_index/mod.rs index 69434a82..979a58ef 100644 --- a/src/index/table_secondary_index/mod.rs +++ b/src/index/table_secondary_index/mod.rs @@ -1,9 +1,10 @@ +use alloc::vec::Vec; mod cdc; mod index_events; mod info; use data_bucket::Link; -use std::collections::HashMap; +use hashbrown::HashMap; use crate::WorkTableError; use crate::{AvailableIndex, Difference}; @@ -13,6 +14,11 @@ pub use index_events::TableSecondaryIndexEventsOps; pub use info::TableSecondaryIndexInfo; pub trait TableSecondaryIndex { + /// Hold through secondary maintenance and authoritative row publication. + /// Non-columnar indexes pay no synchronization cost. + fn row_publication(&self) -> Option> { + None + } fn save_row(&self, row: Row, link: Link) -> Result<(), IndexError>; fn reinsert_row( &self, @@ -93,6 +99,10 @@ pub enum IndexError { at: IndexNameEnum, inserted_already: Vec, }, + ColumnSlotIdExhausted { + bits: u8, + inserted_already: Vec, + }, NotFound, } @@ -106,6 +116,10 @@ where at, inserted_already: _, } => WorkTableError::AlreadyExists(at.to_string_value()), + IndexError::ColumnSlotIdExhausted { + bits, + inserted_already: _, + } => WorkTableError::ColumnSlotIdExhausted(bits), IndexError::NotFound => WorkTableError::NotFound, } } diff --git a/src/index/unique.rs b/src/index/unique.rs index 55e559f3..a6f63724 100644 --- a/src/index/unique.rs +++ b/src/index/unique.rs @@ -4,15 +4,23 @@ //! backend's guard type. That keeps generated code independent from the //! concurrency and reclamation strategy used by each index implementation. -use std::fmt::Debug; -use std::hash::Hash; -use std::ops::RangeBounds; +// Only `UpstreamIndexMap`'s default node type and the tests name `Vec`, and the +// first of those is behind `vanilla-index`. Ungated, this warns on every +// `--no-default-features` build, which is the build that has to stay quiet. +#[cfg(any(feature = "vanilla-index", test))] +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; +use core::ops::RangeBounds; use crate::IndexMap; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::concurrent::map::BTreeMap as VanillaIndexMap; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; /// Point, mutation, and ordered-scan operations used by generated unique @@ -131,6 +139,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl UniqueIndex for VanillaIndexMap where K: Debug + Eq + Hash + Clone + Send + Ord + 'static, @@ -198,15 +207,16 @@ where self.range(range).map(|(_, value)| value.clone()) } } - /// Vanilla upstream IndexSet map, kept distinct from WorkTable's default /// WorkTablesIndex alias so both implementations may coexist in one binary. +#[cfg(feature = "vanilla-index")] pub type UpstreamIndexMap>> = VanillaIndexMap; +#[cfg(feature = "vanilla-index")] pub type UpstreamIndexPair = VanillaPair; #[cfg(test)] mod tests { - use std::sync::Arc; + use alloc::sync::Arc; use super::{UniqueIndex, UpstreamIndexMap}; use crate::{ArcticIndex, CongeeIndex, IndexMap}; @@ -260,7 +270,7 @@ mod tests { let iterated = index.iter_values().find(|(candidate, _)| *candidate == key); panic!( "backend={}, key={key}, point={value:?}, iterated={iterated:?}", - std::any::type_name::(), + core::any::type_name::(), ); } } @@ -292,7 +302,7 @@ mod tests { threads.push(std::thread::spawn(move || { for sequence in 0..1_000_u64 { let key = worker * 1_000 + sequence; - let backend = std::any::type_name::(); + let backend = core::any::type_name::(); assert_eq!( index.insert_value_checked(key, key + 1), Some(()), diff --git a/src/index/unsized_node.rs b/src/index/unsized_node.rs index eefd0c8c..3214cada 100644 --- a/src/index/unsized_node.rs +++ b/src/index/unsized_node.rs @@ -1,11 +1,12 @@ +use alloc::vec::Vec; use data_bucket::{SizeMeasurable, UnsizedIndexPageUtility, VariableSizeMeasurable}; use indexset::core::node::NodeLike; -use std::borrow::Borrow; -use std::collections::Bound; -use std::fmt::Debug; -use std::ops::Deref; -use std::slice::Iter; +use core::borrow::Borrow; +use core::fmt::Debug; +use core::ops::Bound; +use core::ops::Deref; +use core::slice::Iter; pub const UNSIZED_HEADER_LENGTH: u32 = 64; @@ -238,7 +239,7 @@ where fn replace(&mut self, idx: usize, value: T) -> Option { let value_size = value.aligned_size(); if let Some(old) = self.inner.get_mut(idx) { - let old = std::mem::replace(old, value); + let old = core::mem::replace(old, value); self.length += value_size; self.removed_length += old.aligned_size(); if idx + 1 == self.inner.len() { diff --git a/src/lib.rs b/src/lib.rs index 173b172f..828d6862 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,76 +1,243 @@ +#![cfg_attr(not(feature = "std"), no_std)] #![doc = include_str!("../docs/crate.md")] +#[macro_use] +extern crate alloc; + +/// Generated code names `worktable::` paths, which must also resolve inside +/// this crate, where `worktable!` is invoked for the persistence queue. +extern crate self as worktable; + +/// A fixed-capacity table whose rows are claimed without blocking. +pub mod atomic_key_table; +mod columnar; +#[cfg(feature = "std")] +pub mod fsx; pub mod in_memory; mod index; pub mod lock; mod mem_stat; +#[cfg(feature = "std")] pub mod migration; pub mod partition; pub mod persistence; +/// Which async runtime a table's work runs on. +/// +/// The module is available to a `no_std` build even though the backends inside +/// it are not. The trait and the profile machinery are types a table names, and +/// a table names them whether or not it ever spawns; only the impls need +/// threads, and those are gated within. +pub mod runtime; +mod storage_catalog; + mod primary_key; mod row; mod table; mod util; +/// The page codec behind `storage: vec` plus `persist: true`. +pub mod vec_hydrate; #[cfg(feature = "s3-support")] pub mod features; +#[cfg(feature = "s3-support")] +pub use features::{DatabaseS3DiskConfig, DatabaseS3PersistenceEngine}; +pub use columnar::{ + ClusteredColumnarIndex, ColumnCompression, ColumnSlotId, ColumnSlotId8, ColumnSlotId16, ColumnSlotId32, + ColumnSlotId64, ColumnarColumn, ColumnarRowRef, next_columnar_incarnation, +}; pub use index::*; +#[cfg(feature = "std")] pub use persistence::{ LoadMode, PersistedWorkTable, PersistenceConfig, PersistenceLoadError, UnloadFailure, UnloadReport, }; pub use row::*; +pub use storage_catalog::{Database, DatabaseCatalog, GeneratedSystemCatalog, SystemCatalogView}; +#[cfg(feature = "s3-support")] +pub type S3Database = Database; pub use table::*; pub use data_bucket; pub use worktable_codegen::migration_engine; +/// Declares the process's runtime profiles. See `runtime::Profile`. +pub use worktable_codegen::runtimes; pub use worktable_codegen::worktable; pub use worktable_codegen::worktable_version; /// The schema language, so the declaration each table embeds can be read /// without taking a second dependency and matching its version by hand. +#[cfg(feature = "std")] pub use worktable_dsl; +#[cfg(feature = "s3-support")] +pub use worktable_codegen::database_s3_persistence; #[cfg(feature = "s3-support")] pub use worktable_codegen::s3_sync_persistence; +/// Emits its body only when `worktable` itself was built with `std`. +/// +/// `worktable!` expands in the consumer's crate, so a `#[cfg(feature = "std")]` +/// it emits would test the *consumer's* feature of that name, which is a +/// different flag or no flag at all. This macro is expanded here, against this +/// crate's features, and so says what the macro actually needs to ask: does the +/// `worktable` I am generating against have a disk and threads? +/// +/// It exists for the generated `vacuum` method and the `ArtPersistenceKey` +/// impl. Both name types that are std-only for real reasons rather than by +/// grouping: `EmptyDataVacuum` is not empty despite the name and holds the +/// data pages, lock manager and persistence sink. +#[cfg(feature = "std")] +#[macro_export] +#[doc(hidden)] +macro_rules! __wt_if_std { + ($($item:tt)*) => { $($item)* }; +} + +#[cfg(not(feature = "std"))] +#[macro_export] +#[doc(hidden)] +macro_rules! __wt_if_std { + ($($item:tt)*) => {}; +} + pub mod prelude { + /// The filesystem this crate goes through. Generated code opens files by + /// this path, so a consumer of `worktable!` gets the same backend the + /// crate itself uses without naming it. + #[cfg(feature = "std")] + pub use crate::fsx; + /// The runtime a table names, and the registry of pool flavors it can + /// pick between. `worktable!` emits these type names, so they have to + /// resolve in the consumer's crate for the same reason `fsx` does. + pub use crate::runtime::{ + Elapsed, FLAVOR_COUNT, Flavor, FlavorMarker, Profile, Runtime, RuntimeJoinHandle, RuntimeNotified, + RuntimeNotify, RuntimeRwLock, RuntimeSemaphore, RuntimeSemaphorePermit, RuntimeUnpinned, TableRuntime, Tuning, + }; + /// The house backend and its pool flavors, plus the process-level + /// selection a benchmark reads. Gated with the backends themselves: a + /// `no_std` build has the trait but nothing that spawns. + #[cfg(feature = "std")] + pub use crate::runtime::{ + Locality, LowLatency, NagoyaRt, SharedSlot, Spread, Throughput, WideInjector, describe_tuning, engine_executor, + engine_flavor, env_override, executor_for_flavor, parse_selection, + }; + #[cfg(all(feature = "std", feature = "tokio-runtime"))] + pub use crate::runtime::{TokioJoinHandle, TokioRt}; + /// The three async primitives generated code awaits on. Re-exported for + /// the same reason `fsx` is: `worktable!` expands inside the consumer's + /// crate, so every path it emits has to resolve there. Emitting `tokio::` + /// made a whole runtime part of the macro's contract, and every consumer + /// carried it whether or not they ran one. + pub use nagoya::{sleep, timeout, yield_now}; + + pub use alloc::boxed::Box; + /// The `BTreeMap` entry, under a name a macro expansion can write. + /// + /// A `storage: vec` table needs it to refuse a duplicate key in one traversal + /// rather than a `contains_key` followed by an `insert`. The path is + /// re-exported rather than emitted, for the same reason everything else + /// here is: `alloc::` does not resolve in a consumer that never declared + /// `extern crate alloc`. + pub use alloc::collections::btree_map::Entry as BTreeMapEntry; + pub use alloc::collections::{BTreeMap, BTreeSet}; + /// What `using fxhash` stores. + /// + /// `hashbrown` rather than `std::collections::HashMap`, because this crate + /// is `no_std` and `std`'s map is not reachable from one; `FxBuildHasher` + /// rather than SipHash, because the keys here are already-checked column + /// values and not adversarial input, and SipHash is most of a hash map's + /// lookup cost. + /// + /// Re-exported rather than emitted for the same reason as everything else + /// in this module: a consumer that never depended on `hashbrown` or + /// `rustc-hash` still has to be able to compile the expansion. + pub use hashbrown::HashMap as FxHashMapInner; + /// The entry API, so a `vec: true` insert can refuse a duplicate key in one + /// traversal rather than a `contains_key` followed by an `insert`. + pub use hashbrown::hash_map::Entry as HashMapEntry; + pub use rustc_hash::FxBuildHasher; + + /// A hash map from a column value to a row position. + /// + /// Point operations only: it cannot answer a range, which is why the + /// generator refuses to emit `range` and `range_by_` on a table whose index + /// is one of these, and why `using fxhash` is accepted on `vec: true` and + /// refused on a paged table. + pub type FxHashMap = hashbrown::HashMap; + pub use alloc::sync::Arc; + /// `Vec` and `vec!` for the same reason as `Arc` above: a `no_std` + /// consumer has neither in scope, and the expansion uses both. + pub use alloc::vec; + pub use alloc::vec::{IntoIter, Vec}; + /// The one combinator generated code awaits on, re-exported for the same + /// reason as `sleep` and `timeout`: emitting `futures::` made that crate + /// part of the macro's contract, so every consumer had to depend on it. + pub use futures::future::join_all; + pub use hashbrown::{HashMap, HashSet}; + + pub use crate::atomic_key_table::AtomicKeyTable; pub use crate::in_memory::{ArchivedRowWrapper, Data, DataPages, Query, RowWrapper, StorableRow}; pub use crate::lock::FullRowLock; pub use crate::lock::{Lock, RowLock}; pub use crate::lock::{LockAcquirer, LockGuard, LockMap, PendingLock}; pub use crate::mem_stat::MemStat; - pub use crate::partition::{MAX_PARTITIONS, PartRef, PartitionError, PartitionSet}; + pub use crate::partition::{DenseError, DenseRows, MAX_PARTITIONS, PartRef, PartitionError, PartitionSet}; + pub use crate::persistence::{AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, OperationId}; + #[cfg(feature = "std")] pub use crate::persistence::{ - AcknowledgeOperation, ArtPersistenceKey, DeleteOperation, DiskConfig, DiskPersistenceEngine, - IndexTableOfContents, InsertOperation, LoadMode, Operation, OperationId, PersistedWorkTable, PersistenceConfig, - PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, PersistenceMonitor, - PersistenceResult, PersistenceState, PersistenceTask, ReadOnlyPersistenceEngine, SpaceArcticIndex, - SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, - SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, SpaceLogicalMultiIndex, - SpaceLogicalMultiIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, UnloadFailure, UnloadReport, - UpdateOperation, load_persisted_state, map_index_pages_to_toc_and_general, - map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, validate_events, + ArtPersistenceKey, DiskConfig, DiskPersistenceEngine, IndexTableOfContents, LoadMode, PersistedWorkTable, + PersistenceConfig, PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, + PersistenceMonitor, PersistenceResult, PersistenceState, PersistenceTask, ReadOnlyPersistenceEngine, + SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, + SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, + SpaceLogicalMultiIndex, SpaceLogicalMultiIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, + UnloadFailure, UnloadReport, load_persisted_state, map_index_pages_to_toc_and_general, + map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, }; + pub use crate::persistence::{OperationType, UpdateOperation, validate_events}; pub use crate::primary_key::{ PrimaryKeyGenerator, PrimaryKeyGeneratorRange, PrimaryKeyGeneratorState, TablePrimaryKey, }; - pub use crate::table::select::{Order, QueryParams, SelectQueryBuilder, SelectQueryExecutor}; + pub use crate::table::select::{ + Order, QueryParams, SelectQueryAsyncExecutor, SelectQueryBuilder, SelectQueryExecutor, SelectQueryFuture, + }; pub use crate::table::system_info::{IndexInfo, IndexKind, SystemInfo}; pub use crate::util::{OffsetEqLink, OrderedF32Def, OrderedF64Def}; + /// The page codec a `storage: vec` table unloads and loads through. + pub use crate::vec_hydrate::{Codec, LoadError, NotAnArchive, RowTooLarge, from_pages, to_pages}; + #[allow(unused_imports)] + pub use crate::{}; pub use crate::{ ArcticEntry, ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticStringKey, AvailableIndex, BatchDeleteError, - BatchInsertError, CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, + BatchInsertError, ClusteredColumnarIndex, ColumnCompression, ColumnSlotId, ColumnSlotId8, ColumnSlotId16, + ColumnSlotId32, ColumnSlotId64, ColumnarColumn, ColumnarRowRef, CongeeIndex, CongeeKey, Database, + DatabaseCatalog, Difference, GeneratedSystemCatalog, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, PersistentArcticIndex, PersistentArcticMultiIndex, PersistentArtIndex, PersistentCongeeIndex, PersistentWtiIndex, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, - UpstreamIndexMap, UpstreamIndexPair, WorkTable, WorkTableError, vacuum::EmptyDataVacuum, - vacuum::VacuumPersistence, vacuum::WorkTableVacuum, validate_arctic_link, + WorkTable, WorkTableError, next_columnar_incarnation, validate_arctic_link, }; + #[cfg(feature = "s3-support")] + pub use crate::{DatabaseS3DiskConfig, DatabaseS3PersistenceEngine, S3Database}; + /// The upstream IndexSet backend, when the `vanilla-index` feature selects it. + #[cfg(feature = "vanilla-index")] + pub use crate::{UpstreamIndexMap, UpstreamIndexPair}; + #[cfg(feature = "std")] + pub use crate::{ + vacuum::EmptyDataVacuum, vacuum::VacuumPacing, vacuum::VacuumPersistence, vacuum::WorkTableVacuum, + }; + /// `eyre` and `uuid`, for the same reason as `rkyv` above: `worktable!` + /// expands in the consumer's crate, so every path it emits has to resolve + /// there. Emitting a bare `eyre::` made that crate part of the macro's + /// contract, and a consumer who never mentions eyre had to depend on it + /// anyway to compile a table declaration. + #[cfg(feature = "std")] + pub use ::eyre; + pub use ::uuid; pub use data_bucket::{ DATA_VERSION, DataPage, GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, INNER_PAGE_SIZE, IndexPage, Interval, Link, PAGE_SIZE, PageType, Persistable, PersistableIndex, SizeMeasurable, SizeMeasure, SpaceInfoPage, TableOfContentsPage, UnsizedIndexPage, VariableSizeMeasurable, VariableSizeMeasure, align, - get_index_page_size_from_data_length, map_data_pages_to_general, parse_data_page, parse_page, persist_page, + data_page_row_capacity, map_data_pages_to_general, parse_data_page, parse_page, persist_page, seek_to_page_start, update_at, }; pub use derive_more::{Display as MoreDisplay, From, Into}; @@ -80,6 +247,16 @@ pub mod prelude { }; pub use ordered_float::OrderedFloat; pub use parking_lot::RwLock as ParkingRwLock; + pub use parking_lot::RwLockReadGuard as ParkingRwLockReadGuard; + /// rkyv itself, so a generated row can derive its traits without the + /// consumer declaring rkyv. `worktable!`'s paged path still emits a bare + /// `rkyv::` and is the remaining half of that leak. + pub use rkyv; + + /// Node capacity representable by the persisted index's u16 slot format. + pub fn get_index_page_size_from_data_length(length: usize) -> usize { + data_bucket::get_index_page_size_from_data_length::(length).min(usize::from(u16::MAX)) + } pub use worktable_codegen::{MemStat, PersistIndex, PersistTable}; pub const WT_INDEX_EXTENSION: &str = ".wt.idx"; diff --git a/src/lock/map.rs b/src/lock/map.rs index ea79742e..065a7591 100644 --- a/src/lock/map.rs +++ b/src/lock/map.rs @@ -1,10 +1,11 @@ -use std::collections::HashMap; -use std::collections::hash_map::DefaultHasher; -use std::fmt::Debug; -use std::hash::{Hash, Hasher}; -use std::ops::Deref; -use std::sync::Arc; -use std::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::{Hash, Hasher}; +use core::ops::Deref; +use core::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering}; +use hashbrown::HashMap; +use rustc_hash::FxHasher as DefaultHasher; use parking_lot::RwLock; @@ -42,7 +43,7 @@ pub struct BulkMutationGuard { #[derive(Debug)] struct LockEntry { - lock: Arc>, + lock: Arc>, acquirers: Arc, } @@ -57,7 +58,7 @@ where LockType: RowLock, PrimaryKey: Hash + Eq + Debug + Clone, { - lock: Option>>, + lock: Option>>, acquirers: Arc, lock_map: Arc>, primary_key: PrimaryKey, @@ -84,7 +85,7 @@ where LockType: RowLock, PrimaryKey: Hash + Eq + Debug + Clone, { - type Target = tokio::sync::RwLock; + type Target = nagoya::sync::RwLock; fn deref(&self) -> &Self::Target { self.lock.as_deref().expect("the acquisition lock exists until drop") @@ -120,7 +121,7 @@ impl Drop for BulkMutationGuard { /// # Sync/async lock boundary /// /// The `parking_lot` map guard is never returned and never crosses an -/// `.await`. Acquisition clones a tracked `Arc>` before +/// `.await`. Acquisition clones a tracked `Arc>` before /// releasing the map guard. Cleanup may synchronously take the short-lived map /// write guard, but only probes the per-row lock with `try_read`; it never waits /// on a Tokio lock while holding the map. This one-way boundary prevents a @@ -138,7 +139,7 @@ impl Default for LockMap { Self { map: RwLock::new(HashMap::new()), next_id: AtomicU16::default(), - mutation_stripes: Arc::new(std::array::from_fn(|_| MutationStripe::default())), + mutation_stripes: Arc::new(core::array::from_fn(|_| MutationStripe::default())), bulk_mutations: Arc::default(), } } @@ -157,8 +158,8 @@ where pub fn insert( &self, key: PrimaryKey, - lock: Arc>, - ) -> Option>> { + lock: Arc>, + ) -> Option>> { self.map .write() .insert( @@ -173,7 +174,7 @@ where /// Returns an untracked raw lock clone, which keeps the map entry alive /// until that clone is dropped. - pub fn get(&self, key: &PrimaryKey) -> Option>> { + pub fn get(&self, key: &PrimaryKey) -> Option>> { self.map.read().get(key).map(|entry| entry.lock.clone()) } @@ -208,7 +209,7 @@ where let mut map = self.map.write(); // Re-check: another task can insert between the read and write guards. let entry = map.entry(key.clone()).or_insert_with(|| LockEntry { - lock: Arc::new(tokio::sync::RwLock::new(f())), + lock: Arc::new(nagoya::sync::RwLock::new(f())), acquirers: Arc::new(AtomicUsize::new(0)), }); entry.acquirers.fetch_add(1, Ordering::AcqRel); @@ -230,7 +231,7 @@ where { let mut set = self.map.write(); let should_remove = set.get(key).is_some_and(|entry| { - let Ok(guard) = entry.lock.try_read() else { + let Some(guard) = entry.lock.try_read() else { return false; }; !guard.is_locked() @@ -283,7 +284,7 @@ where } fn stripe_of(key: &PrimaryKey) -> usize { - let mut hasher = DefaultHasher::new(); + let mut hasher = DefaultHasher::default(); key.hash(&mut hasher); (hasher.finish() as usize) % MUTATION_STRIPE_COUNT } @@ -343,9 +344,9 @@ where while gate.serving.load(Ordering::Acquire) != ticket { if spins < 16 { spins += 1; - std::hint::spin_loop(); + core::hint::spin_loop(); } else { - std::thread::yield_now(); + crate::util::yield_now(); } } diff --git a/src/lock/mod.rs b/src/lock/mod.rs index db352d32..15c7bced 100644 --- a/src/lock/mod.rs +++ b/src/lock/mod.rs @@ -1,15 +1,16 @@ +use alloc::vec::Vec; mod map; mod row_lock; -use std::cell::Cell; -use std::fmt::Debug; -use std::future::Future; -use std::hash::{Hash, Hasher}; -use std::marker::PhantomData; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::task::{Context, Poll}; +use alloc::sync::Arc; +use core::cell::Cell; +use core::fmt::Debug; +use core::future::Future; +use core::hash::{Hash, Hasher}; +use core::marker::PhantomData; +use core::pin::Pin; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::{Context, Poll}; use futures::task::AtomicWaker; use parking_lot::Mutex; @@ -76,8 +77,7 @@ where /// Explicitly unlocks the [`Lock`] before the [`LockGuard`] is [`Drop`]ped. pub fn unlock(self) { - self.lock.unlock(); - self.lock_map.remove_with_lock_check(&self.primary_key); + drop(self); } } @@ -166,6 +166,8 @@ where #[derive(Debug)] pub struct Lock { + // A wrapping diagnostic label, not dependency identity. The existing + // locked allocation stays unique and stable for this lock lifetime. id: u16, locked: Arc, wakers: Mutex>>, @@ -173,7 +175,7 @@ pub struct Lock { impl PartialEq for Lock { fn eq(&self, other: &Self) -> bool { - self.id.eq(&other.id) + Arc::ptr_eq(&self.locked, &other.locked) } } @@ -181,7 +183,7 @@ impl Eq for Lock {} impl Hash for Lock { fn hash(&self, state: &mut H) { - Hash::hash(&self.id, state) + Hash::hash(&Arc::as_ptr(&self.locked), state) } } @@ -213,6 +215,7 @@ impl Lock { } } + /// Diagnostic label; labels may repeat and do not define lock equality. pub fn id(&self) -> u16 { self.id } @@ -261,7 +264,7 @@ impl Future for LockWait { // Spin phase: try up to MAX_SPINS before going async for _ in 0..MAX_SPINS { - std::hint::spin_loop(); + core::hint::spin_loop(); if !self.locked.load(Ordering::Acquire) { return Poll::Ready(()); } @@ -282,6 +285,20 @@ mod tests { use super::*; use std::panic::AssertUnwindSafe; + #[test] + #[allow(clippy::mutable_key_type)] + fn repeated_labels_do_not_remove_distinct_dependencies() { + let first = Arc::new(Lock::new(7)); + let second = Arc::new(Lock::new(7)); + let dependencies: hashbrown::HashSet<_> = + hashbrown::HashSet::from_iter([first.clone(), first.clone(), second.clone()]); + assert_eq!(dependencies.len(), 2); + first.unlock(); + assert_eq!(dependencies.iter().filter(|lock| lock.is_locked()).count(), 1); + second.unlock(); + assert!(dependencies.iter().all(|lock| !lock.is_locked())); + } + #[test] fn test_unlock_on_drop() { let lock = Arc::new(Lock::new(1)); @@ -372,7 +389,7 @@ mod tests { // Create and insert a lock let (lock_type, lock) = FullRowLock::with_lock(lock_map.next_id()); - let rw_lock = Arc::new(tokio::sync::RwLock::new(lock_type)); + let rw_lock = Arc::new(nagoya::sync::RwLock::new(lock_type)); lock_map.insert(pk, rw_lock); // Verify the lock is in the map diff --git a/src/lock/row_lock.rs b/src/lock/row_lock.rs index 9e8aff49..e9022790 100644 --- a/src/lock/row_lock.rs +++ b/src/lock/row_lock.rs @@ -1,7 +1,7 @@ -use std::collections::HashSet; -use std::fmt::Debug; -use std::hash::Hash; -use std::sync::Arc; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::hash::Hash; +use hashbrown::HashSet; use crate::lock::{Lock, LockGuard, LockMap, LockWait}; diff --git a/src/mem_stat/mod.rs b/src/mem_stat/mod.rs index 8b45b3db..f5b22646 100644 --- a/src/mem_stat/mod.rs +++ b/src/mem_stat/mod.rs @@ -1,9 +1,11 @@ +use alloc::{boxed::Box, string::String, vec::Vec}; mod primitives; -use std::collections::HashMap; -use std::fmt::Debug; -use std::rc::Rc; -use std::sync::Arc; +use alloc::collections::{BTreeMap, BTreeSet}; +use alloc::rc::Rc; +use alloc::sync::Arc; +use core::fmt::Debug; +use hashbrown::HashMap; use data_bucket::Link; use data_bucket::page::PageId; @@ -14,16 +16,22 @@ use ordered_float::OrderedFloat; use psc_nanoid::PackedNanoid; use psc_nanoid::packed::AlphabetPackExt; use uuid::Uuid; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; +#[cfg(feature = "vanilla-index")] +use crate::UpstreamIndexMap; use crate::in_memory::{RowWrapper, StorableRow}; use crate::persistence::OperationType; use crate::prelude::OperationId; use crate::util::OffsetEqLink; +#[allow(unused_imports)] +use crate::{}; use crate::{ ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticValue, CongeeIndex, CongeeKey, IndexMultiMap, PersistentArtIndex, - PersistentWtiIndex, UniqueIndex, UpstreamIndexMap, WorkTable, + PersistentWtiIndex, UniqueIndex, WorkTable, }; use crate::{IndexMap, impl_memstat_zero}; @@ -55,7 +63,7 @@ impl< PkMap, > where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, PkMap: UniqueIndex> + MemStat, @@ -70,6 +78,33 @@ where } } +macro_rules! impl_tuple_mem_stat { + ($($name:ident),+) => { + impl<$($name: MemStat),+> MemStat for ($($name,)+) { + fn heap_size(&self) -> usize { + #[allow(non_snake_case)] + let ($($name,)+) = self; + 0usize $(+ $name.heap_size())+ + } + + fn used_size(&self) -> usize { + #[allow(non_snake_case)] + let ($($name,)+) = self; + 0usize $(+ $name.used_size())+ + } + } + }; +} + +impl_tuple_mem_stat!(A); +impl_tuple_mem_stat!(A, B); +impl_tuple_mem_stat!(A, B, C); +impl_tuple_mem_stat!(A, B, C, D); +impl_tuple_mem_stat!(A, B, C, D, E); +impl_tuple_mem_stat!(A, B, C, D, E, F); +impl_tuple_mem_stat!(A, B, C, D, E, F, G); +impl_tuple_mem_stat!(A, B, C, D, E, F, G, H); + impl MemStat for Option { fn heap_size(&self) -> usize { self.as_ref().map_or(0, |v| v.heap_size()) @@ -81,10 +116,20 @@ impl MemStat for Option { impl MemStat for Vec { fn heap_size(&self) -> usize { - self.capacity() * std::mem::size_of::() + self.iter().map(|v| v.heap_size()).sum::() + self.capacity() * core::mem::size_of::() + self.iter().map(|v| v.heap_size()).sum::() } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::() + self.iter().map(|v| v.used_size()).sum::() + self.len() * core::mem::size_of::() + self.iter().map(|v| v.used_size()).sum::() + } +} + +impl MemStat for [T; N] { + fn heap_size(&self) -> usize { + self.iter().map(MemStat::heap_size).sum() + } + + fn used_size(&self) -> usize { + self.iter().map(MemStat::used_size).sum() } } @@ -104,7 +149,7 @@ where Node: NodeLike> + Send + 'static, { fn heap_size(&self) -> usize { - let slot_size = std::mem::size_of::>(); + let slot_size = core::mem::size_of::>(); let base_heap = self.capacity() * slot_size; let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); @@ -113,7 +158,7 @@ where } fn used_size(&self) -> usize { - let pair_size = std::mem::size_of::>(); + let pair_size = core::mem::size_of::>(); let base = self.len() * pair_size; let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); @@ -122,6 +167,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl MemStat for UpstreamIndexMap where K: Debug + Ord + Clone + 'static + MemStat + Send, @@ -129,14 +175,14 @@ where Node: VanillaNodeLike> + Send + 'static, { fn heap_size(&self) -> usize { - let slot_size = std::mem::size_of::>(); + let slot_size = core::mem::size_of::>(); let base_heap = self.capacity() * slot_size; let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); base_heap + kv_heap } fn used_size(&self) -> usize { - let pair_size = std::mem::size_of::>(); + let pair_size = core::mem::size_of::>(); let base = self.len() * pair_size; let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); base + used @@ -151,7 +197,7 @@ where fn heap_size(&self) -> usize { let values = self.iter_values().map(|(_, value)| value.heap_size()).sum::(); self.allocated_node_bytes() - + self.len() * (std::mem::size_of::() + 2 * std::mem::size_of::()) + + self.len() * (core::mem::size_of::() + 2 * core::mem::size_of::()) + values } @@ -170,7 +216,7 @@ where } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + self.len() * core::mem::size_of::<(K, V)>() } } @@ -184,7 +230,7 @@ where } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + self.len() * core::mem::size_of::<(K, V)>() } } @@ -220,7 +266,7 @@ where Node: NodeLike> + Send + 'static, { fn heap_size(&self) -> usize { - let slot_size = std::mem::size_of::>(); + let slot_size = core::mem::size_of::>(); let base_heap = self.capacity() * slot_size; let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); @@ -229,7 +275,7 @@ where } fn used_size(&self) -> usize { - let pair_size = std::mem::size_of::>(); + let pair_size = core::mem::size_of::>(); let base = self.len() * pair_size; let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); @@ -240,32 +286,36 @@ where impl MemStat for Box { fn heap_size(&self) -> usize { - std::mem::size_of::() + (**self).heap_size() + core::mem::size_of::() + (**self).heap_size() } fn used_size(&self) -> usize { - std::mem::size_of::() + (**self).used_size() + core::mem::size_of::() + (**self).used_size() } } impl MemStat for Arc { fn heap_size(&self) -> usize { - std::mem::size_of::() + (**self).heap_size() + core::mem::size_of::() + (**self).heap_size() } fn used_size(&self) -> usize { - std::mem::size_of::() + (**self).used_size() + core::mem::size_of::() + (**self).used_size() } } impl MemStat for Rc { fn heap_size(&self) -> usize { - std::mem::size_of::() + (**self).heap_size() + core::mem::size_of::() + (**self).heap_size() } fn used_size(&self) -> usize { - std::mem::size_of::() + (**self).used_size() + core::mem::size_of::() + (**self).used_size() } } -impl MemStat for HashMap { +// Generic over the hasher: `using fxhash` stores an `FxHashMap`, which is this +// type with `FxBuildHasher` rather than the default. Counting capacity rather +// than length is right for a hash map and is what makes a reserved index show +// its reservation. +impl MemStat for HashMap { fn heap_size(&self) -> usize { let bucket_size = size_of::<(K, V)>(); let base_heap = self.capacity() * bucket_size; @@ -284,6 +334,40 @@ impl MemStat for HashMap { } } +impl MemStat for BTreeMap { + fn heap_size(&self) -> usize { + self.len() * core::mem::size_of::<(K, V)>() + + self + .iter() + .map(|(key, value)| key.heap_size() + value.heap_size()) + .sum::() + } + + fn used_size(&self) -> usize { + self.heap_size() + } +} + +impl MemStat for BTreeSet { + fn heap_size(&self) -> usize { + self.len() * core::mem::size_of::() + self.iter().map(MemStat::heap_size).sum::() + } + + fn used_size(&self) -> usize { + self.heap_size() + } +} + +impl MemStat for parking_lot::RwLock { + fn heap_size(&self) -> usize { + self.read().heap_size() + } + + fn used_size(&self) -> usize { + self.read().used_size() + } +} + impl MemStat for OrderedFloat where T: MemStat, diff --git a/src/mem_stat/primitives.rs b/src/mem_stat/primitives.rs index ee159f96..7802e103 100644 --- a/src/mem_stat/primitives.rs +++ b/src/mem_stat/primitives.rs @@ -13,6 +13,7 @@ macro_rules! impl_memstat_zero { } impl_memstat_zero!( + (), u8, i8, u16, @@ -29,23 +30,24 @@ impl_memstat_zero!( char, u128, i128, - std::num::NonZeroU8, - std::num::NonZeroU16, - std::num::NonZeroU32, - std::num::NonZeroU64, - std::num::NonZeroU128, - std::num::NonZeroUsize, - std::num::NonZeroI8, - std::num::NonZeroI16, - std::num::NonZeroI32, - std::num::NonZeroI64, - std::num::NonZeroI128, - std::num::NonZeroIsize, - std::time::Duration, - std::time::SystemTime, - std::time::Instant + core::num::NonZeroU8, + core::num::NonZeroU16, + core::num::NonZeroU32, + core::num::NonZeroU64, + core::num::NonZeroU128, + core::num::NonZeroUsize, + core::num::NonZeroI8, + core::num::NonZeroI16, + core::num::NonZeroI32, + core::num::NonZeroI64, + core::num::NonZeroI128, + core::num::NonZeroIsize, + core::time::Duration ); +#[cfg(feature = "std")] +impl_memstat_zero!(std::time::SystemTime, std::time::Instant); + impl_memstat_zero!( [u8], [i8], diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 5074ed81..ca4e34dc 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use rkyv::api::high::HighDeserializer; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -5,7 +6,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize}; -use tokio::fs::File; use crate::prelude::{GeneralPage, Persistable, SpaceInfoPage, WT_DATA_EXTENSION, parse_page}; @@ -24,7 +24,7 @@ where SpaceInfoPage: Persistable, { let data_file_path = format!("{}/{}", table_path, WT_DATA_EXTENSION); - let mut file = File::open(&data_file_path).await?; - let info: GeneralPage> = parse_page::<_, 4096>(&mut file, 0).await?; + let mut file = crate::fsx::open(&data_file_path).await?; + let info: GeneralPage> = parse_page::<_, 4096, DEFAULT_PAGE_STRIDE>(&mut file, 0).await?; Ok(info.inner.version) } diff --git a/src/partition/dense.rs b/src/partition/dense.rs new file mode 100644 index 00000000..d52c7b0f --- /dev/null +++ b/src/partition/dense.rs @@ -0,0 +1,733 @@ +//! The payload behind a narrow `partition_max_size`. +//! +//! A partition declared `partition_max_size: u8` holds at most 256 rows. At +//! that size the apparatus a full generated table carries is the entire cost: +//! an empty partition of the 832-byte-row shape web3.trading runs measures +//! 28,395 bytes, and the same partition holding three rows measures 28,459. +//! The rows are free. Everything else is fixed overhead allocated at partition +//! creation, and it is paid once per partition, so two thousand symbols pay it +//! two thousand times. +//! +//! # What a dense partition drops, and why each is safe to drop here +//! +//! - **The primary index.** Position *is* the key. A dense unsigned key in +//! `0..cap` indexes the row vector directly, so the lookup is a bounds check +//! and a load rather than a tree descent. This is also the largest saving: +//! arctic holds about 600 bytes per 24-byte row at 64 rows and does not +//! settle until a thousand, so at 23 rows the index is most of the table. +//! - **Pages, links, the free list and the epoch domain.** Rows do not move, +//! because a row's position is its key and never changes. +//! - **The lock map.** A dense key means a lock map would be an array, and an +//! array of locks over 23 rows is not worth the indirection. See the +//! granularity note below. +//! - **CDC.** Nothing is persisted, so there is nothing to replay. +//! +//! # Granularity, stated rather than implied +//! +//! Writes serialise **per partition**, not per cell. The full table gives +//! cell-level serialisation through `LockMap` because its writes are async and +//! a query can hold a column across an await; nothing here is async and no +//! write spans a suspension point, so the lock is held for the duration of one +//! `insert`, `update` or `delete` and released. +//! +//! That is a coarser lock over a much smaller thing. A partition is the unit +//! of contention, and there are thousands of them: at 2,000 symbols and 10,000 +//! writes a second, two writers collide only when they touch the same symbol. +//! Readers never block each other and never block on a writer they do not +//! share a partition with. +//! +//! # Memory +//! +//! The row vector grows to the highest key inserted, not to the declared cap. +//! An empty partition is one lock, one counter and an empty `Vec`: no +//! allocation at all until the first insert. A partition holding keys 0..23 of +//! an 832-byte row holds 23 slots, which is what a hand-written +//! `HashMap>>` holds and 2.5x less than the full table. +//! +//! The declared cap is therefore a bound and not a reservation. It exists to +//! reject a key that does not belong in this partition, and to pick this shape +//! over the full table in the first place. + +use alloc::vec::Vec; +use core::fmt; + +#[cfg(not(wt_loom))] +use core::sync::atomic::{AtomicUsize, Ordering}; +#[cfg(wt_loom)] +use loom::sync::RwLock; +#[cfg(wt_loom)] +use loom::sync::atomic::{AtomicUsize, Ordering}; +#[cfg(not(wt_loom))] +use parking_lot::RwLock; + +use crate::mem_stat::MemStat; + +/// Why a write to a dense partition was refused. +/// +/// Both variants are programming errors rather than conditions to retry, and +/// both name the key, because a caller that hits one is looking at a key it +/// computed wrongly. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DenseError { + /// The key is at or past the declared `partition_max_size`. + /// + /// A partition declared `u8` holds keys `0..256`. This is the check that + /// makes the declared width mean something at run time rather than only + /// selecting a shape at compile time. + OutOfRange { + /// The key that was offered. + /// + /// `u64` rather than `usize` so a key that does not fit a `usize` at + /// all, which is a 32-bit target holding a `u64` key, can still be + /// reported as the number the caller wrote. + key: u64, + /// The declared cap, exclusive. + cap: usize, + }, + /// A row already occupies that key. + /// + /// `insert` refuses rather than overwriting, the same way the full table's + /// does. `upsert` is the one that replaces. + Duplicate { + /// The occupied key. + key: usize, + }, +} + +impl DenseError { + /// The refusal a key that does not fit a `usize` earns. + /// + /// Reachable only on a 32-bit target with a key above `u32::MAX`, and a + /// cap is at most 65,536, so such a key is out of range by construction. + #[must_use] + pub fn out_of_range(key: u64, cap: usize) -> Self { + Self::OutOfRange { key, cap } + } +} + +impl fmt::Display for DenseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::OutOfRange { key, cap } => write!( + f, + "key {key} is outside this partition: `partition_max_size` declares {cap} rows, so keys run 0..{cap}" + ), + Self::Duplicate { key } => write!(f, "key {key} already holds a row in this partition"), + } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for DenseError {} + +/// Rows of one dense partition, addressed by position. +/// +/// See the module documentation for what this drops relative to a full +/// generated table and why. Generated code wraps this in a typed facade; the +/// storage lives here so the expansion per partitioned table stays small, for +/// the same reason [`super::PartitionSet`] does. +#[derive(Debug)] +pub struct DenseRows { + /// Indexed by key. `None` is a key in range that holds no row, which is + /// every key below the highest one inserted that nobody has used. + /// + /// One lock rather than one per slot: a per-slot lock over 23 rows costs + /// more in indirection than it saves in contention, and the vector has to + /// be guarded anyway because growing it moves the rows. + rows: RwLock>>, + /// Rows actually present, so `row_count` and `is_empty` do not take the + /// lock. Kept in step with `rows` under the write lock. + live: AtomicUsize, + /// The declared `partition_max_size`, exclusive. Not a capacity: nothing + /// is allocated against it. + cap: usize, +} + +impl DenseRows { + /// The rows, for reading. + /// + /// A shim, because the two `RwLock`s this compiles against do not agree on + /// the signature: `parking_lot`'s `read` hands back the guard, and loom's + /// hands back a `Result` because it models poisoning. Normalising here + /// keeps the eight call sites below free of `cfg`. + #[cfg(not(wt_loom))] + fn read(&self) -> impl core::ops::Deref>> + '_ { + self.rows.read() + } + + #[cfg(wt_loom)] + fn read(&self) -> impl core::ops::Deref>> + '_ { + self.rows.read().expect("nothing panics while holding this lock") + } + + /// The rows, for writing. See [`Self::read`]. + #[cfg(not(wt_loom))] + fn write(&self) -> impl core::ops::DerefMut>> + '_ { + self.rows.write() + } + + #[cfg(wt_loom)] + fn write(&self) -> impl core::ops::DerefMut>> + '_ { + self.rows.write().expect("nothing panics while holding this lock") + } + + /// A partition holding at most `cap` rows, allocating nothing yet. + #[must_use] + pub fn new(cap: usize) -> Self { + Self { + rows: RwLock::new(Vec::new()), + live: AtomicUsize::new(0), + cap, + } + } + + /// The declared cap, exclusive. Keys run `0..cap`. + #[must_use] + pub fn cap(&self) -> usize { + self.cap + } + + /// Rows present. Does not take the lock. + #[must_use] + pub fn row_count(&self) -> usize { + self.live.load(Ordering::Acquire) + } + + /// Whether any row is present. Does not take the lock. + #[must_use] + pub fn is_empty(&self) -> bool { + self.row_count() == 0 + } + + fn in_range(&self, key: usize) -> Result<(), DenseError> { + if key < self.cap { + Ok(()) + } else { + Err(DenseError::OutOfRange { + key: key as u64, + cap: self.cap, + }) + } + } + + /// Grow to hold `key`, filling the gap with absent slots. + /// + /// Called under the write lock. The vector reaches the highest key used + /// and no further, which is why a `u16` partition holding three rows costs + /// three slots rather than 65,536. + fn make_room(rows: &mut Vec>, key: usize) { + if key >= rows.len() { + rows.resize_with(key + 1, || None); + } + } +} + +impl DenseRows { + /// The row at `key`, cloned out. + /// + /// Cloned rather than borrowed because the rows sit behind a lock that + /// cannot outlive this call, which is the same reason the paged table's + /// `select` clones. A key past the end is absent rather than an error: it + /// is a key nobody has written, which is what `None` means. + #[must_use] + pub fn get(&self, key: usize) -> Option { + self.read().get(key)?.clone() + } + + /// Whether `key` holds a row. + #[must_use] + pub fn contains(&self, key: usize) -> bool { + self.read().get(key).is_some_and(Option::is_some) + } + + /// Every row present, ascending by key, with its key. + #[must_use] + pub fn iter(&self) -> Vec<(usize, T)> { + let rows = self.read(); + rows.iter() + .enumerate() + .filter_map(|(key, slot)| slot.clone().map(|row| (key, row))) + .collect() + } +} + +impl DenseRows { + /// Place `row` at `key`, refusing a key that is occupied or out of range. + /// + /// `Err` carries the reason and not the row. The row is recoverable from + /// the caller's own value in the generated facade, which is where the row + /// type is known. + pub fn insert(&self, key: usize, row: T) -> Result<(), DenseError> { + self.in_range(key)?; + let mut rows = self.write(); + Self::make_room(&mut rows, key); + if rows[key].is_some() { + return Err(DenseError::Duplicate { key }); + } + rows[key] = Some(row); + self.live.fetch_add(1, Ordering::Release); + Ok(()) + } + + /// Place `row` at `key`, returning whatever it replaced. + pub fn upsert(&self, key: usize, row: T) -> Result, DenseError> { + self.in_range(key)?; + let mut rows = self.write(); + Self::make_room(&mut rows, key); + let previous = rows[key].replace(row); + if previous.is_none() { + self.live.fetch_add(1, Ordering::Release); + } + Ok(previous) + } + + /// Take the row at `key` out. + /// + /// The slot stays, holding nothing. Nothing shifts, because a position is + /// a key: compacting would renumber every row above it. + pub fn remove(&self, key: usize) -> Option { + let mut rows = self.write(); + let taken = rows.get_mut(key)?.take(); + if taken.is_some() { + self.live.fetch_sub(1, Ordering::Release); + } + taken + } + + /// Run `edit` against the row at `key`, in place. + /// + /// The lock is held across the call, so `edit` must not reach back into + /// this partition. It is the only way to change part of a row without + /// cloning it out and back, which at an 832-byte row is the difference + /// between touching one field and copying the row twice. + pub fn update(&self, key: usize, edit: impl FnOnce(&mut T) -> R) -> Option { + let mut rows = self.write(); + rows.get_mut(key)?.as_mut().map(edit) + } + + /// Slots allocated, present or not. + /// + /// One past the highest key ever inserted, not the declared cap. Exposed + /// because it is the figure that explains this shape's memory, and a test + /// that asserts the cap is not allocated needs to be able to see it. + #[must_use] + pub fn slots(&self) -> usize { + self.read().len() + } +} + +impl Default for DenseRows { + /// A partition with no cap, for a caller that has not declared one. + /// + /// Generated code never reaches this: it always knows the declared width + /// and calls [`DenseRows::new`]. It exists because the router's + /// `partition_or_create` names `Default`, and a facade that wraps this has + /// to be able to derive it. + fn default() -> Self { + Self::new(usize::MAX) + } +} + +impl MemStat for DenseRows { + fn heap_size(&self) -> usize { + let rows = self.read(); + rows.capacity() * core::mem::size_of::>() + rows.iter().map(|slot| slot.heap_size()).sum::() + } + + fn used_size(&self) -> usize { + let rows = self.read(); + rows.len() * core::mem::size_of::>() + rows.iter().map(|slot| slot.used_size()).sum::() + } +} + +#[cfg(all(test, not(wt_loom)))] +mod tests { + use super::*; + + #[test] + fn position_is_the_key() { + let rows = DenseRows::new(256); + rows.insert(7, "seven").expect("fresh"); + rows.insert(0, "zero").expect("fresh"); + + assert_eq!(rows.get(7), Some("seven")); + assert_eq!(rows.get(0), Some("zero")); + // In range, allocated, and holding nothing: not the same as absent. + assert_eq!(rows.get(3), None); + assert_eq!(rows.row_count(), 2); + } + + #[test] + fn the_cap_is_a_bound_and_not_a_reservation() { + // The point of the shape. A `u16` partition declares 65,536 rows and a + // partition holding one row must not allocate 65,536 slots, or the + // whole saving is spent before any row arrives. + let rows: DenseRows = DenseRows::new(65_536); + assert_eq!(rows.slots(), 0, "an empty partition allocates nothing"); + + rows.insert(2, 20).expect("fresh"); + assert_eq!(rows.slots(), 3, "grown to the key used, not to the cap"); + assert_eq!(rows.cap(), 65_536); + } + + #[test] + fn a_key_past_the_cap_is_refused_by_name() { + let rows: DenseRows = DenseRows::new(4); + let error = rows.insert(4, 1).expect_err("4 is not in 0..4"); + assert_eq!(error, DenseError::OutOfRange { key: 4, cap: 4 }); + + rows.insert(3, 1).expect("3 is the last key in range"); + } + + #[test] + fn insert_refuses_a_duplicate_and_upsert_replaces_it() { + let rows = DenseRows::new(8); + rows.insert(1, 10).expect("fresh"); + assert_eq!(rows.insert(1, 99), Err(DenseError::Duplicate { key: 1 })); + assert_eq!(rows.get(1), Some(10), "the refused insert changed nothing"); + + assert_eq!(rows.upsert(1, 99), Ok(Some(10))); + assert_eq!(rows.get(1), Some(99)); + assert_eq!(rows.row_count(), 1, "replacing is not a second row"); + } + + #[test] + fn upsert_into_an_empty_key_counts_a_new_row() { + // The other half of `upsert`. The test above covers replacing, which + // must *not* count; this covers arriving, which must. Dropping the + // increment here survived every other test in this module, which is how + // it was found. + let rows = DenseRows::new(8); + assert_eq!(rows.upsert(4, 40), Ok(None), "nothing was there"); + assert_eq!(rows.row_count(), 1); + + // And again into a key that was emptied rather than never used, which + // is a different path through the slot vector. + rows.remove(4).expect("just upserted"); + assert_eq!(rows.row_count(), 0); + assert_eq!(rows.upsert(4, 41), Ok(None)); + assert_eq!(rows.row_count(), 1); + assert_eq!(rows.get(4), Some(41)); + } + + #[test] + fn removing_leaves_the_positions_of_everything_else_alone() { + // The reason nothing is compacted: a position is a key, so shifting + // rows down would silently renumber them. + let rows = DenseRows::new(8); + for key in 0..4 { + rows.insert(key, key * 10).expect("fresh"); + } + assert_eq!(rows.remove(1), Some(10)); + + assert_eq!(rows.get(0), Some(0)); + assert_eq!(rows.get(1), None); + assert_eq!(rows.get(2), Some(20), "key 2 did not become key 1"); + assert_eq!(rows.get(3), Some(30)); + assert_eq!(rows.row_count(), 3); + assert_eq!(rows.slots(), 4, "the slot stays, holding nothing"); + + // And the freed key takes a new row without complaint. + rows.insert(1, 111).expect("free again"); + assert_eq!(rows.get(1), Some(111)); + } + + #[test] + fn removing_what_was_never_there_is_not_a_row_lost() { + let rows: DenseRows = DenseRows::new(8); + assert_eq!(rows.remove(3), None); + assert_eq!(rows.row_count(), 0, "the counter must not go negative"); + rows.insert(3, 1).expect("fresh"); + assert_eq!(rows.remove(3), Some(1)); + assert_eq!(rows.remove(3), None); + assert_eq!(rows.row_count(), 0); + } + + #[test] + fn update_edits_in_place_and_says_whether_it_found_anything() { + let rows = DenseRows::new(8); + rows.insert(2, 5u64).expect("fresh"); + + assert_eq!(rows.update(2, |row| core::mem::replace(row, 6)), Some(5)); + assert_eq!(rows.get(2), Some(6)); + + assert_eq!(rows.update(1, |row| *row), None, "in range, holding nothing"); + assert_eq!(rows.update(99, |row| *row), None, "past the end"); + } + + #[test] + fn iter_skips_the_holes_and_carries_the_keys() { + let rows = DenseRows::new(16); + rows.insert(5, "five").expect("fresh"); + rows.insert(1, "one").expect("fresh"); + assert_eq!(rows.iter(), alloc::vec![(1, "one"), (5, "five")]); + } + + #[test] + fn writes_through_a_shared_reference_do_not_lose_rows() { + // The property the whole shape rests on: `partition_or_create` hands + // out `Arc`, so every mutation goes through `&self`. + use alloc::sync::Arc; + use std::thread; + + let rows: Arc> = Arc::new(DenseRows::new(256)); + let threads: Vec<_> = (0..8) + .map(|worker| { + let rows = Arc::clone(&rows); + thread::spawn(move || { + for step in 0..32 { + rows.insert(worker * 32 + step, worker) + .expect("each key is written once"); + } + }) + }) + .collect(); + for thread in threads { + thread.join().expect("worker"); + } + + assert_eq!(rows.row_count(), 256); + assert_eq!(rows.slots(), 256); + for key in 0..256 { + assert_eq!(rows.get(key), Some(key / 32), "key {key}"); + } + } + + #[test] + fn concurrent_inserts_of_one_key_produce_exactly_one_winner() { + use alloc::sync::Arc; + use std::sync::atomic::AtomicUsize as StdAtomicUsize; + use std::thread; + + let rows: Arc> = Arc::new(DenseRows::new(4)); + let won = Arc::new(StdAtomicUsize::new(0)); + let threads: Vec<_> = (0..8) + .map(|worker| { + let rows = Arc::clone(&rows); + let won = Arc::clone(&won); + thread::spawn(move || { + if rows.insert(2, worker).is_ok() { + won.fetch_add(1, Ordering::Relaxed); + } + }) + }) + .collect(); + for thread in threads { + thread.join().expect("worker"); + } + + assert_eq!(won.load(Ordering::Relaxed), 1, "exactly one insert may succeed"); + assert_eq!(rows.row_count(), 1); + } +} + +#[cfg(all(test, wt_loom))] +mod loom_tests { + //! Loom models of the dense partition's counter and lock protocol. + //! + //! Run with: + //! + //! ```text + //! RUSTFLAGS="--cfg wt_loom" cargo test --release --lib partition::dense::loom_tests + //! ``` + //! + //! **Manual only. Never wired into CI**, by instruction, the same as the + //! models in [`super::super::loom_tests`] and the Miri runs. + //! + //! # What is under test, and what is not + //! + //! The rows sit behind one `RwLock`, and loom models that lock, so mutual + //! exclusion over the vector is not the interesting question: loom would be + //! checking its own primitive. + //! + //! What is interesting is `live`, the row counter, because it is written + //! under the lock and read **without** it. Three things could go wrong and + //! each has a model here: it could underflow when a remove races an insert, + //! it could settle on the wrong value when several writers finish at once, + //! and it could be read as a number that no interleaving ever produced. + //! + //! # These models are narrow, and that is on purpose + //! + //! The rows sit behind one lock, so loom serialises nearly everything and + //! the state space is tiny: all five run in about ten milliseconds. That is + //! a fair reflection of how little unsynchronised state this type has, not + //! a sign the models are cheap to the point of being useless. Each was + //! checked by breaking the thing it claims to check and confirming it + //! fails: an unguarded `fetch_sub` in `remove` fails two of them, and an + //! increment moved above the duplicate check in `insert` fails a third. + //! + //! That check found a real defect in the first version of these models, + //! which is recorded on + //! [`racing_removes_on_an_empty_slot_never_underflow_the_count`]. + //! + //! The rows themselves are `u64` here rather than a `loom::cell::UnsafeCell` + //! payload. That is deliberate and it is a limitation: loom cannot see + //! inside a plain value, so these models say nothing about publication of + //! the row's contents. They do not need to. Every read of a row goes through + //! the same `RwLock` as every write, so publication is the lock's guarantee + //! and not an `Ordering` this module chose. The partition set's models need + //! `Guarded` because its readers deliberately run outside its mutex; this + //! one has no such path. + + use super::*; + use loom::sync::Arc; + use loom::thread; + + /// Two threads inserting the same key: one wins, and the count agrees. + /// + /// The count is the point. `insert` increments only on the branch that + /// actually stored a row, so a version that incremented before checking for + /// an occupant would leave `row_count` at 2 with one row present. + #[test] + fn one_of_two_racing_inserts_wins_and_the_count_agrees() { + loom::model(|| { + let rows: Arc> = Arc::new(DenseRows::new(4)); + + let a = { + let rows = Arc::clone(&rows); + thread::spawn(move || rows.insert(1, 10).is_ok()) + }; + let b = { + let rows = Arc::clone(&rows); + thread::spawn(move || rows.insert(1, 20).is_ok()) + }; + + let won = usize::from(a.join().unwrap()) + usize::from(b.join().unwrap()); + assert_eq!(won, 1, "exactly one insert may store a row"); + assert_eq!(rows.row_count(), 1); + assert!(matches!(rows.get(1), Some(10) | Some(20))); + }); + } + + /// Two removes racing over one empty-but-allocated slot must not take the + /// counter below zero. + /// + /// `fetch_sub` on a `usize` wraps, so an unguarded decrement does not panic + /// in release: it reports a partition holding eighteen quintillion rows. + /// The guard is that `remove` decrements only when it actually took + /// something out. + /// + /// The setup matters and the first version of this model got it wrong. It + /// raced a remove against an insert on a *fresh* partition, where the + /// remove finds the row vector still empty, `get_mut` returns `None`, and + /// the method returns before reaching the decrement at all. That model + /// passed against a deliberately unguarded `remove`, which is the only + /// thing a concurrency model must never do. The slot has to exist and hold + /// nothing for the guard to be the thing under test, so it is allocated and + /// emptied first. + #[test] + fn racing_removes_on_an_empty_slot_never_underflow_the_count() { + loom::model(|| { + let rows: Arc> = Arc::new(DenseRows::new(4)); + // Allocate slot 2, then empty it: present in the vector, holding + // nothing, which is the state `remove` has to handle without + // counting a row it did not take. + rows.insert(2, 7).expect("fresh"); + rows.remove(2).expect("just inserted"); + assert_eq!(rows.row_count(), 0); + + let a = { + let rows = Arc::clone(&rows); + thread::spawn(move || rows.remove(2)) + }; + let b = { + let rows = Arc::clone(&rows); + thread::spawn(move || rows.remove(2)) + }; + assert!(a.join().unwrap().is_none()); + assert!(b.join().unwrap().is_none()); + + assert_eq!( + rows.row_count(), + 0, + "removing nothing twice must leave the count at zero, not wrap" + ); + }); + } + + /// And the same guard under a remove racing an insert on an existing slot. + #[test] + fn a_remove_racing_an_insert_counts_the_row_at_most_once() { + loom::model(|| { + let rows: Arc> = Arc::new(DenseRows::new(4)); + // Slot 2 allocated and empty, so neither thread returns early. + rows.insert(2, 7).expect("fresh"); + rows.remove(2).expect("just inserted"); + + let inserter = { + let rows = Arc::clone(&rows); + thread::spawn(move || rows.insert(2, 9).is_ok()) + }; + let remover = { + let rows = Arc::clone(&rows); + thread::spawn(move || rows.remove(2)) + }; + let inserted = inserter.join().unwrap(); + let taken = remover.join().unwrap(); + + let count = rows.row_count(); + assert!(count <= 1, "a one-key partition cannot hold {count} rows"); + assert_eq!( + count, + usize::from(inserted && taken.is_none()), + "the row is present exactly when it was inserted and not taken" + ); + }); + } + + /// Two writers on different keys both land, and the count sees both. + /// + /// This is where a `Relaxed` increment would show: the second writer's + /// `fetch_add` has to be ordered against the first's for the final load to + /// observe two. + #[test] + fn concurrent_writers_on_distinct_keys_both_count() { + loom::model(|| { + let rows: Arc> = Arc::new(DenseRows::new(4)); + + let a = { + let rows = Arc::clone(&rows); + thread::spawn(move || rows.insert(0, 1).expect("key 0 is written once")) + }; + let b = { + let rows = Arc::clone(&rows); + thread::spawn(move || rows.insert(1, 2).expect("key 1 is written once")) + }; + a.join().unwrap(); + b.join().unwrap(); + + assert_eq!(rows.row_count(), 2); + assert_eq!(rows.get(0), Some(1)); + assert_eq!(rows.get(1), Some(2)); + }); + } + + /// A reader running beside a writer sees a count that some interleaving + /// produced, never a torn one. + /// + /// `row_count` deliberately does not take the lock, so it may be stale. + /// Stale is fine and is documented; a value that was never true is not. + #[test] + fn an_unlocked_count_is_always_a_value_some_interleaving_produced() { + loom::model(|| { + let rows: Arc> = Arc::new(DenseRows::new(4)); + rows.insert(0, 1).expect("fresh"); + + let writer = { + let rows = Arc::clone(&rows); + thread::spawn(move || { + let _ = rows.insert(1, 2); + }) + }; + let reader = { + let rows = Arc::clone(&rows); + thread::spawn(move || rows.row_count()) + }; + + writer.join().unwrap(); + let seen = reader.join().unwrap(); + assert!(seen == 1 || seen == 2, "count {seen} matches no interleaving"); + assert_eq!(rows.row_count(), 2, "and it settles once the writer is done"); + }); + } +} diff --git a/src/partition/mod.rs b/src/partition/mod.rs index fdc2779b..d91535fc 100644 --- a/src/partition/mod.rs +++ b/src/partition/mod.rs @@ -55,22 +55,27 @@ //! instance measures 110 KB and 6.1 ms to construct, of which 95 percent is //! inside `PersistenceEngine::new`. +mod dense; + +pub use dense::{DenseError, DenseRows}; + +use alloc::{boxed::Box, vec::Vec}; // Under `--cfg wt_loom` the atomics and the mutex come from loom, which explores // every interleaving of them rather than whichever one this machine happened // to produce. `Arc` stays `std`: loom's has no `into_raw` or // `increment_strong_count`, and std's own atomics are already model-checked // upstream. What loom is being asked about here is the slot protocol and the // double-checked lock in `get_or_create`, not reference counting. +use alloc::collections::VecDeque; +use alloc::sync::Arc; +#[cfg(not(wt_loom))] +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; #[cfg(wt_loom)] use loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; #[cfg(wt_loom)] use loom::sync::{Mutex, MutexGuard}; #[cfg(not(wt_loom))] use parking_lot::{Mutex, MutexGuard}; -use std::collections::VecDeque; -use std::sync::Arc; -#[cfg(not(wt_loom))] -use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use crate::mem_stat::MemStat; #[cfg(not(wt_loom))] @@ -109,7 +114,7 @@ struct Chunk { impl Chunk { fn empty() -> Box { Box::new(Chunk { - slots: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + slots: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), }) } } @@ -144,7 +149,7 @@ pub struct PartRef<'a, T> { value: &'a T, } -impl std::ops::Deref for PartRef<'_, T> { +impl core::ops::Deref for PartRef<'_, T> { type Target = T; fn deref(&self) -> &T { @@ -180,7 +185,7 @@ impl Default for PartitionSet { impl PartitionSet { pub fn new() -> Self { Self { - spine: (0..MAX_CHUNKS).map(|_| AtomicPtr::new(std::ptr::null_mut())).collect(), + spine: (0..MAX_CHUNKS).map(|_| AtomicPtr::new(core::ptr::null_mut())).collect(), live: AtomicUsize::new(0), grow: Mutex::new(VecDeque::new()), #[cfg(not(wt_loom))] @@ -293,13 +298,14 @@ impl PartitionSet { /// So a tick loop should not pin per lookup. Pin once, read many: /// /// ``` - /// # fn main() { futures::executor::block_on(async { + /// # fn main() { nagoya::block_on(async { /// use worktable::prelude::*; /// use worktable::worktable; /// /// worktable!( /// name: Price, /// partition_by: symbol_id: u16, + /// partition_max_size: u64, /// columns: { /// exchange_id: u8 primary_key, /// bid: f64 @@ -488,7 +494,7 @@ impl PartitionSet { let table = { let mut retired = self.lock(); let chunk = self.chunk(idx)?; - let p = chunk.slots[idx % CHUNK].swap(std::ptr::null_mut(), Ordering::AcqRel); + let p = chunk.slots[idx % CHUNK].swap(core::ptr::null_mut(), Ordering::AcqRel); if p.is_null() { return None; } @@ -594,13 +600,13 @@ impl PartitionSet { } } -impl std::fmt::Debug for PartitionSet { +impl core::fmt::Debug for PartitionSet { /// Deliberately shallow: a partition set can hold thousands of tables and /// printing them would be useless as well as slow. - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PartitionSet") .field("live", &self.len()) - .field("table", &std::any::type_name::()) + .field("table", &core::any::type_name::()) .finish() } } @@ -608,7 +614,7 @@ impl std::fmt::Debug for PartitionSet { impl Drop for PartitionSet { fn drop(&mut self) { for cell in &self.spine { - let p = cell.swap(std::ptr::null_mut(), Ordering::AcqRel); + let p = cell.swap(core::ptr::null_mut(), Ordering::AcqRel); if p.is_null() { continue; } @@ -616,7 +622,7 @@ impl Drop for PartitionSet { // published exactly once, and nothing else frees it. let chunk = unsafe { Box::from_raw(p) }; for slot in chunk.slots.iter() { - let sp = slot.swap(std::ptr::null_mut(), Ordering::AcqRel); + let sp = slot.swap(core::ptr::null_mut(), Ordering::AcqRel); if !sp.is_null() { // Safety: a live slot owns one strong reference. drop(unsafe { Arc::from_raw(sp as *const T) }); @@ -652,8 +658,8 @@ pub enum PartitionError { OutOfRange { key: u64 }, } -impl std::fmt::Display for PartitionError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for PartitionError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { PartitionError::OutOfRange { key } => { write!(f, "partition key {key} exceeds the maximum of {MAX_PARTITIONS}") @@ -662,7 +668,7 @@ impl std::fmt::Display for PartitionError { } } -impl std::error::Error for PartitionError {} +impl core::error::Error for PartitionError {} #[cfg(all(test, not(wt_loom)))] mod tests; diff --git a/src/partition/tests.rs b/src/partition/tests.rs index a18f9e68..1a920c2d 100644 --- a/src/partition/tests.rs +++ b/src/partition/tests.rs @@ -1,5 +1,6 @@ use super::*; -use std::sync::atomic::AtomicU32; +use alloc::{string::ToString, vec::Vec}; +use core::sync::atomic::AtomicU32; #[derive(Debug, PartialEq)] struct Counted(u64); @@ -119,7 +120,7 @@ fn concurrent_creation_of_one_key_makes_one_table() { #[test] fn concurrent_readers_see_a_partition_created_under_them() { let set: Arc> = Arc::new(PartitionSet::new()); - let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop = Arc::new(core::sync::atomic::AtomicBool::new(false)); let reader = { let set = set.clone(); let stop = stop.clone(); @@ -389,7 +390,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { let drops = Arc::new(AtomicU32::new(0)); let set: Arc> = Arc::new(PartitionSet::new()); - let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop = Arc::new(core::sync::atomic::AtomicBool::new(false)); let seen = Arc::new(AtomicU32::new(0)); let readers: Vec<_> = (0..if cfg!(miri) { 2 } else { 3 }) @@ -434,7 +435,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { for k in 0..KEYS { set.get_or_create(k, || make(k, &drops)).unwrap(); } - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let deadline = std::time::Instant::now() + core::time::Duration::from_secs(30); while seen.load(Ordering::Relaxed) == 0 && std::time::Instant::now() < deadline { std::thread::yield_now(); } @@ -447,7 +448,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { // read to finish, never a zero-reader instant. The pre-epoch retire list // could only assert the opposite here (everything retired, nothing // freed, unbounded growth through the shared router). - let reclaim_deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let reclaim_deadline = std::time::Instant::now() + core::time::Duration::from_secs(30); while drops.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < reclaim_deadline { set.collect(); std::thread::yield_now(); @@ -473,7 +474,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { // Drain the remainder through the shared handle: no `&mut`, no // `Arc::try_unwrap` gymnastics needed for reclamation any more. - let drain_deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let drain_deadline = std::time::Instant::now() + core::time::Duration::from_secs(30); while set.retired_len() > 0 && std::time::Instant::now() < drain_deadline { set.collect(); } @@ -669,7 +670,7 @@ fn mem_stat_reports_each_partition_and_their_sum() { // `size_of::()` on top of the payload, because that is what the // allocation actually holds. Derived rather than hard coded so the // expectation is the rule, not one machine's numbers. - let overhead = std::mem::size_of::(); + let overhead = core::mem::size_of::(); for (k, size) in [(3u64, 17usize), (1, 5), (2048, 300)] { set.get_or_create(k, || Sized_(size)).unwrap(); } @@ -698,6 +699,6 @@ fn partition_error_says_which_key_and_which_bound() { ); // The `Error` impl is what a caller using `?` and `eyre` will format. - let as_error: &dyn std::error::Error = &out_of_range; + let as_error: &dyn core::error::Error = &out_of_range; assert_eq!(as_error.to_string(), text); } diff --git a/src/persistence/engine.rs b/src/persistence/engine.rs index 37254f6f..0c85ee66 100644 --- a/src/persistence/engine.rs +++ b/src/persistence/engine.rs @@ -1,8 +1,9 @@ -use std::fmt::Debug; +use alloc::{string::String, vec::Vec}; +use core::fmt::Debug; +use core::future::Future; +use core::hash::Hash; +use core::marker::PhantomData; use std::fs; -use std::future::Future; -use std::hash::Hash; -use std::marker::PhantomData; use std::panic::{AssertUnwindSafe, resume_unwind}; use std::path::Path; @@ -182,9 +183,15 @@ where &mut self, op: Operation, ) -> eyre::Result<()> { + let mut row_mutations = crate::persistence::space::BatchData::new(); + for (link, bytes) in op.row_mutations() { + row_mutations.entry(link.page_id).or_default().push((link, bytes)); + } + if !row_mutations.is_empty() { + self.data.save_batch_data(row_mutations).await?; + } match op { Operation::Insert(insert) => { - self.data.save_data(insert.link, insert.bytes.as_ref()).await?; for event in insert.primary_key_events { self.primary_index.process_change_event(event).await?; } @@ -196,7 +203,6 @@ where .await } Operation::Update(update) => { - self.data.save_data(update.link, update.bytes.as_ref()).await?; for event in update.primary_key_events { self.primary_index.process_change_event(event).await?; } diff --git a/src/persistence/error.rs b/src/persistence/error.rs index fbfac49e..1acb39a2 100644 --- a/src/persistence/error.rs +++ b/src/persistence/error.rs @@ -1,9 +1,10 @@ -use std::error::Error; -use std::fmt::{Display, Formatter}; -use std::future::Future; +use alloc::sync::Arc; +use alloc::{string::String, string::ToString}; +use core::error::Error; +use core::fmt::{Display, Formatter}; +use core::future::Future; use std::panic::AssertUnwindSafe; use std::path::{Path, PathBuf}; -use std::sync::Arc; use futures::FutureExt; @@ -39,7 +40,7 @@ impl PersistenceLoadError { } impl Display for PersistenceLoadError { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { write!( formatter, "torn or corrupt persisted table at {}: {}", @@ -111,7 +112,7 @@ impl PersistenceIndexCorruption { } impl Display for PersistenceIndexCorruption { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { write!( formatter, "persisted index at {} was quarantined: {}", @@ -137,7 +138,7 @@ pub enum PersistenceError { } impl Display for PersistenceError { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { match self { Self::Closing => formatter.write_str("persistence task is closing"), Self::Closed => formatter.write_str("persistence task is closed"), diff --git a/src/persistence/event_ledger.rs b/src/persistence/event_ledger.rs new file mode 100644 index 00000000..00ef9f4d --- /dev/null +++ b/src/persistence/event_ledger.rs @@ -0,0 +1,774 @@ +//! Pairs index change event id *assignment* with event *queueing*, so that a +//! persistence stall on an event gap names its own cause instead of only its +//! symptom. +//! +//! # The defect this exists for +//! +//! `BatchOperation::validate` refuses to apply an event stream with a hole in +//! it (see the guard there and commit `c0c06ba`). A hole is normally +//! transient: the operation carrying the missing id has been produced but not +//! yet batched. A hole that survives the whole deferral budget means something +//! else, and the old message could not tell the two apart. It reported the +//! range and nothing more: +//! +//! ```text +//! persistence stalled on primary index event gap: last applied Id(1439), +//! next available Id(1455) (attempt 9) +//! ``` +//! +//! Two very different bugs produce that line: +//! +//! 1. **A leak.** An id was assigned by the index and its event was then +//! dropped instead of being pushed onto the persistence queue. Nothing will +//! ever deliver it and the stream is permanently gapped. +//! 2. **A collection failure.** The operation carrying the id *was* queued and +//! is still sitting in the analyzer, but batch collection keeps assembling +//! batches that exclude it. +//! +//! Distinguishing those needs a record of what was queued, which is what this +//! ledger keeps. It is deliberately not a fix for either: it only observes. +//! +//! # Where the two sides are observed +//! +//! Assignment happens inside the index (`indexset` bumps an `AtomicU64` and +//! stamps the event in the same commit), which this crate cannot hook. What it +//! *can* hook is the other end: every event that reaches persistence passes +//! through [`crate::persistence::task::Queue`], so an id present in the stream +//! but absent from this ledger was assigned and never queued. That is the leak +//! signature, and it is what [`EventLedger::gap_report`] reports. +//! +//! The producer is named by [`std::panic::Location`], captured with +//! `#[track_caller]` at the queue push, so a leak points at the call site that +//! produced its neighbours: the generated query, `insert_many`, an +//! acknowledge path, or vacuum's `apply_move`. A full backtrace is captured +//! too, but only when `RUST_BACKTRACE` is set; `Backtrace::capture` is +//! essentially free otherwise. +//! +//! # Gating: `debug_assertions`, not a cargo feature +//! +//! Recording is compiled in unconditionally and gated at run time by +//! [`enabled`], which is true when `debug_assertions` is on or when +//! `WT_EVENT_LEDGER` is set in the environment. +//! +//! `debug_assertions` was chosen over a new cargo feature for one reason: the +//! stall is only ever observed under a full `cargo test --workspace +//! --all-targets --all-features` run, and that run is a debug build, so the +//! instrumentation is on exactly where the bug appears. A feature would also +//! have been enabled by `--all-features`, but it would have to be declared in +//! `Cargo.toml`, and a diagnostic that lives behind a flag nobody sets in the +//! failing configuration is worthless. Release builds fold `enabled()` down +//! to the environment check and every recording call returns immediately, so +//! they pay a predictable-branch and nothing else. +//! +//! `WT_EVENT_LEDGER=1` exists so a release build can be told to record without +//! being rebuilt, for the day the stall shows up outside a test run. +//! +//! Memory is bounded by [`WINDOW`] ids per stream. The gap sits at the head of +//! the stream by construction, so a recent window always covers it; the report +//! states the window it holds so a reader can see that for themselves. + +// The ledger itself is arithmetic and bookkeeping, so it takes `core` and +// `alloc`. Two of its capabilities genuinely need an operating system, and only +// those are gated: `Backtrace`, and reading `WT_EVENT_LEDGER` from the +// environment. Without `std` the ledger is simply never enabled, which is the +// right answer for a diagnostic that an environment variable turns on. +use alloc::borrow::ToOwned as _; +// Only the gated backtrace field boxes anything. +#[cfg(feature = "std")] +use alloc::boxed::Box; +use alloc::collections::BTreeMap; +use alloc::string::{String, ToString as _}; +use alloc::vec::Vec; +use core::fmt::Write as _; +use core::panic::Location; +use hashbrown::HashMap; +#[cfg(feature = "std")] +use std::backtrace::Backtrace; +#[cfg(feature = "std")] +use std::sync::LazyLock; + +use data_bucket::Link; +use indexset::cdc::change::ChangeEvent; +use indexset::core::pair::Pair; +use parking_lot::Mutex; + +use crate::persistence::{OperationId, OperationType}; + +/// Ids retained per stream. The gap the guard reports is at the head of the +/// stream, so a window this size covers it many times over: the one observed +/// stall was 16 ids wide. +const WINDOW: usize = 8192; + +/// Gap ids listed individually in a report before it summarises the rest. +const MAX_LISTED_GAP_IDS: usize = 64; + +#[cfg(feature = "std")] +static ENABLED: LazyLock = LazyLock::new(|| { + if cfg!(debug_assertions) { + return true; + } + match std::env::var("WT_EVENT_LEDGER") { + Ok(value) => !value.is_empty() && value != "0" && !value.eq_ignore_ascii_case("false"), + Err(_) => false, + } +}); + +/// Without `std` there is no environment to read the switch from, so the +/// ledger stays off. Everything below still compiles and can be driven +/// directly by a caller that wants it; what is missing is the ambient way to +/// turn it on. +#[cfg(not(feature = "std"))] +static ENABLED: bool = false; + +/// Whether event bookkeeping is recording in this process. +/// +/// See the module comment for why this is `debug_assertions` plus an +/// environment override rather than a cargo feature. +#[inline] +pub fn enabled() -> bool { + #[cfg(feature = "std")] + { + *ENABLED + } + #[cfg(not(feature = "std"))] + { + ENABLED + } +} + +/// Which index's event id sequence a record belongs to. +/// +/// Every index keeps its own counter, so ids only mean anything relative to a +/// stream. `Primary` allocates nothing, which keeps the hot path free of +/// allocation; secondary streams are labelled by the `Debug` rendering of the +/// table's `AvailableIndexes` value, which is the only name available to +/// non-generic code here. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum EventStream { + Primary, + Secondary(String), +} + +impl core::fmt::Display for EventStream { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + EventStream::Primary => f.write_str("primary"), + EventStream::Secondary(index) => write!(f, "secondary {index}"), + } + } +} + +/// What has been observed happening to one event id. +/// +/// Flags, not a state machine: an id is queued, then collected into a batch, +/// then possibly trimmed back out and requeued, possibly several times over. +/// The report reads the whole set. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Stages(u8); + +impl Stages { + /// Pushed onto the persistence queue by a producer. + pub const QUEUED: Stages = Stages(1 << 0); + /// Pulled into a `BatchOperation` by the analyzer. + pub const COLLECTED: Stages = Stages(1 << 1); + /// Handed back to the analyzer's queue after a deferral or a trim. + pub const REQUEUED: Stages = Stages(1 << 2); + /// Removed from a batch by `remove_operations_from_events`. + pub const TRIMMED: Stages = Stages(1 << 3); + /// Covered by the applied watermark reported after a batch. + pub const APPLIED: Stages = Stages(1 << 4); + + fn insert(&mut self, other: Stages) { + self.0 |= other.0; + } + + /// Both flag sets at once. + pub const fn union(self, other: Stages) -> Stages { + Stages(self.0 | other.0) + } + + fn contains(self, other: Stages) -> bool { + self.0 & other.0 == other.0 + } +} + +impl core::fmt::Display for Stages { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut first = true; + for (flag, name) in [ + (Stages::QUEUED, "queued"), + (Stages::COLLECTED, "collected"), + (Stages::REQUEUED, "requeued"), + (Stages::TRIMMED, "trimmed"), + (Stages::APPLIED, "applied"), + ] { + if self.contains(flag) { + if !first { + f.write_str("+")?; + } + f.write_str(name)?; + first = false; + } + } + if first { + f.write_str("none")?; + } + Ok(()) + } +} + +#[derive(Debug)] +struct IdRecord { + stages: Stages, + /// The operation that carried this id when it was first queued. + op_id: Option, + op_type: Option, + /// Producer call site, from `#[track_caller]` at the queue push. + site: Option<&'static Location<'static>>, + /// Captured only when `RUST_BACKTRACE` is set; otherwise `Disabled` and + /// free. Boxed so that the common empty case costs a pointer instead of an + /// inline `Backtrace` in each of the thousands of records held per stream. + // Gated with the capture below: without `std` there is no `Backtrace`, + // and the ledger keeps the call site from `#[track_caller]` regardless, + // which is the half that names the producer. + #[cfg(feature = "std")] + backtrace: Option>, + collected: u32, + requeued: u32, +} + +impl IdRecord { + fn new() -> Self { + Self { + stages: Stages::default(), + op_id: None, + op_type: None, + site: None, + #[cfg(feature = "std")] + backtrace: None, + collected: 0, + requeued: 0, + } + } +} + +#[derive(Debug, Default)] +struct StreamLedger { + ids: BTreeMap, + /// Ids below this were evicted by the window and cannot be answered for. + evicted_below: u64, + /// Highest id ever seen at any stage on this stream. + highest_seen: u64, + /// Highest id the analyzer reported as applied. + applied_upto: u64, + queued_total: u64, +} + +impl StreamLedger { + fn entry(&mut self, id: u64) -> &mut IdRecord { + if id > self.highest_seen { + self.highest_seen = id; + } + self.ids.entry(id).or_insert_with(IdRecord::new) + } + + fn trim(&mut self) { + while self.ids.len() > WINDOW { + // Ids are close to monotonic, so the lowest key is the oldest. + if let Some((id, _)) = self.ids.pop_first() { + self.evicted_below = self.evicted_below.max(id + 1); + } else { + break; + } + } + } + + /// Lowest id this ledger can still answer for. + fn window_start(&self) -> u64 { + self.ids.keys().next().copied().unwrap_or(self.evicted_below) + } +} + +/// Per-table record of which index change event ids reached the persistence +/// queue, and what happened to them afterwards. +/// +/// Shared by `Arc` between the queue (the producer side), the analyzer, and +/// the `BatchOperation` whose guard reads it. +#[derive(Debug)] +pub struct EventLedger { + label: String, + /// True when no producer writes to this ledger, so "never queued" here + /// means "never recorded", not "leaked". Reports say so rather than + /// accusing a producer that was never watched. + detached: bool, + streams: Mutex>, +} + +impl EventLedger { + pub fn new(label: impl Into) -> Self { + Self { + label: label.into(), + detached: false, + streams: Mutex::new(HashMap::new()), + } + } + + /// A ledger attached to nothing, for analyzers and `BatchOperation`s built + /// outside `run_engine` (unit tests, defensive callers). It records + /// normally; it is simply never shared with a producer, so its reports say + /// so instead of reading a missing record as a leak. + pub fn detached() -> Self { + Self { + label: "".to_owned(), + detached: true, + streams: Mutex::new(HashMap::new()), + } + } + + pub fn label(&self) -> &str { + &self.label + } + + /// Collects the event ids of `evs`, for a later [`EventLedger::record_queued`]. + /// + /// Split from the recording itself because the queue only knows a push was + /// accepted after the operation has been moved into it, and a refused push + /// must not be recorded as queued. + pub fn event_ids(evs: &[ChangeEvent>]) -> Vec { + if !enabled() { + return Vec::new(); + } + evs.iter().map(|ev| ev.id().inner()).collect() + } + + /// Records every id in `ids` as queued by `site`. + /// + /// Called from the persistence queue push, which is the single point every + /// operation passes through on its way to the engine. An id in the applied + /// stream that never appears here was assigned by the index and dropped + /// before it reached persistence. + pub fn record_queued( + &self, + stream: EventStream, + ids: &[u64], + op_id: OperationId, + op_type: OperationType, + site: &'static Location<'static>, + ) { + if !enabled() || ids.is_empty() { + return; + } + // Costs nothing unless RUST_BACKTRACE is set: `capture` returns + // `Disabled` without walking any frames. One capture per push, moved + // onto the first newly recorded id, because every id in this vector + // came from the same producer. + #[cfg(feature = "std")] + let backtrace = Backtrace::capture(); + #[cfg(feature = "std")] + let mut backtrace = + matches!(backtrace.status(), std::backtrace::BacktraceStatus::Captured).then_some(backtrace); + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + for id in ids.iter().copied() { + let record = ledger.entry(id); + let first_time = !record.stages.contains(Stages::QUEUED); + record.stages.insert(Stages::QUEUED); + if first_time { + record.op_id = Some(op_id); + record.op_type = Some(op_type); + record.site = Some(site); + // One backtrace per push, not per id: every id in this vector + // has the same producer, and keeping one each would multiply + // the cost of a `RUST_BACKTRACE` run for no extra signal. + #[cfg(feature = "std")] + if record.backtrace.is_none() { + record.backtrace = backtrace.take().map(Box::new); + } + } else { + record.stages.insert(Stages::REQUEUED); + record.requeued = record.requeued.saturating_add(1); + } + } + ledger.queued_total = ledger.queued_total.saturating_add(ids.len() as u64); + ledger.trim(); + } + + /// Records a stage transition for a single id already known to a stream. + pub fn record_stage(&self, stream: EventStream, id: u64, stage: Stages) { + if !enabled() { + return; + } + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + let record = ledger.entry(id); + record.stages.insert(stage); + if stage.contains(Stages::COLLECTED) { + record.collected = record.collected.saturating_add(1); + } + if stage.contains(Stages::REQUEUED) { + record.requeued = record.requeued.saturating_add(1); + } + ledger.trim(); + } + + /// Records a stage transition for every id in `evs`. + pub fn record_stage_for_events(&self, stream: EventStream, evs: &[ChangeEvent>], stage: Stages) { + if !enabled() || evs.is_empty() { + return; + } + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + for ev in evs { + let record = ledger.entry(ev.id().inner()); + record.stages.insert(stage); + if stage.contains(Stages::COLLECTED) { + record.collected = record.collected.saturating_add(1); + } + if stage.contains(Stages::REQUEUED) { + record.requeued = record.requeued.saturating_add(1); + } + } + ledger.trim(); + } + + /// Records the applied watermark reported after a batch was accepted. + pub fn record_applied_upto(&self, stream: EventStream, id: u64) { + if !enabled() || id == 0 { + return; + } + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + if id > ledger.applied_upto { + ledger.applied_upto = id; + } + if id > ledger.highest_seen { + ledger.highest_seen = id; + } + for (_, record) in ledger.ids.range_mut(..=id) { + record.stages.insert(Stages::APPLIED); + } + } + + /// Explains the gap between `last_applied` and `next_available`. + /// + /// This is the whole point of the ledger. For every id in the hole it says + /// whether the id ever reached the persistence queue, and if it did, what + /// happened to it afterwards, so the reader can tell a leaked event from a + /// batch collection that keeps missing one. + pub fn gap_report(&self, stream: &EventStream, last_applied: u64, next_available: u64) -> String { + let mut out = String::new(); + if !enabled() { + let _ = write!( + out, + " Event bookkeeping is off in this build, so the gap cannot be attributed. \ + Re-run with WT_EVENT_LEDGER=1 (and RUST_BACKTRACE=1 for producer backtraces) \ + to have the next occurrence name its own cause." + ); + return out; + } + + if self.detached { + let _ = write!( + out, + " This analyzer holds detached event bookkeeping: no producer ever wrote to it, \ + so the gap cannot be attributed. Only analyzers built outside `run_engine` are detached." + ); + return out; + } + + let streams = self.streams.lock(); + let Some(ledger) = streams.get(stream) else { + let _ = write!( + out, + " Event bookkeeping for table {label} holds no records at all for the {stream} stream, \ + which should be impossible while it is applying that stream's events.", + label = self.label, + ); + return out; + }; + + let window_start = ledger.window_start(); + let first_missing = last_applied.saturating_add(1); + if next_available <= first_missing { + return out; + } + + // Bounded on purpose: this runs while building a panic message, and a + // corrupt watermark could otherwise make it walk billions of ids. + let scan_end = next_available.min(first_missing.saturating_add(WINDOW as u64)); + let mut never_queued = Vec::new(); + let mut queued_not_applied = Vec::new(); + for id in first_missing..scan_end { + match ledger.ids.get(&id) { + Some(record) if record.stages.contains(Stages::QUEUED) => queued_not_applied.push((id, record)), + _ => never_queued.push(id), + } + } + + let _ = write!( + out, + " Bookkeeping for table {label}, {stream} stream: window covers ids {window_start}..={highest}, \ + applied watermark {applied}, {total} id(s) queued in all, \ + {gap_len} id(s) in the gap ({scanned} scanned), {never} never queued, {queued} queued.", + label = self.label, + window_start = window_start, + highest = ledger.highest_seen, + applied = ledger.applied_upto, + total = ledger.queued_total, + gap_len = next_available - first_missing, + scanned = scan_end - first_missing, + never = never_queued.len(), + queued = queued_not_applied.len(), + ); + + if first_missing < window_start { + let _ = write!( + out, + " CAUTION: part of the gap ({first_missing}..{window_start}) fell out of the retained window, \ + so those ids are unattributable rather than proven missing." + ); + } + + if !never_queued.is_empty() { + let _ = write!(out, " ASSIGNED BUT NEVER QUEUED: {}.", format_ids(&never_queued)); + let _ = write!( + out, + " Those ids were consumed by the index and their events never reached the persistence queue, \ + so nothing will ever deliver them: this is an event leak upstream of the analyzer, \ + not a batch collection problem." + ); + let _ = write!(out, "{}", bracketing_producers(ledger, last_applied, next_available)); + } + + if !queued_not_applied.is_empty() { + let _ = write!(out, " QUEUED BUT NOT APPLIED:"); + for (id, record) in queued_not_applied.iter().take(MAX_LISTED_GAP_IDS) { + let _ = write!( + out, + " [{id}: {stages}, collected {collected}x, requeued {requeued}x, {op_type} op {op_id} from {site}]", + stages = record.stages, + collected = record.collected, + requeued = record.requeued, + op_type = OptionDisplay(record.op_type.as_ref().map(|t| format!("{t:?}"))), + op_id = OptionDisplay(record.op_id.as_ref().map(|id| format!("{id:?}"))), + site = OptionDisplay(record.site.map(|site| site.to_string())), + ); + } + if queued_not_applied.len() > MAX_LISTED_GAP_IDS { + let _ = write!(out, " and {} more", queued_not_applied.len() - MAX_LISTED_GAP_IDS); + } + let _ = write!( + out, + ". Those events did reach the queue, so their operations are still somewhere in the analyzer \ + and batch collection is failing to assemble them: the bug is in collection, not in event production." + ); + } + + out + } +} + +/// Names the producers on either side of the hole. +/// +/// A leaked id has no record of its own, so the closest evidence about who +/// should have produced it is who produced its neighbours. One index allocates +/// its ids from one counter, so the neighbours are almost always the same call +/// path. +fn bracketing_producers(ledger: &StreamLedger, last_applied: u64, next_available: u64) -> String { + let mut out = String::new(); + let before = ledger.ids.range(..=last_applied).next_back(); + let after = ledger.ids.range(next_available..).next(); + for (side, entry) in [("before the gap", before), ("after the gap", after)] { + let Some((id, record)) = entry else { + let _ = write!(out, " No record {side}."); + continue; + }; + let _ = write!( + out, + " Producer {side} (id {id}): {op_type}, op {op_id}, pushed from {site}.", + op_type = OptionDisplay(record.op_type.as_ref().map(|t| format!("{t:?}"))), + op_id = OptionDisplay(record.op_id.as_ref().map(|id| format!("{id:?}"))), + site = OptionDisplay(record.site.map(|site| site.to_string())), + ); + #[cfg(feature = "std")] + if let Some(backtrace) = &record.backtrace { + let _ = write!(out, " Backtrace:\n{backtrace}\n"); + } + } + #[cfg(feature = "std")] + if !ledger.ids.values().any(|record| record.backtrace.is_some()) { + let _ = write!( + out, + " Re-run with RUST_BACKTRACE=1 for the full producer backtraces, which this run did not capture." + ); + } + out +} + +impl Default for EventLedger { + fn default() -> Self { + Self::detached() + } +} + +struct OptionDisplay(Option); + +impl core::fmt::Display for OptionDisplay { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match &self.0 { + Some(value) => f.write_str(value), + None => f.write_str(""), + } + } +} + +/// Renders a sorted id list compactly, collapsing runs. +fn format_ids(ids: &[u64]) -> String { + let mut out = String::new(); + let mut listed = 0usize; + let mut i = 0usize; + while i < ids.len() && listed < MAX_LISTED_GAP_IDS { + let start = ids[i]; + let mut end = start; + while i + 1 < ids.len() && ids[i + 1] == end + 1 { + i += 1; + end = ids[i]; + } + if !out.is_empty() { + out.push_str(", "); + } + if start == end { + let _ = write!(out, "{start}"); + } else { + let _ = write!(out, "{start}..={end}"); + } + listed += 1; + i += 1; + } + if i < ids.len() { + let _ = write!(out, ", and {} more", ids.len() - i); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn insert_at(id: u64) -> ChangeEvent> { + ChangeEvent::InsertAt { + event_id: id.into(), + max_value: Pair { + key: id, + value: Link::default(), + }, + value: Pair { + key: id, + value: Link::default(), + }, + index: 0, + } + } + + #[track_caller] + fn queue(ledger: &EventLedger, ids: &[u64], op_type: OperationType) { + let evs = ids.iter().copied().map(insert_at).collect::>(); + let ids = EventLedger::event_ids(&evs); + ledger.record_queued( + EventStream::Primary, + &ids, + OperationId::Single(uuid::Uuid::from_u128(1)), + op_type, + Location::caller(), + ); + } + + /// These tests assert on what the ledger recorded, so they are only + /// meaningful where it records. That is every normal test run + /// (`debug_assertions`); a `--release` test run skips them rather than + /// failing on a report that correctly says bookkeeping was off. + fn recording() -> bool { + enabled() + } + + #[test] + fn names_ids_that_were_never_queued() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3], OperationType::Insert); + // 4 and 5 are assigned by the index and leaked: nothing queues them. + queue(&ledger, &[6, 7], OperationType::Update); + + let report = ledger.gap_report(&EventStream::Primary, 3, 6); + + assert!(report.contains("ASSIGNED BUT NEVER QUEUED"), "{report}"); + assert!(report.contains("4..=5"), "{report}"); + assert!(report.contains("event leak upstream of the analyzer"), "{report}"); + } + + #[test] + fn distinguishes_a_queued_id_from_a_leaked_one() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3], OperationType::Insert); + queue(&ledger, &[4], OperationType::Update); + queue(&ledger, &[6], OperationType::Insert); + + let report = ledger.gap_report(&EventStream::Primary, 3, 6); + + assert!(report.contains("QUEUED BUT NOT APPLIED"), "{report}"); + assert!(report.contains("the bug is in collection"), "{report}"); + // Only id 5 is missing; 4 was queued. + assert!(report.contains("ASSIGNED BUT NEVER QUEUED: 5."), "{report}"); + } + + #[test] + fn reports_a_gapless_stream_as_nothing() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3], OperationType::Insert); + assert!(ledger.gap_report(&EventStream::Primary, 3, 4).is_empty()); + } + + #[test] + fn window_eviction_is_reported_rather_than_guessed() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + let ids = (1..=(WINDOW as u64 + 64)).collect::>(); + queue(&ledger, &ids, OperationType::Insert); + + // Ask about a gap far below the retained window. + let report = ledger.gap_report(&EventStream::Primary, 1, 20); + assert!(report.contains("fell out of the retained window"), "{report}"); + } + + #[test] + fn applied_watermark_marks_everything_behind_it() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3, 5], OperationType::Insert); + ledger.record_applied_upto(EventStream::Primary, 3); + + let report = ledger.gap_report(&EventStream::Primary, 3, 5); + assert!(report.contains("ASSIGNED BUT NEVER QUEUED: 4."), "{report}"); + } + + #[test] + fn stages_render_every_flag_set() { + let mut stages = Stages::default(); + assert_eq!(stages.to_string(), "none"); + stages.insert(Stages::QUEUED); + stages.insert(Stages::TRIMMED); + assert_eq!(stages.to_string(), "queued+trimmed"); + } +} diff --git a/src/persistence/mod.rs b/src/persistence/mod.rs index b5e03ce3..e1d1ccf2 100644 --- a/src/persistence/mod.rs +++ b/src/persistence/mod.rs @@ -1,20 +1,26 @@ -use std::future::Future; - -use data_bucket::page::PageId; +#[cfg(feature = "std")] +use core::future::Future; +#[cfg(feature = "std")] use crate::persistence::operation::BatchOperation; +#[cfg(feature = "std")] pub use engine::DiskConfig; +#[cfg(feature = "std")] pub use engine::DiskPersistenceEngine; +#[cfg(feature = "std")] pub use error::{ PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, PersistenceResult, PersistenceState, load_persisted_state, }; +pub use event_ledger::{EventLedger, EventStream, Stages}; pub use operation::{ AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, OperationId, OperationType, UpdateOperation, validate_events, }; +#[cfg(feature = "std")] pub use readonly_engine::ReadOnlyPersistenceEngine; +#[cfg(feature = "std")] pub use space::{ ArtPersistenceKey, IndexTableOfContents, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, @@ -22,6 +28,7 @@ pub use space::{ TocEntryOversizedError, map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, }; +#[cfg(feature = "std")] pub use task::{PersistenceMonitor, PersistenceTask}; /// Result of retiring one Arc-owned persisted table generation. @@ -38,14 +45,16 @@ pub struct UnloadReport { /// Failures before shutdown return ownership of the generation so the caller /// can keep serving it or retry. A failure returned by `close` has no retained /// generation because shutdown was already attempted and consumed it. +#[cfg(feature = "std")] pub struct UnloadFailure { - generation: Option>, + generation: Option>, error: eyre::Report, } +#[cfg(feature = "std")] impl UnloadFailure { #[doc(hidden)] - pub fn retained(generation: std::sync::Arc, error: eyre::Report) -> Self { + pub fn retained(generation: alloc::sync::Arc, error: eyre::Report) -> Self { Self { generation: Some(generation), error, @@ -61,7 +70,7 @@ impl UnloadFailure { } /// Returns the still-live generation when shutdown never began. - pub fn into_generation(self) -> Option> { + pub fn into_generation(self) -> Option> { self.generation } @@ -71,8 +80,9 @@ impl UnloadFailure { } } -impl std::fmt::Debug for UnloadFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +#[cfg(feature = "std")] +impl core::fmt::Debug for UnloadFailure { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_struct("UnloadFailure") .field("generation_retained", &self.generation.is_some()) @@ -81,19 +91,30 @@ impl std::fmt::Debug for UnloadFailure { } } -impl std::fmt::Display for UnloadFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +#[cfg(feature = "std")] +impl core::fmt::Display for UnloadFailure { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.error.fmt(formatter) } } -impl std::error::Error for UnloadFailure {} +#[cfg(feature = "std")] +impl core::error::Error for UnloadFailure {} + +#[cfg(feature = "std")] +use data_bucket::page::PageId; +#[cfg(feature = "std")] mod engine; +#[cfg(feature = "std")] mod error; +pub mod event_ledger; pub mod operation; +#[cfg(feature = "std")] mod readonly_engine; +#[cfg(feature = "std")] mod space; +#[cfg(feature = "std")] mod task; // TODO: remove this @@ -123,6 +144,7 @@ pub enum LoadMode { Recovery, } +#[cfg(feature = "std")] pub trait PersistedWorkTable: Sized where E: Send, @@ -142,6 +164,7 @@ where } } +#[cfg(feature = "std")] pub trait PersistenceEngine { type Config: PersistenceConfig; diff --git a/src/persistence/operation/batch.rs b/src/persistence/operation/batch.rs index 958bcfc0..1cb93144 100644 --- a/src/persistence/operation/batch.rs +++ b/src/persistence/operation/batch.rs @@ -1,7 +1,10 @@ -use std::collections::HashMap; -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::boxed::Box; +use alloc::sync::Arc; +use alloc::{string::ToString, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; +use hashbrown::HashMap; use data_bucket::page::PageId; use data_bucket::{Link, SizeMeasurable}; @@ -10,10 +13,11 @@ use indexset::core::pair::Pair; use worktable_codegen::{MemStat, worktable}; use crate::persistence::OperationType; +use crate::persistence::event_ledger::{self, EventLedger, EventStream, Stages}; use crate::persistence::space::{BatchChangeEvent, BatchData}; use crate::persistence::task::{LastEventIds, QueueInnerRow}; use crate::prelude::*; -use crate::prelude::{From, Order, SelectQueryExecutor}; +use crate::prelude::{Order, SelectQueryExecutor}; /// Cycles of a persistently gapped event stream before the engine gives up and /// fails the table. @@ -83,17 +87,20 @@ impl From for BatchInnerRow { } /// Coalesces durable row writes by physical storage slot and preserves their -/// creation order. +/// mutation order. /// /// `Link::length` can change when an unsized row is reinserted into a reused /// `(page_id, offset)`. Treating the two lengths as different keys leaves /// overlapping writes in the same batch. The newest operation must be the only /// write for an identical physical start, and writes at different starts must /// still be applied oldest-to-newest: range splitting can make them overlap. -/// WorkTable-generated operation IDs use `Uuid::now_v7`, whose shared process -/// context guarantees creation-order sorting even within one millisecond; -/// callers constructing `Operation` values manually must preserve that -/// ordering contract. +/// Primary-index event ids are assigned while the in-memory mutation is in +/// progress. Operation ids are minted later, and concurrent writers can be +/// descheduled between those two points. The durable primary index is replayed +/// in event-id order, so row mutations that carry primary events must use that +/// same order or a reused slot can finish with the new index entry and the old +/// row bytes. Event-less data updates retain operation-id order; their row +/// mutation gate keeps that order stable for a physical slot. fn latest_data_writes( ops: &[Operation], ) -> BatchData { @@ -103,52 +110,67 @@ fn latest_data_writes( ops: &[Operation], order: impl Iterator + Clone, ) -> BatchData { - let mut latest: HashMap = HashMap::with_capacity(ops.len()); - for sequence in order.clone() { - let op = &ops[sequence]; - if op.bytes().is_some() { - let link = op.link(); - latest.insert((link.page_id, link.offset), sequence); - } + let mutations: Vec<_> = order.flat_map(|sequence| ops[sequence].row_mutations()).collect(); + let mut latest: HashMap = HashMap::with_capacity(mutations.len()); + for (sequence, (link, _)) in mutations.iter().enumerate() { + latest.insert((link.page_id, link.offset), sequence); } - let mut ordered = HashMap::new(); - for sequence in order { - let op = &ops[sequence]; - let Some(bytes) = op.bytes() else { - continue; - }; - let link = op.link(); + for (sequence, (link, bytes)) in mutations.into_iter().enumerate() { if latest.get(&(link.page_id, link.offset)) != Some(&sequence) { continue; } - ordered - .entry(link.page_id) - .or_insert_with(Vec::new) - .push((link, bytes.to_vec())); + ordered.entry(link.page_id).or_insert_with(Vec::new).push((link, bytes)); } ordered } - // The analyzer already establishes this order. Keep that production path - // linear; only defensive callers that construct an unsorted BatchOperation - // pay for an index sort. - if ops - .windows(2) - .all(|pair| pair[0].operation_id() <= pair[1].operation_id()) - { - collect_in_order(ops, 0..ops.len()) - } else { - let mut order = (0..ops.len()).collect::>(); - order.sort_unstable_by_key(|sequence| (ops[*sequence].operation_id(), *sequence)); - collect_in_order(ops, order.into_iter()) + let mut order = (0..ops.len()).collect::>(); + order.sort_unstable_by_key(|sequence| (ops[*sequence].operation_id(), *sequence)); + + // Preserve event-less operations at their operation-id positions, while + // putting every primary-event mutation into the order used by the durable + // index. Replacing those positions avoids a mixed-key comparator: event + // ids and UUIDs are independent clocks and cannot form one total order. + let event_positions = order + .iter() + .enumerate() + .filter_map(|(position, sequence)| { + ops[*sequence] + .primary_key_events() + .is_some_and(|events| !events.is_empty()) + .then_some(position) + }) + .collect::>(); + let mut event_sequences = event_positions + .iter() + .map(|position| order[*position]) + .collect::>(); + event_sequences.sort_unstable_by_key(|sequence| { + ( + ops[*sequence] + .primary_key_events() + .and_then(|events| events.first()) + .expect("event-carrying operation has a first event") + .id(), + *sequence, + ) + }); + for (position, sequence) in event_positions.into_iter().zip(event_sequences) { + order[position] = sequence; } + + collect_in_order(ops, order.into_iter()) } #[derive(Debug)] pub struct BatchOperation { ops: Vec>, info_wt: BatchInnerWorkTable, + /// Event bookkeeping shared with the queue that produced `ops`, read by + /// the event-gap guard in `validate` so a stall names its own cause. + /// Diagnostics only, and `None` for batches built outside the analyzer. + event_ledger: Option>, prepared_index_evs: Option>, phantom_data: PhantomData, } @@ -173,11 +195,23 @@ where Self { ops, info_wt, + event_ledger: None, prepared_index_evs: None, phantom_data: PhantomData, } } + /// Attaches the analyzer's event bookkeeping, so the gap guard below can + /// say which ids in a gap ever reached the persistence queue. + /// + /// A builder method rather than a `new` parameter, so every existing + /// caller of `new` keeps working unchanged and the batch stays usable + /// without any bookkeeping at all. + pub fn with_event_ledger(mut self, ledger: Arc) -> Self { + self.event_ledger = Some(ledger); + self + } + /// Remove metadata immediately after `self.ops.remove(removed_pos)`. /// /// At entry, `self.ops.len()` is already one shorter while `info_wt` still @@ -272,21 +306,68 @@ where prepared_evs.secondary_evs.remove(op_secondary); } + self.record_stage(&removed_ops, Stages::TRIMMED); + Ok(removed_ops) } + /// Records `stage` against every event id carried by `ops`. + /// + /// Diagnostics only. Skipped entirely when bookkeeping is off, which keeps + /// the `Debug` formatting of secondary index labels out of release builds. + fn record_stage(&self, ops: &[Operation], stage: Stages) { + let Some(ledger) = &self.event_ledger else { + return; + }; + if !event_ledger::enabled() { + return; + } + for op in ops { + if let Some(evs) = op.primary_key_events() { + ledger.record_stage_for_events(EventStream::Primary, evs, stage); + } + // See the matching note in `QueueAnalyzer::record_ops_stage`: the + // producer side records primary ids only, so an id observed on a + // secondary stream here is known to have been queued. + for (index, id) in op.secondary_key_events().iter_event_ids() { + ledger.record_stage( + EventStream::Secondary(format!("{index:?}")), + id.inner(), + stage.union(Stages::QUEUED), + ); + } + } + } + + /// The bookkeeping's account of a gap, or a note saying there is none. + fn gap_report( + &self, + stream: &EventStream, + last_applied: Option, + next_available: IndexChangeEventId, + ) -> String { + match &self.event_ledger { + // Nothing applied yet reports as `0`, which is what the ledger + // means by "everything from the start is missing": the first id is + // 0, so there is no id below it to name. + Some(ledger) => ledger.gap_report(stream, last_applied.map_or(0, |id| id.inner()), next_available.inner()), + None => { + " This batch was built without event bookkeeping attached, so the gap cannot be attributed.".to_owned() + } + } + } + pub fn get_last_event_ids(&self) -> LastEventIds { let prepared_evs = self .prepared_index_evs .as_ref() .expect("should be set before 0 iteration"); - let primary_id = prepared_evs.primary_evs.last().map(|ev| ev.id()).unwrap_or_default(); - let secondary_ids = prepared_evs.secondary_evs.last_evs(); - let secondary_ids = secondary_ids - .into_iter() - .map(|(i, v)| (i, v.unwrap_or_default())) - .collect(); + // `None` where a stream contributed no events, rather than `default()`. + // Event ids start at 0 and so does `default()`, so collapsing the two + // reported "applied up to event 0" for a batch that applied nothing. + let primary_id = prepared_evs.primary_evs.last().map(|ev| ev.id()); + let secondary_ids = prepared_evs.secondary_evs.last_evs().into_iter().collect(); LastEventIds { primary_id, secondary_ids, @@ -344,9 +425,26 @@ where .prepared_index_evs .as_ref() .expect("should be set before 0 iteration"); + // No exemption for the first batch any more. It used to carry + // `&& last_ids.primary_id != IndexChangeEventId::default()`, so a + // stream with nothing applied accepted *any* starting id, and that + // is exactly where the ids can be wrong: event ids are allocated + // during the index mutation while the operation is enqueued + // afterwards, so two concurrent writers invert the two orders (see + // `COLLECT_WHOLE_QUEUE_AFTER_ATTEMPTS`). Measured, a first batch of + // ids 3..=28 was internally gapless, passed the exemption, and + // advanced a node's maximum to key 28; events 0..=2 then arrived + // naming a node whose maximum was still 1, resolved against + // nothing, and failed with a missing page having already written + // the file. + // + // The exemption was not gratuitous, which is why the fix is in + // `LastEventIds` rather than here: ids start at 0 and `default()` + // is 0, so the old representation could not tell "nothing applied" + // from "applied event 0" and had to wave the first batch through. + // `follows` asks the question that representation could not. if let Some(id) = prepared_evs.primary_evs.first().map(|ev| ev.id()) - && !id.is_next_for(last_ids.primary_id) - && last_ids.primary_id != IndexChangeEventId::default() + && !LastEventIds::::follows(last_ids.primary_id, id) { // Change events are positional (InsertAt/RemoveAt carry node // indices), so a stream with a missing id must never be applied: @@ -358,8 +456,9 @@ where // that persists is a bug upstream of the analyzer; report it // loudly instead of force-applying and corrupting the file. if attempts > GIVE_UP_AFTER_ATTEMPTS { + let report = self.gap_report(&EventStream::Primary, last_ids.primary_id, id); return Err(eyre::eyre!( - "persistence stalled on primary index event gap: last applied {:?}, next available {:?} after {attempts} attempts, with {} operations queued. Every one of them was collected and the stream is still gapped, so the operation carrying the missing id never reached the queue: an event id was consumed without its event being pushed. The producer is upstream of the analyzer, not here.", + "persistence stalled on primary index event gap: last applied {:?}, next available {:?} after {attempts} attempts, with {} operations queued.{report}", last_ids.primary_id, id, self.ops.len() @@ -373,16 +472,21 @@ where let Some(last) = last_ids.secondary_ids.get(&index) else { continue; }; + // Same rule as the primary above, including the absence of a + // first-batch exemption. A stream with no entry at all is + // skipped by the `continue` above; an entry holding `None` is + // a stream nothing has been applied to yet, which is the case + // that needs checking rather than the case to wave through. if let Some(id) = id - && !id.is_next_for(*last) - && *last != IndexChangeEventId::default() + && !LastEventIds::::follows(*last, id) { // Same rule as the primary index above: never apply a gapped // stream, defer until the missing event arrives, and report // a persistent gap as the bug it is. if attempts > GIVE_UP_AFTER_ATTEMPTS { + let report = self.gap_report(&EventStream::Secondary(format!("{index:?}")), *last, id); return Err(eyre::eyre!( - "persistence stalled on secondary index {index:?} event gap: last applied {last:?}, next available {id:?} after {attempts} attempts, with {} operations queued. All of them were collected and the stream is still gapped, so the operation carrying the missing id never reached the queue.", + "persistence stalled on secondary index {index:?} event gap: last applied {last:?}, next available {id:?} after {attempts} attempts, with {} operations queued.{report}", self.ops.len() )); } @@ -484,9 +588,10 @@ where #[cfg(test)] mod tests { - use std::collections::HashMap; + use hashbrown::HashMap; use data_bucket::Link; + use data_bucket::page::PageId; use indexset::core::pair::Pair; use uuid::Uuid; @@ -498,6 +603,7 @@ mod tests { fn insert(id: u128, link: Link, bytes: Vec) -> Operation<(), u64, ()> { Operation::Insert(InsertOperation { + retired_link: None, id: OperationId::Single(Uuid::from_u128(id)), primary_key_events: vec![], secondary_keys_events: (), @@ -509,6 +615,7 @@ mod tests { fn multi_insert(id: u128, link: Link, bytes: Vec) -> Operation<(), u64, ()> { Operation::Insert(InsertOperation { + retired_link: None, id: OperationId::Multi(Uuid::from_u128(id)), primary_key_events: vec![], secondary_keys_events: (), @@ -534,7 +641,7 @@ mod tests { // Deliberately reverse vector order: operation ids, not incidental // collection order, define which bytes are newest. let batch = latest_data_writes(&[insert(2, new_link, vec![2; 6]), insert(1, old_link, vec![1; 4])]); - let writes = batch.get(&1.into()).unwrap(); + let writes = batch.get(&PageId::from(1u32)).unwrap(); assert_eq!(writes, &vec![(new_link, vec![2; 6])]); } @@ -554,7 +661,7 @@ mod tests { for _ in 0..128 { let batch = latest_data_writes(&[insert(2, newer_link, vec![2; 8]), insert(1, older_link, vec![1; 8])]); - let writes = batch.get(&1.into()).unwrap(); + let writes = batch.get(&PageId::from(1u32)).unwrap(); assert_eq!(writes, &vec![(older_link, vec![1; 8]), (newer_link, vec![2; 8])]); } @@ -579,7 +686,7 @@ mod tests { ]); assert_eq!( - batch.get(&1.into()).unwrap(), + batch.get(&PageId::from(1u32)).unwrap(), &vec![(older_link, vec![1; 8]), (newer_link, vec![2; 8])] ); } @@ -602,7 +709,7 @@ mod tests { multi_insert(1, new_link, vec![2; 6]), ]); - assert_eq!(batch.get(&1.into()).unwrap(), &vec![(new_link, vec![2; 6])]); + assert_eq!(batch.get(&PageId::from(1u32)).unwrap(), &vec![(new_link, vec![2; 6])]); } #[tokio::test] @@ -634,7 +741,7 @@ mod tests { } fn iter_event_ids(&self) -> impl Iterator { - std::iter::empty() + core::iter::empty() } fn sort(&mut self) {} @@ -669,6 +776,7 @@ mod tests { fn event_insert(id: u128, link: Link, bytes: Vec, event_ids: Vec) -> Operation<(), u64, TestEvents> { Operation::Insert(InsertOperation { + retired_link: None, id: OperationId::Single(Uuid::from_u128(id)), primary_key_events: event_ids.into_iter().map(primary_event).collect(), secondary_keys_events: TestEvents, @@ -678,6 +786,88 @@ mod tests { }) } + #[test] + fn reused_slot_follows_primary_event_order_when_operation_ids_invert() { + let link = link_at(128); + + // The old row received event 0 and the replacement received event 1, + // but the producers reached operation-id creation in reverse order. + // The primary index therefore finishes at the replacement link, and + // the data batch must finish with the replacement bytes as well. + let replacement = event_insert(1, link, vec![2; 4], vec![1]); + let old = event_insert(2, link, vec![1; 4], vec![0]); + + let batch = latest_data_writes(&[replacement, old]); + + assert_eq!(batch.get(&PageId::from(1u32)).unwrap(), &vec![(link, vec![2; 4])]); + } + + async fn batch_of(op: Operation<(), u64, TestEvents>) -> BatchOperation<(), u64, TestEvents, TestIndex> { + let info_wt = BatchInnerWorkTable::default(); + info_wt + .insert(BatchInnerRow { + id: 0, + operation_id: op.operation_id(), + page_id: op.link().page_id, + link: op.link(), + op_type: OperationType::Insert, + pos: 0, + }) + .await + .unwrap(); + BatchOperation::new(vec![op], info_wt) + } + + fn link_at(offset: u32) -> Link { + Link { + page_id: 1.into(), + offset, + length: 4, + } + } + + /// A first batch that does not start at the head of the stream must defer. + /// + /// The gap check used to exempt the first batch outright, because event + /// ids start at 0 and so does `IndexChangeEventId::default()`: the + /// watermark could not tell "nothing applied yet" from "applied event 0", + /// so asking whether the batch followed what came before would have + /// deferred every stream's opening batch forever. + /// + /// The cost of that exemption is this: a first batch of ids 3.. is + /// internally gapless, so event validation passes it and nothing else + /// looks. It gets applied, advancing the on-disk node maxima, and events + /// 0..=2 then arrive naming nodes whose maxima no longer exist. Making the + /// watermark an `Option` lets the question be asked of the first batch too. + #[tokio::test] + async fn a_first_batch_that_skips_the_head_of_the_stream_defers() { + let op = event_insert(1, link_at(0), vec![1; 4], vec![3, 4, 5]); + let mut batch = batch_of(op).await; + + let outcome = batch.validate(&LastEventIds::default(), 0).await.unwrap(); + + assert!( + outcome.is_none(), + "a first batch starting at event 3 must be deferred until events 0..=2 arrive" + ); + } + + /// The other half: the exemption existed for a reason, and removing it + /// must not deadlock a legitimate opening batch. Event 0 is a real id, not + /// the absence of one. + #[tokio::test] + async fn a_first_batch_starting_at_event_zero_applies() { + let op = event_insert(1, link_at(0), vec![1; 4], vec![0, 1, 2]); + let mut batch = batch_of(op).await; + + let outcome = batch.validate(&LastEventIds::default(), 0).await.unwrap(); + + assert!( + outcome.is_some(), + "a first batch starting at the first event must be applied, not deferred" + ); + } + /// Regression: removing the last event-carrying operation from a batch /// discarded the surviving data-only operations. /// @@ -734,7 +924,7 @@ mod tests { let data = batch.get_batch_data_op().unwrap(); assert_eq!( - data.get(&1.into()).unwrap(), + data.get(&PageId::from(1u32)).unwrap(), &vec![(survivor_link, vec![7; 4])], "the surviving data-only write must stay in the applied batch" ); diff --git a/src/persistence/operation/clock.rs b/src/persistence/operation/clock.rs new file mode 100644 index 00000000..558b3c52 --- /dev/null +++ b/src/persistence/operation/clock.rs @@ -0,0 +1,48 @@ +use uuid::Uuid; + +#[cfg(feature = "std")] +pub(crate) fn new_operation_uuid() -> Uuid { + Uuid::now_v7() +} + +// ContextV7 preserves process-local ordering across equal or regressing clock +// readings. OS entropy remains supplied by uuid/getrandom without Rust std. +#[cfg(not(feature = "std"))] +pub(crate) fn new_operation_uuid() -> Uuid { + static CONTEXT: parking_lot::Mutex = parking_lot::Mutex::new(uuid::ContextV7::new()); + let (seconds, nanos) = unix_time(); + let context = CONTEXT.lock(); + Uuid::new_v7(uuid::Timestamp::from_unix(&*context, seconds, nanos)) +} + +#[cfg(all(not(feature = "std"), unix))] +fn unix_time() -> (u64, u32) { + let mut value = core::mem::MaybeUninit::::uninit(); + // SAFETY: the OS writes a timespec to a valid, aligned output pointer. + let result = unsafe { libc::clock_gettime(libc::CLOCK_REALTIME, value.as_mut_ptr()) }; + assert_eq!(result, 0, "system clock unavailable for operation identifiers"); + // SAFETY: a successful clock_gettime initialized both fields. + let value = unsafe { value.assume_init() }; + assert!(value.tv_sec >= 0, "system clock precedes the Unix epoch"); + (value.tv_sec as u64, value.tv_nsec as u32) +} + +#[cfg(all(not(feature = "std"), windows))] +fn unix_time() -> (u64, u32) { + use windows_sys::Win32::Foundation::FILETIME; + use windows_sys::Win32::System::SystemInformation::GetSystemTimePreciseAsFileTime; + let mut value = core::mem::MaybeUninit::::uninit(); + // SAFETY: the API initializes the FILETIME at this valid output pointer. + let value = unsafe { + GetSystemTimePreciseAsFileTime(value.as_mut_ptr()); + value.assume_init() + }; + let ticks = (u64::from(value.dwHighDateTime) << 32) | u64::from(value.dwLowDateTime); + let ticks = ticks + .checked_sub(116_444_736_000_000_000) + .expect("system clock precedes the Unix epoch"); + (ticks / 10_000_000, ((ticks % 10_000_000) * 100) as u32) +} + +#[cfg(all(not(feature = "std"), not(any(unix, windows))))] +compile_error!("WorkTable currently requires Unix or Windows OS services without std"); diff --git a/src/persistence/operation/mod.rs b/src/persistence/operation/mod.rs index 5f7954fc..442e0637 100644 --- a/src/persistence/operation/mod.rs +++ b/src/persistence/operation/mod.rs @@ -1,11 +1,14 @@ +#[cfg(feature = "std")] mod batch; +mod clock; #[allow(clippy::module_inception)] mod operation; +pub(crate) use clock::new_operation_uuid; mod util; -use std::cmp::Ordering; -use std::fmt::Debug; -use std::hash::{Hash, Hasher}; +use core::cmp::Ordering; +use core::fmt::Debug; +use core::hash::{Hash, Hasher}; use data_bucket::SizeMeasurable; use derive_more::Display; @@ -14,6 +17,7 @@ use uuid::Uuid; use crate::prelude::From; +#[cfg(feature = "std")] pub use batch::{BatchInnerRow, BatchInnerWorkTable, BatchOperation}; pub use operation::{AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, UpdateOperation}; pub use util::validate_events; @@ -70,7 +74,7 @@ impl SizeMeasurable for OperationId { impl Default for OperationId { fn default() -> Self { - OperationId::Single(Uuid::now_v7()) + OperationId::Single(new_operation_uuid()) } } diff --git a/src/persistence/operation/operation.rs b/src/persistence/operation/operation.rs index 36af8298..4609af1d 100644 --- a/src/persistence/operation/operation.rs +++ b/src/persistence/operation/operation.rs @@ -1,5 +1,6 @@ -use std::fmt::Debug; -use std::hash::{Hash, Hasher}; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::{Hash, Hasher}; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; @@ -69,6 +70,28 @@ impl Operation Vec<(Link, Vec)> { + let mut mutations = Vec::new(); + let retired_link = match self { + Self::Insert(insert) => insert.retired_link, + Self::Update(update) => update.retired_link, + _ => None, + }; + if let Some(link) = retired_link { + mutations.push((link, Vec::new())); + } + if let Self::Delete(delete) = self { + mutations.push((delete.link, Vec::new())); + } + if let Some(bytes) = self.bytes() { + mutations.push((self.link(), bytes.to_vec())); + } + mutations + } + pub fn primary_key_events(&self) -> Option<&Vec>>> { match &self { Operation::Insert(insert) => Some(&insert.primary_key_events), @@ -111,6 +134,8 @@ impl Operation { + /// Previous physical row retired by a successful reinsert. + pub retired_link: Option, pub id: OperationId, pub primary_key_events: Vec>>, pub secondary_keys_events: SecondaryKeys, @@ -121,6 +146,8 @@ pub struct InsertOperation { #[derive(Clone, Debug)] pub struct UpdateOperation { + /// Previous physical row retired by a successful move. + pub retired_link: Option, pub id: OperationId, pub primary_key_events: Vec>>, pub secondary_keys_events: SecondaryKeys, diff --git a/src/persistence/operation/util.rs b/src/persistence/operation/util.rs index 405a9ce3..dd58bb20 100644 --- a/src/persistence/operation/util.rs +++ b/src/persistence/operation/util.rs @@ -1,7 +1,8 @@ +use alloc::vec::Vec; +use core::fmt::Debug; use data_bucket::Link; use indexset::cdc::change::{self, ChangeEvent}; use indexset::core::pair::Pair; -use std::fmt::Debug; pub fn validate_events(evs: &mut Vec>>) -> Vec>> where @@ -20,7 +21,7 @@ where } } - removed_events.sort_by_key(|ev2| std::cmp::Reverse(ev2.id())); + removed_events.sort_by_key(|ev2| core::cmp::Reverse(ev2.id())); removed_events } diff --git a/src/persistence/readonly_engine.rs b/src/persistence/readonly_engine.rs index 27a19d25..b35389bc 100644 --- a/src/persistence/readonly_engine.rs +++ b/src/persistence/readonly_engine.rs @@ -1,5 +1,5 @@ -use std::fmt::Debug; -use std::hash::Hash; +use core::fmt::Debug; +use core::hash::Hash; use crate::TableSecondaryIndexEventsOps; use crate::persistence::operation::{BatchOperation, Operation}; diff --git a/src/persistence/space/art_index.rs b/src/persistence/space/art_index.rs index d2db86c6..352e8ee0 100644 --- a/src/persistence/space/art_index.rs +++ b/src/persistence/space/art_index.rs @@ -6,17 +6,18 @@ //! applies the WAL, writes a new native checkpoint atomically, and drops the //! temporary tree; no duplicate ART is retained during normal operation. -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::{borrow::ToOwned, string::String, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; use std::path::{Path, PathBuf}; +use crate::fsx::File; use data_bucket::{Link, page::PageId}; use eyre::{Context, bail, eyre}; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; -use tokio::fs::{File, OpenOptions}; -use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use nagoya::io::{Read as _, Seek as _, Write as _}; use crate::index::{ ArcticIndex, ArcticKey, ArcticMultiIndex, CongeeIndex, CongeeKey, PersistentArcticIndex, @@ -76,14 +77,14 @@ macro_rules! impl_art_persistence_key { ($($type:ty),+ $(,)?) => { $( impl ArtPersistenceKey for $type { - const WIDTH: u8 = std::mem::size_of::() as u8; + const WIDTH: u8 = core::mem::size_of::() as u8; fn encode_art_key(&self, output: &mut Vec) { output.extend_from_slice(&self.to_be_bytes()); } fn decode_art_key(bytes: &[u8]) -> eyre::Result { - let bytes: [u8; std::mem::size_of::()] = bytes + let bytes: [u8; core::mem::size_of::()] = bytes .try_into() .map_err(|_| eyre!("invalid {}-byte ART key", Self::WIDTH))?; Ok(Self::from_be_bytes(bytes)) @@ -112,7 +113,7 @@ macro_rules! impl_art_persistence_key_signed { ($($type:ty => $raw:ty),+ $(,)?) => { $( impl ArtPersistenceKey for $type { - const WIDTH: u8 = std::mem::size_of::() as u8; + const WIDTH: u8 = core::mem::size_of::() as u8; fn encode_art_key(&self, output: &mut Vec) { let raw = (*self as $raw) ^ ((1 as $raw) << (<$raw>::BITS - 1)); @@ -120,7 +121,7 @@ macro_rules! impl_art_persistence_key_signed { } fn decode_art_key(bytes: &[u8]) -> eyre::Result { - let bytes: [u8; std::mem::size_of::()] = bytes + let bytes: [u8; core::mem::size_of::()] = bytes .try_into() .map_err(|_| eyre!("invalid {}-byte ART key", Self::WIDTH))?; let raw = <$raw>::from_be_bytes(bytes) ^ ((1 as $raw) << (<$raw>::BITS - 1)); @@ -200,17 +201,17 @@ impl ArtFile { // with live state or block a future rename. let stale_temporary = temporary_path(&path); if stale_temporary.exists() { - tokio::fs::remove_file(&stale_temporary).await?; + crate::fsx::remove_file(&stale_temporary).await?; } if !path.exists() { Self::write_new_file(&path, backend, table_version, &empty_snapshot).await?; } let image = Self::read_image(&path, backend, table_version).await?; - let mut file = OpenOptions::new().read(true).write(true).open(&path).await?; + let mut file = crate::fsx::open(&path).await?; // Remove an incomplete final frame before appending. Leaving it in // place would make every later valid frame unreachable on recovery. - file.set_len(image.durable_len).await?; - file.seek(std::io::SeekFrom::End(0)).await?; + crate::fsx::set_len(&mut file, image.durable_len).await?; + file.seek(nagoya::io::SeekFrom::End(0)).await?; Ok(Self { path, file, @@ -222,7 +223,7 @@ impl ArtFile { } async fn read_image(path: &Path, backend: Backend, table_version: u32) -> eyre::Result> { - let mut file = File::open(path) + let mut file = crate::fsx::open(path) .await .wrap_err_with(|| format!("open ART index {}", path.display()))?; let mut bytes = Vec::new(); @@ -339,8 +340,8 @@ impl ArtFile { async fn rewrite(&mut self, snapshot: &[u8]) -> eyre::Result<()> { Self::write_file_atomically(&self.path, self.backend, self.table_version, snapshot).await?; - self.file = OpenOptions::new().read(true).write(true).open(&self.path).await?; - self.file.seek(std::io::SeekFrom::End(0)).await?; + self.file = crate::fsx::open(&self.path).await?; + self.file.seek(nagoya::io::SeekFrom::End(0)).await?; self.wal_bytes = 0; Ok(()) } @@ -356,7 +357,7 @@ impl ArtFile { ) -> eyre::Result<()> { let temporary = temporary_path(path); Self::write_new_file(&temporary, backend, table_version, snapshot).await?; - tokio::fs::rename(&temporary, path).await?; + crate::fsx::rename(&temporary, path).await?; Ok(()) } @@ -372,11 +373,11 @@ impl ArtFile { header.extend_from_slice(&0u32.to_le_bytes()); debug_assert_eq!(header.len(), HEADER_LEN); - let mut file = File::create(path).await?; + let mut file = crate::fsx::create(path).await?; file.write_all(&header).await?; file.write_all(snapshot).await?; file.flush().await?; - file.sync_data().await?; + crate::fsx::sync_data(&mut file).await?; Ok(()) } } @@ -384,7 +385,7 @@ impl ArtFile { fn encode_wal_record(record: &WalRecord) -> Vec { let mut key = Vec::new(); record.key.encode_art_key(&mut key); - let variable_prefix = usize::from(K::WIDTH == 0) * std::mem::size_of::(); + let variable_prefix = usize::from(K::WIDTH == 0) * core::mem::size_of::(); let mut bytes = Vec::with_capacity(9 + variable_prefix + key.len() + 12); bytes.extend_from_slice(&record.event_id.to_le_bytes()); match record.op { @@ -451,7 +452,7 @@ fn decode_wal_record(bytes: &[u8]) -> eyre::Result eyre::Result<()> { - crate::validate_arctic_link(link) + Ok(crate::validate_arctic_link(link)?) } fn logical_record(event: ChangeEvent>) -> eyre::Result> { @@ -717,7 +718,7 @@ where path, Backend::ArcticVariable, table_version, - encode_multi_pairs(std::iter::empty::<(K, Link)>()), + encode_multi_pairs(core::iter::empty::<(K, Link)>()), ) .await?, }) @@ -922,7 +923,7 @@ where K: ArtPersistenceKey + ArcticKey, { async fn new(path: PathBuf, table_version: u32) -> eyre::Result { - let snapshot = encode_multi_pairs(std::iter::empty::<(K, Link)>()); + let snapshot = encode_multi_pairs(core::iter::empty::<(K, Link)>()); Ok(Self { file: ArtFile::open(path, Backend::ArcticMulti, table_version, snapshot).await?, }) @@ -1356,15 +1357,15 @@ mod tests { space.process_change_event(set_event(0, 7, link(7))).await.unwrap(); drop(space); - let durable_len = tokio::fs::metadata(&path).await.unwrap().len(); - let mut file = OpenOptions::new().append(true).open(&path).await.unwrap(); + let durable_len = crate::fsx::metadata(&path).await.unwrap(); + let mut file = crate::fsx::append(&path).await.unwrap(); file.write_all(&WAL_MAGIC[..2]).await.unwrap(); file.flush().await.unwrap(); drop(file); - assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), durable_len + 2); + assert_eq!(crate::fsx::metadata(&path).await.unwrap(), durable_len + 2); let mut space = SpaceArcticIndex::::new(path.clone(), 1).await.unwrap(); - assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), durable_len); + assert_eq!(crate::fsx::metadata(&path).await.unwrap(), durable_len); space.process_change_event(set_event(1, 8, link(8))).await.unwrap(); drop(space); @@ -1373,7 +1374,7 @@ mod tests { .unwrap(); assert_eq!(index.get_value(&7).unwrap().0, link(7)); assert_eq!(index.get_value(&8).unwrap().0, link(8)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1401,7 +1402,7 @@ mod tests { .unwrap(); assert_eq!(index.len(), 128); assert_eq!(index.get_value(&91).unwrap().0, link(92)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } fn remove_event(id: u64, key: u64, value: Link) -> ChangeEvent> { @@ -1439,7 +1440,7 @@ mod tests { assert_eq!(decode_multi_pairs::(&bytes).unwrap(), pairs); assert!(decode_multi_pairs::(&bytes[..bytes.len() - 1]).is_err()); assert_eq!( - decode_multi_pairs::(&encode_multi_pairs(std::iter::empty::<(u64, Link)>())).unwrap(), + decode_multi_pairs::(&encode_multi_pairs(core::iter::empty::<(u64, Link)>())).unwrap(), vec![] ); } @@ -1500,7 +1501,7 @@ mod tests { // A unique reader must refuse the multi file outright. assert!(ArtFile::::read_image(&path, Backend::Arctic, 5).await.is_err()); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1522,7 +1523,7 @@ mod tests { .unwrap(); assert_eq!(reloaded.len(), 50); assert_eq!(reloaded.get(&(u128::MAX - 3)).len(), 10); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1551,7 +1552,7 @@ mod tests { .unwrap(); assert_eq!(reloaded.len(), 4); assert_eq!(reloaded.get_value(&"🦀".to_owned()).unwrap().0, link(4)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[test] @@ -1569,7 +1570,7 @@ mod tests { // A leftover temporary from a crashed checkpoint must be cleaned up // when the index opens. - tokio::fs::write(&temporary, b"crashed checkpoint leftovers") + crate::fsx::write(&temporary, b"crashed checkpoint leftovers") .await .unwrap(); let mut space = SpaceArcticIndex::::new(path.clone(), 1).await.unwrap(); @@ -1597,7 +1598,7 @@ mod tests { .unwrap(); assert_eq!(reloaded.get_value(&7).unwrap().0, link(7)); assert_eq!(reloaded.get_value(&9).unwrap().0, link(9)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1611,16 +1612,16 @@ mod tests { assert!(ArtFile::::read_image(&path, Backend::Congee, 7).await.is_err()); assert!(ArtFile::::read_image(&path, Backend::Arctic, 7).await.is_err()); - let mut file = OpenOptions::new().read(true).write(true).open(&path).await.unwrap(); - file.seek(std::io::SeekFrom::End(-1)).await.unwrap(); + let mut file = crate::fsx::open(&path).await.unwrap(); + file.seek(nagoya::io::SeekFrom::End(-1)).await.unwrap(); let mut last = [0u8; 1]; file.read_exact(&mut last).await.unwrap(); - file.seek(std::io::SeekFrom::End(-1)).await.unwrap(); + file.seek(nagoya::io::SeekFrom::End(-1)).await.unwrap(); file.write_all(&[last[0] ^ 0x80]).await.unwrap(); file.flush().await.unwrap(); drop(file); assert!(ArtFile::::read_image(&path, Backend::Arctic, 7).await.is_err()); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } } diff --git a/src/persistence/space/data.rs b/src/persistence/space/data.rs index dd200019..e806819d 100644 --- a/src/persistence/space/data.rs +++ b/src/persistence/space/data.rs @@ -1,15 +1,17 @@ -use std::collections::HashSet; -use std::io::SeekFrom; +use alloc::{string::String, string::ToString, vec::Vec}; +use hashbrown::HashSet; use std::path::Path; +use crate::fsx::File; use crate::persistence::SpaceDataOps; use crate::persistence::space::{BatchData, open_or_create_file}; use crate::prelude::WT_DATA_EXTENSION; use convert_case::{Case, Casing}; use data_bucket::{ DataPage, GeneralHeader, GeneralPage, Link, PageType, Persistable, SizeMeasurable, SpaceInfoPage, - parse_data_pages_batch, parse_general_header_by_index, parse_page, persist_page, persist_pages_batch, update_at, + parse_data_pages_batch, parse_general_header_by_index, persist_page, persist_pages_batch, }; +use nagoya::io::{Read as _, Write as _}; use rkyv::api::high::HighDeserializer; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -17,8 +19,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize}; -use tokio::fs::File; -use tokio::io::{AsyncSeekExt, AsyncWriteExt}; fn link_sort_key(link: &Link) -> (u32, u32) { (link.page_id.into(), link.offset) @@ -144,18 +144,32 @@ pub struct SpaceData SpaceData { - async fn update_data_length(&mut self) -> eyre::Result<()> { - let offset = (u32::default().aligned_size() * 6) as u64; - // The multiplication must happen in u64: `last_page_id * PAGE_SIZE` - // in u32 wraps once the file passes 4 GiB, and the wrapped position - // lands inside a live early page, overwriting its header in place. - self.data_file - .seek(SeekFrom::Start( - u64::from(self.last_page_id) * u64::from(PAGE_SIZE) + offset, - )) - .await?; - let bytes = rkyv::to_bytes::(&self.current_data_length)?; - self.data_file.write_all(bytes.as_ref()).await?; + /// Creates every page from the current high-water mark through `target`. + /// + /// A link can name a page more than one past `last_page_id`: two writers + /// allocating pages at once hand the queue the higher page first. Creating + /// only the named page left the skipped ids as holes of zeros that the + /// file nonetheless spans, and a hole is not a page. The batch path then + /// classifies a skipped id as already existing, parses the zeroed header + /// back as page 0 and looks up a key the batch never held; reload reads + /// the same junk. So close the gap at the moment it opens. + /// + /// `already_written` names ids the caller persists itself, so the batch + /// path does not pay a second write for each page it is about to write + /// with its rows in it. + async fn create_pages_up_to(&mut self, target: u32, already_written: &HashSet) -> eyre::Result<()> { + while self.last_page_id < target { + let id = self.last_page_id + 1; + if !already_written.contains(&id) { + let mut page = GeneralPage { + header: GeneralHeader::new(id.into(), PageType::Data, 0.into()), + inner: DataPage::::new(), + }; + persist_page::<_, PAGE_SIZE>(&mut page, &mut self.data_file).await?; + } + self.last_page_id = id; + self.current_data_length = 0; + } Ok(()) } @@ -206,7 +220,7 @@ impl SpaceData

) -> bool { - let free_ranges = std::mem::take(&mut self.info.inner.empty_links_list); + let free_ranges = core::mem::take(&mut self.info.inner.empty_links_list); let (remaining, changed) = subtract_used_ranges(free_ranges, used_links); self.info.inner.empty_links_list = remaining; changed @@ -224,6 +238,8 @@ where ::Archived: Deserialize>, SpaceInfoPage: Persistable, { + const PAGE_STRIDE: u32 = PAGE_SIZE; + async fn from_table_files_path + Send>(table_path: S, version: u32) -> eyre::Result { let path = format!("{}/{}", table_path.as_ref(), WT_DATA_EXTENSION); let mut data_file = if !Path::new(&path).exists() { @@ -241,8 +257,32 @@ where } else { open_or_create_file(path).await? }; - let info = parse_page::<_, PAGE_SIZE>(&mut data_file, 0).await?; - let file_length = data_file.metadata().await?.len(); + // The metadata occupies the payload, not the page stride. Read its + // declared length after validating against that payload. DataBucket's + // generic metadata reader takes a u32 capacity while ours is usize; + // stable Rust cannot cast a generic const in another const argument. + let header = parse_general_header_by_index::(&mut data_file, 0).await?; + eyre::ensure!( + header.page_type == PageType::SpaceInfo, + "expected a WorkTable space-info page" + ); + let capacity = (PAGE_SIZE as usize) + .checked_sub(data_bucket::GENERAL_HEADER_SIZE) + .ok_or_else(|| eyre::eyre!("page stride is smaller than its header"))?; + eyre::ensure!(INNER_PAGE_SIZE <= capacity, "inner page exceeds page payload"); + let length = if header.data_length == 0 { + capacity + } else { + header.data_length as usize + }; + eyre::ensure!(length <= capacity, "metadata exceeds page payload capacity"); + let mut bytes = vec![0; length]; + data_file.read_exact(&mut bytes).await?; + let info = GeneralPage { + inner: SpaceInfoPage::from_bytes(&bytes, header.data_version), + header, + }; + let file_length = crate::fsx::file_metadata(&mut data_file).await?; // Mirror the index file's ceil logic: a file whose length is an exact // page multiple ends with a full last page, so the plain floor // division names a page id one past EOF and reopening the table fails @@ -253,7 +293,7 @@ where } else { file_length / PAGE_SIZE as u64 }; - let last_page_header = parse_general_header_by_index(&mut data_file, page_id as u32).await?; + let last_page_header = parse_general_header_by_index::(&mut data_file, page_id as u32).await?; Ok(Self { data_file, @@ -279,57 +319,19 @@ where header: GeneralHeader::new(0.into(), PageType::SpaceInfo, 0.into()), inner: info, }; - persist_page(&mut page, file).await + Ok(persist_page::<_, PAGE_SIZE>(&mut page, file).await?) } async fn save_data(&mut self, link: Link, bytes: &[u8]) -> eyre::Result<()> { - if self.consume_reusable_ranges([link]) { - self.save_info().await?; - } - if link.page_id > self.last_page_id.into() { - let mut page = GeneralPage { - header: GeneralHeader::new(link.page_id, PageType::Data, 0.into()), - inner: DataPage { - length: 0, - data: [0; 1], - }, - }; - persist_page(&mut page, &mut self.data_file).await?; - self.current_data_length = 0; - // High-water mark, as in the batch path below: the new page is the - // one the link names, which can be more than one past the current - // last page. A bare increment left `last_page_id` behind it, so a - // later write to that page would re-create it zero-filled. - self.last_page_id = self.last_page_id.max(link.page_id.into()); - } - // `current_data_length` mirrors the last page's persisted data_length: - // the number of bytes occupied from the page start. Only a write that - // lands on the last page AND ends past the currently occupied extent - // grows it. Rewrites of an existing link and writes into reused free - // ranges (which always sit inside previously occupied extents) must - // not touch it: unconditionally adding `link.length` inflated the - // persisted length on every hot-row update until it exceeded the page - // capacity and a later batch persist sliced out of range. - if u32::from(link.page_id) == self.last_page_id { - let link_end = link - .offset - .checked_add(link.length) - .ok_or_else(|| eyre::eyre!("link range {link:?} overflows u32"))?; - if link_end > self.current_data_length { - self.current_data_length = link_end; - self.update_data_length().await?; - } - } - update_at::<{ PAGE_SIZE }>(&mut self.data_file, link, bytes).await?; - // `update_at` ends with a buffered `write_all` that `tokio::fs::File` - // completes on a background blocking task. Flush before reporting the - // save done so the bytes are visible to any other handle. - self.data_file.flush().await?; - Ok(()) + let mut batch = BatchData::new(); + batch.insert(link.page_id, vec![(link, bytes.to_vec())]); + self.save_batch_data(batch).await } async fn save_batch_data(&mut self, batch_data: BatchData) -> eyre::Result<()> { - let used_links = batch_data.values().flat_map(|ops| ops.iter().map(|(link, _)| *link)); + let used_links = batch_data + .values() + .flat_map(|ops| ops.iter().filter(|(_, bytes)| !bytes.is_empty()).map(|(link, _)| *link)); if self.consume_reusable_ranges(used_links) { self.save_info().await?; } @@ -351,24 +353,28 @@ where // creating several pages could leave `last_page_id` below a page that // now exists. The next batch touching that page would see it as "new" // and re-create it zero-filled, wiping the rows persisted before. - if let Some(max) = ids_to_create.iter().max() { - // High-water mark: every id in `ids_to_create` is > last_page_id by - // construction, but state the monotonic invariant directly so a - // future refactor of the filter above cannot regress it. - self.last_page_id = self.last_page_id.max(*max); + // + // Moving the mark to the maximum is necessary but not sufficient: the + // ids between it and the old mark that this batch does not touch have + // to become real pages too, or they stay holes. `create_pages_up_to` + // skips the ids this batch writes for itself below. + if let Some(max) = ids_to_create.iter().max().copied() { + let written_by_this_batch = ids_to_create.iter().copied().collect::>(); + self.create_pages_up_to(max, &written_by_this_batch).await?; } let created_pages = ids_to_create .into_iter() .map(|id| GeneralPage { header: GeneralHeader::new(id.into(), PageType::Data, 0.into()), inner: DataPage { + rows: Vec::new(), length: 0, data: [0; INNER_PAGE_SIZE], }, }) .collect::>(); let parsed_pages = - parse_data_pages_batch::(&mut self.data_file, ids_to_parse).await?; + parse_data_pages_batch::(&mut self.data_file, ids_to_parse).await?; let updated_pages = vec![parsed_pages, created_pages] .into_iter() @@ -379,7 +385,11 @@ where .get(&id) .expect("should be available as pages parsed from these ids"); for (link, bytes) in ops { - page.inner.update_at(*link, bytes)?; + if bytes.is_empty() { + page.inner.remove_at(*link); + } else { + page.inner.update_at(*link, bytes)?; + } } Ok::<_, eyre::Report>(page) }) @@ -397,7 +407,7 @@ where self.current_data_length = page.inner.length; } - persist_pages_batch(updated_pages, &mut self.data_file).await?; + persist_pages_batch::<_, PAGE_SIZE>(updated_pages, &mut self.data_file).await?; // The batch's last page write is a buffered `write_all`; flush so the // batch is visible to other handles once it reports done. self.data_file.flush().await?; @@ -418,6 +428,18 @@ where return Ok(()); } + // A reclaimed page must contain no live directory entries. Persist + // that state before advertising the whole page as reusable. + let cleared = page_ids + .iter() + .map(|page_id| GeneralPage { + header: GeneralHeader::new(*page_id, PageType::Data, 0.into()), + inner: DataPage::::new(), + }) + .collect(); + persist_pages_batch::<_, PAGE_SIZE>(cleared, &mut self.data_file).await?; + self.data_file.flush().await?; + self.info .inner .empty_links_list @@ -447,7 +469,7 @@ where // Single choke point for the info page reaching disk: enforce the // page-0 slot budget however the free-range list was mutated. self.bound_empty_links_list(); - persist_page(&mut self.info, &mut self.data_file).await?; + persist_page::<_, PAGE_SIZE>(&mut self.info, &mut self.data_file).await?; // A generated table may immediately reopen this file through a // separate handle. Make the updated metadata visible before reporting // success, just as `save_data` does for row bytes. @@ -460,7 +482,9 @@ where mod tests { use data_bucket::page::PageId; - use super::subtract_used_ranges; + use super::{SpaceData, subtract_used_ranges}; + use crate::persistence::SpaceDataOps; + use crate::persistence::space::BatchData; use crate::prelude::Link; fn link(page_id: u32, offset: u32, length: u32) -> Link { @@ -533,4 +557,58 @@ mod tests { assert_eq!(actual, expected, "case {case}"); } } + + /// A gap in the page sequence must not make the batch path parse a hole. + /// + /// Two writers allocating pages at once can hand the queue a link on the + /// higher page first. `save_data` then creates only that page and moves + /// `last_page_id` up to it, so the skipped page is a hole of zeros that + /// the file nonetheless spans. The next batch touching the skipped page + /// classifies it as already existing (`id <= last_page_id`), parses the + /// hole back, reads a `page_id` of 0 out of the zeroed header and looks up + /// a key the batch never contained. + /// + /// This is the mechanism behind + /// `tests/persistence/concurrent_upsert_batch.rs`, reduced to the two + /// calls that produce it so it takes milliseconds instead of forty + /// minutes. + #[tokio::test] + async fn a_batch_touching_a_skipped_page_does_not_parse_a_hole() { + const PAGE: u32 = data_bucket::PAGE_SIZE as u32; + const INNER: usize = data_bucket::PAGE_SIZE - data_bucket::GENERAL_HEADER_SIZE; + + let dir = std::env::temp_dir().join(format!("wt-page-gap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("a scratch dir"); + let path = dir.to_str().expect("utf-8 path").to_owned(); + + let mut space = SpaceData::::from_table_files_path(path, 1) + .await + .expect("a fresh space"); + + // Page 3, with 1 and 2 never written: the out-of-order case. + space + .save_data(link(3, 0, 8), &[1u8; 8]) + .await + .expect("the high page saves"); + assert_eq!(space.last_page_id, 3, "the high-water mark follows the link"); + + // Now hand the batch path the page that was skipped. + let mut batch = BatchData::new(); + batch.insert(PageId::from(1u32), vec![(link(1, 0, 8), vec![2u8; 8])]); + space.save_batch_data(batch).await.expect("the skipped page saves"); + + // The point of the fix is on disk, not in the call returning: every id + // through the high-water mark has to carry its own header. Reading + // them back is what distinguishes a filled gap from a hole that this + // particular call happened to survive. + for id in 1..=3u32 { + let header = super::parse_general_header_by_index::(&mut space.data_file, id) + .await + .expect("a header at every page through the mark"); + assert_eq!(u32::from(header.page_id), id, "page {id} is a page, not a hole"); + } + + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/src/persistence/space/index/mod.rs b/src/persistence/space/index/mod.rs index d4cbc625..bc256825 100644 --- a/src/persistence/space/index/mod.rs +++ b/src/persistence/space/index/mod.rs @@ -1,27 +1,30 @@ +use alloc::{string::String, string::ToString, vec::Vec}; mod page_aliases; mod reconstruct; mod table_of_contents; mod unsized_; mod util; -use std::collections::HashMap; -use std::fmt::Debug; -use std::hash::Hash; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::hash::Hash; +use core::sync::atomic::{AtomicU32, Ordering}; +use hashbrown::HashMap; use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use crate::fsx::File; use convert_case::{Case, Casing}; use data_bucket::page::{IndexValue, PageId}; use data_bucket::{ GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, IndexPage, IndexPageUtility, Link, PageType, SizeMeasurable, - SpaceId, SpaceInfoPage, get_index_page_size_from_data_length, parse_page, persist_page, persist_pages_batch, + SpaceId, SpaceInfoPage, parse_page, persist_page, persist_pages_batch, }; use eyre::eyre; use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; use indexset::concurrent::multimap::BTreeMultiMap; use indexset::core::pair::Pair; +use nagoya::io::Write as _; use rkyv::de::Pool; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -29,8 +32,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; use crate::persistence::SpaceIndexOps; use crate::persistence::space::{BatchChangeEvent, open_or_create_file}; @@ -41,17 +42,23 @@ pub use table_of_contents::{IndexTableOfContents, TocEntryOversizedError}; pub use unsized_::SpaceIndexUnsized; pub use util::{map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general}; +// Both the live B-tree node and its persisted page must use this same capacity. +// Large byte strides do not widen the page's u16 counts and slot identifiers. +fn get_index_page_size_from_data_length(length: usize) -> usize { + crate::prelude::get_index_page_size_from_data_length::(length) +} + #[derive(Debug)] -pub struct SpaceIndex { +pub struct SpaceIndex { space_id: SpaceId, - table_of_contents: IndexTableOfContents<(T, Link), INNER_PAGE_SIZE>, + table_of_contents: IndexTableOfContents<(T, Link), INNER_PAGE_SIZE, STRIDE>, next_page_id: Arc, index_file: File, #[allow(dead_code)] info: GeneralPage>, } -impl SpaceIndex +impl SpaceIndex where T: Archive + Ord @@ -110,9 +117,9 @@ where } else { open_or_create_file(index_file_path).await? }; - let info = parse_page::<_, INNER_PAGE_SIZE>(&mut index_file, 0).await?; + let info = parse_page::<_, INNER_PAGE_SIZE, STRIDE>(&mut index_file, 0).await?; - let file_length = index_file.metadata().await?.len(); + let file_length = crate::fsx::file_metadata(&mut index_file).await?; let page_id = if file_length % (INNER_PAGE_SIZE as u64 + GENERAL_HEADER_SIZE as u64) == 0 { file_length / (INNER_PAGE_SIZE as u64 + GENERAL_HEADER_SIZE as u64) } else { @@ -146,7 +153,7 @@ where async fn add_index_page(&mut self, node: IndexPage, page_id: PageId) -> eyre::Result<()> { let header = GeneralHeader::new(page_id, PageType::Index, self.space_id); let mut general_page = GeneralPage { inner: node, header }; - persist_page(&mut general_page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut general_page, &mut self.index_file).await?; Ok(()) } @@ -160,7 +167,7 @@ where let mut new_node_id = None; let size = get_index_page_size_from_data_length::(INNER_PAGE_SIZE as usize); - let mut utility = IndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + let mut utility = IndexPage::::parse_index_page_utility::(&mut self.index_file, page_id).await?; utility.slots.insert(index, utility.current_index); utility.slots.remove(size); utility.current_length += 1; @@ -168,16 +175,21 @@ where key: value.key.clone(), link: value.value, }; - utility.current_index = - IndexPage::::persist_value(&mut self.index_file, page_id, size, index_value, utility.current_index) - .await?; + utility.current_index = IndexPage::::persist_value::( + &mut self.index_file, + page_id, + size, + index_value, + utility.current_index, + ) + .await?; if node_id.key < value.key { utility.node_id = value.clone().into(); new_node_id = Some(value); } - IndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + IndexPage::::persist_index_page_utility::(&mut self.index_file, page_id, utility).await?; Ok(new_node_id) } @@ -192,7 +204,7 @@ where let mut new_node_id = None; let size = get_index_page_size_from_data_length::(INNER_PAGE_SIZE as usize); - let mut utility = IndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + let mut utility = IndexPage::::parse_index_page_utility::(&mut self.index_file, page_id).await?; let value_position = *utility .slots .get(index) @@ -203,7 +215,7 @@ where utility.slots.remove(index); utility.slots.push(0); utility.current_length -= 1; - IndexPage::::remove_value(&mut self.index_file, page_id, size, utility.current_index).await?; + IndexPage::::remove_value::(&mut self.index_file, page_id, size, utility.current_index).await?; if node_id.key == value.key { let index = *utility @@ -211,11 +223,12 @@ where .get(index - 1) .expect("slots always should exist in `size` bounds"); utility.node_id = - IndexPage::::read_value_with_index(&mut self.index_file, page_id, size, index as usize).await?; + IndexPage::::read_value_with_index::(&mut self.index_file, page_id, size, index as usize) + .await?; new_node_id = Some(utility.node_id.clone().into()) } - IndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + IndexPage::::persist_index_page_utility::(&mut self.index_file, page_id, utility).await?; Ok(new_node_id) } @@ -292,7 +305,8 @@ where .table_of_contents .get(&(node_id.key.clone(), node_id.value)) .ok_or(eyre!("Node with {:?} id is not found", node_id))?; - let mut page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, page_id.into()).await?; + let mut page = + parse_page::, INNER_PAGE_SIZE, STRIDE>(&mut self.index_file, page_id.into()).await?; let splitted_page = page.inner.split(split_index); let new_page_id = if let Some(id) = self.table_of_contents.pop_empty_page_id() { id @@ -316,7 +330,7 @@ where // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. self.add_index_page(splitted_page, new_page_id).await?; - persist_page(&mut page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut page, &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; Ok(()) @@ -327,7 +341,8 @@ where let indexset = BTreeMap::::with_maximum_node_size(size); let mut nodes = Vec::with_capacity(self.table_of_contents.iter().count()); for (_, page_id) in self.table_of_contents.iter() { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, (*page_id).into()).await?; + let page = + parse_page::, INNER_PAGE_SIZE, STRIDE>(&mut self.index_file, (*page_id).into()).await?; nodes.push(page.inner.get_node()); } indexset.attach_nodes(nodes); @@ -343,7 +358,8 @@ where let indexset = BTreeMultiMap::::with_maximum_node_size(size); let mut pages = Vec::with_capacity(self.table_of_contents.iter().count()); for ((key, link), page_id) in self.table_of_contents.iter() { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, (*page_id).into()).await?; + let page = + parse_page::, INNER_PAGE_SIZE, STRIDE>(&mut self.index_file, (*page_id).into()).await?; pages.push(( Pair { key: key.clone(), @@ -357,7 +373,7 @@ where } } -impl SpaceIndexOps for SpaceIndex +impl SpaceIndexOps for SpaceIndex where T: Archive + Ord @@ -410,7 +426,7 @@ where header: GeneralHeader::new(0.into(), PageType::SpaceInfo, 0.into()), inner: info, }; - persist_page(&mut page, file).await + Ok(persist_page::<_, STRIDE>(&mut page, file).await?) } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -442,7 +458,7 @@ where } => self.process_split_node(node_id, split_index).await, }?; // The partial page writes above can end with a buffered `write_all` - // that `tokio::fs::File` completes on a background blocking task. + // that the file behind `fsx` completes on the calling thread. // Flush before reporting the event processed so the bytes are visible // to any other handle that opens this file afterwards. self.index_file.flush().await?; @@ -471,19 +487,31 @@ where // identity without predicting DataBucket's mutation rules. let Some((page_index, aliased_page_key)) = self.resolve_batch_page(&page_aliases, &event_page_key) else { + // Naming the event and the identity it wanted, not + // just the sizes. The counts alone say a lookup failed + // and nothing about why; the id and the key together + // say which event arrived out of order and against + // what node maximum, which is what distinguishes a + // stream applied out of order from two writers sharing + // one file. return Err(eyre!( - "index event references a missing page (toc_segments={}, buffered_pages={}, aliases={})", + "index event {:?} references a missing page {:?} (toc_segments={}, buffered_pages={}, aliases={})", + ev.id(), + event_page_key, self.table_of_contents.pages.len(), pages.len(), - page_aliases.len() + page_aliases.len(), )); }; let page = pages.get_mut(&page_index); let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, page_index.into()) - .await?; + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( + &mut self.index_file, + page_index.into(), + ) + .await?; pages.insert(page_index, page); pages .get_mut(&page_index) @@ -564,8 +592,11 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, page_index.into()) - .await?; + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( + &mut self.index_file, + page_index.into(), + ) + .await?; pages.insert(page_index, page); pages .get_mut(&page_index) @@ -628,7 +659,7 @@ where // authority (`parse_indexset` and the strict load audit iterate TOC // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. - persist_pages_batch(pages.values().cloned().collect(), &mut self.index_file).await?; + persist_pages_batch::<_, STRIDE>(pages.values().cloned().collect(), &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; // The batch's last write is buffered; flush so the batch is visible // to other handles once it reports done. @@ -639,6 +670,7 @@ where #[cfg(test)] mod test { + use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::{INNER_PAGE_SIZE, IndexPage, IndexValue, Persistable, get_index_page_size_from_data_length}; use super::*; @@ -651,7 +683,7 @@ mod test { #[tokio::test] async fn orphan_page_from_crash_between_page_and_toc_write_is_ignored_on_reload() { use indexset::cdc::change::ChangeEvent; - use tokio::io::AsyncWriteExt; + use nagoya::io::Write as _; let dir = std::env::temp_dir().join(format!("wt_orphan_crash_{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); @@ -669,7 +701,7 @@ mod test { }; { - let mut index = SpaceIndex::::new(&path, 0.into(), 1) + let mut index = SpaceIndex::::new(&path, 0.into(), 1) .await .unwrap(); index @@ -687,7 +719,7 @@ mod test { index.index_file.flush().await.unwrap(); } - let mut reloaded = SpaceIndex::::new(&path, 0.into(), 1) + let mut reloaded = SpaceIndex::::new(&path, 0.into(), 1) .await .unwrap(); let restored = reloaded.parse_indexset().await.unwrap(); diff --git a/src/persistence/space/index/page_aliases.rs b/src/persistence/space/index/page_aliases.rs index 131dd05d..627588e4 100644 --- a/src/persistence/space/index/page_aliases.rs +++ b/src/persistence/space/index/page_aliases.rs @@ -5,6 +5,7 @@ //! when a split or a max-remove re-keys a page mid-batch while later events //! still name a historical maximum. +use alloc::vec::Vec; use data_bucket::Link; use data_bucket::page::PageId; use eyre::eyre; @@ -35,7 +36,7 @@ pub(super) struct PageAliasEntry { impl Default for PageAliases { fn default() -> Self { Self { - inline: std::array::from_fn(|_| None), + inline: core::array::from_fn(|_| None), overflow: Vec::new(), } } diff --git a/src/persistence/space/index/reconstruct.rs b/src/persistence/space/index/reconstruct.rs index dab116ca..64ca1c13 100644 --- a/src/persistence/space/index/reconstruct.rs +++ b/src/persistence/space/index/reconstruct.rs @@ -4,7 +4,8 @@ //! generic function that can be unit-tested with synthetic pages; the proc //! macro only generates type plumbing and node attachment. -use std::fmt::Debug; +use alloc::vec::Vec; +use core::fmt::Debug; use indexset::core::multipair::MultiPair; use indexset::core::pair::Pair; @@ -173,7 +174,7 @@ mod tests { for b in nodes.iter().skip(i + 1) { assert_ne!( a.last().unwrap().cmp(b.last().unwrap()), - std::cmp::Ordering::Equal, + core::cmp::Ordering::Equal, "two node maxima compare Equal: {:?} vs {:?}", a.last().unwrap(), b.last().unwrap() @@ -247,7 +248,7 @@ mod tests { assert_eq!(flatten(&nodes[..1]), vec![(1, 1), (1, 2), (2, 30)]); // Every stored entry is distinct. That is what the discriminator counter was // for; identity is now the `(key, value)` pair itself. - let mut seen = std::collections::BTreeSet::new(); + let mut seen = alloc::collections::BTreeSet::new(); for p in nodes.iter().flatten() { assert!(seen.insert((p.key, p.value)), "duplicate entry {:?}", (p.key, p.value)); } diff --git a/src/persistence/space/index/table_of_contents.rs b/src/persistence/space/index/table_of_contents.rs index 633c1cf5..064b6081 100644 --- a/src/persistence/space/index/table_of_contents.rs +++ b/src/persistence/space/index/table_of_contents.rs @@ -1,7 +1,9 @@ -use std::fmt::Debug; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::sync::atomic::{AtomicU32, Ordering}; +use crate::fsx::File; use data_bucket::page::PageId; use data_bucket::{ GeneralHeader, GeneralPage, PageType, SizeMeasurable, SpaceId, TableOfContentsPage, parse_page, persist_page, @@ -13,7 +15,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; /// A table-of-contents entry whose serialized size can never fit a segment. /// @@ -26,8 +27,8 @@ pub struct TocEntryOversizedError { pub segment_capacity: usize, } -impl std::fmt::Display for TocEntryOversizedError { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for TocEntryOversizedError { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( formatter, "table-of-contents entry needs {} bytes but a whole empty segment holds only {}", @@ -36,16 +37,16 @@ impl std::fmt::Display for TocEntryOversizedError { } } -impl std::error::Error for TocEntryOversizedError {} +impl core::error::Error for TocEntryOversizedError {} #[derive(Debug)] -pub struct IndexTableOfContents { +pub struct IndexTableOfContents { current_page: usize, next_page_id: Arc, pub pages: Vec>>, } -impl IndexTableOfContents +impl IndexTableOfContents where T: Debug + SizeMeasurable + Ord + Eq, { @@ -248,7 +249,7 @@ where + for<'a> rkyv::bytecheck::CheckBytes>, { for page in &mut self.pages { - persist_page(page, file).await?; + persist_page::<_, STRIDE>(page, file).await?; } Ok(()) @@ -265,7 +266,7 @@ where + Eq + for<'a> rkyv::bytecheck::CheckBytes>, { - let first_page = parse_page::, DATA_LENGTH>(file, 1).await; + let first_page = parse_page::, DATA_LENGTH, STRIDE>(file, 1).await; let page = match first_page { Ok(page) => page, Err(error) => { @@ -275,11 +276,14 @@ where // file extends into page 1's slot, the parse failure means a // torn or truncated table of contents, and silently starting // empty would discard the whole index. - let file_length = file.metadata().await?.len(); - if file_length <= data_bucket::PAGE_SIZE as u64 { + let file_length = crate::fsx::file_metadata(file).await?; + if file_length <= u64::from(STRIDE) { return Ok(Self::new(space_id, next_page_id)); } - return Err(error.wrap_err(format!( + // `wrap_err` belonged to `eyre::Report`. The parse error is a + // concrete `data_bucket::error::Error` now, so it becomes a report + // first and keeps the same context message. + return Err(eyre::Report::new(error).wrap_err(format!( "table of contents page 1 failed to parse in a {file_length}-byte index file that should contain it" ))); } @@ -297,7 +301,7 @@ where let mut ind = false; while !ind { - let page = parse_page::, DATA_LENGTH>(file, index).await?; + let page = parse_page::, DATA_LENGTH, STRIDE>(file, index).await?; ind = page.header.next_id.is_empty(); index = page.header.next_id.into(); table_of_contents_pages.push(page); @@ -316,13 +320,14 @@ where #[cfg(test)] mod tests { use crate::persistence::space::index::table_of_contents::IndexTableOfContents; + use alloc::sync::Arc; + use core::sync::atomic::AtomicU32; + use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::page::PageId; - use std::sync::Arc; - use std::sync::atomic::AtomicU32; #[test] fn empty() { - let toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); assert_eq!( toc.current_page, 0, "`current_page` is not set to 0, it is {}", @@ -333,7 +338,7 @@ mod tests { #[test] fn insert_to_empty() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); let key = 1; toc.insert(key, 1.into()); @@ -352,7 +357,7 @@ mod tests { #[test] fn checked_update_reports_a_missing_identity_without_mutating_the_toc() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); toc.insert(7, 2.into()); assert!(!toc.try_update_key(&8, 9).unwrap()); @@ -363,7 +368,10 @@ mod tests { #[test] fn growing_key_update_moves_the_entry_instead_of_overflowing_the_segment() { const DATA_LENGTH: u32 = 128; - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new( + 0.into(), + Arc::new(AtomicU32::new(1)), + ); // Fill the first segment close to capacity with short keys. let mut key = 0; @@ -395,7 +403,10 @@ mod tests { use crate::persistence::TocEntryOversizedError; const DATA_LENGTH: u32 = 128; - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new( + 0.into(), + Arc::new(AtomicU32::new(1)), + ); toc.insert("small".to_string(), PageId::from(9)); let oversized = "x".repeat(4 * DATA_LENGTH as usize); @@ -411,7 +422,7 @@ mod tests { #[test] fn insert_more_than_one_page() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); let mut keys = vec![]; for key in 0..10 { toc.insert(key, 1.into()); @@ -437,7 +448,7 @@ mod tests { #[test] fn insert_reaches_existing_tail_after_reload_resets_cursor() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); for key in 0..10 { toc.insert(key, u32::from(key).into()); } @@ -465,7 +476,7 @@ mod tests { #[test] fn insert_reports_a_truncated_segment_chain() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); for key in 0..10 { toc.insert(key, u32::from(key).into()); } @@ -480,7 +491,7 @@ mod tests { #[test] fn reinsert_on_empty_space() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); let mut keys = vec![]; for key in 0..10 { toc.insert(key, 1.into()); diff --git a/src/persistence/space/index/unsized_.rs b/src/persistence/space/index/unsized_.rs index dd7cdad0..babafe6b 100644 --- a/src/persistence/space/index/unsized_.rs +++ b/src/persistence/space/index/unsized_.rs @@ -1,9 +1,11 @@ -use std::collections::HashMap; -use std::fmt::Debug; -use std::hash::Hash; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; - +use alloc::sync::Arc; +use alloc::{string::String, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::sync::atomic::{AtomicU32, Ordering}; +use hashbrown::HashMap; + +use crate::fsx::File; use data_bucket::page::PageId; use data_bucket::{ GeneralHeader, GeneralPage, IndexPageUtility, IndexValue, Link, PageType, SizeMeasurable, SpaceId, SpaceInfoPage, @@ -14,6 +16,7 @@ use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; use indexset::concurrent::multimap::BTreeMultiMap; use indexset::core::pair::Pair; +use nagoya::io::Write as _; use rkyv::de::Pool; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -21,8 +24,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; use super::page_aliases::PageAliases; use crate::UnsizedNode; @@ -31,16 +32,16 @@ use crate::persistence::{IndexTableOfContents, SpaceIndex, SpaceIndexOps, recons use crate::prelude::WT_INDEX_EXTENSION; #[derive(Debug)] -pub struct SpaceIndexUnsized { +pub struct SpaceIndexUnsized { space_id: SpaceId, - table_of_contents: IndexTableOfContents<(T, Link), DATA_LENGTH>, + table_of_contents: IndexTableOfContents<(T, Link), DATA_LENGTH, STRIDE>, next_page_id: Arc, index_file: File, #[allow(dead_code)] info: GeneralPage>, } -impl SpaceIndexUnsized +impl SpaceIndexUnsized where T: Archive + Ord @@ -103,7 +104,7 @@ where } pub async fn new>(index_file_path: S, space_id: SpaceId, version: u32) -> eyre::Result { - let space_index = SpaceIndex::::new(index_file_path, space_id, version).await?; + let space_index = SpaceIndex::::new(index_file_path, space_id, version).await?; Ok(Self { space_id, table_of_contents: space_index.table_of_contents, @@ -126,7 +127,7 @@ where // happened to create the page. let header = GeneralHeader::new(page_id, PageType::IndexUnsized, self.space_id); let mut general_page = GeneralPage { inner: node, header }; - persist_page(&mut general_page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut general_page, &mut self.index_file).await?; Ok(()) } @@ -187,7 +188,8 @@ where let mut new_node_id = None; let mut utility = - UnsizedIndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + UnsizedIndexPage::::parse_index_page_utility::(&mut self.index_file, page_id) + .await?; let index_value = IndexValue { key: value.key.clone(), link: value.value, @@ -201,9 +203,11 @@ where let future_utility_size = UnsizedIndexPageUtility::::persisted_size(utility.slots_size as usize + 1, future_node_id_size); if future_utility_size + utility.last_value_offset as usize + value_size > DATA_LENGTH as usize { - let mut page = - parse_page::, DATA_LENGTH>(&mut self.index_file, page_id.into()) - .await?; + let mut page = parse_page::, DATA_LENGTH, STRIDE>( + &mut self.index_file, + page_id.into(), + ) + .await?; page.inner.apply_change_event(ChangeEvent::InsertAt { // The page mutation ignores event ids; this synthetic event // exists only to reuse the same insertion accounting. @@ -215,11 +219,11 @@ where Self::compact_page_if_needed(&mut page.inner)?; let changed_node_id = (page.inner.node_id.key != utility.node_id.key).then(|| Pair::from(page.inner.node_id.clone())); - persist_page(&mut page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut page, &mut self.index_file).await?; return Ok(changed_node_id); } let previous_offset = utility.last_value_offset; - let value_offset = UnsizedIndexPage::::persist_value( + let value_offset = UnsizedIndexPage::::persist_value::( &mut self.index_file, page_id, previous_offset, @@ -237,7 +241,12 @@ where new_node_id = Some(value); } - UnsizedIndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + UnsizedIndexPage::::persist_index_page_utility::( + &mut self.index_file, + page_id, + utility, + ) + .await?; Ok(new_node_id) } @@ -273,7 +282,8 @@ where let mut new_node_id = None; let mut utility = - UnsizedIndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + UnsizedIndexPage::::parse_index_page_utility::(&mut self.index_file, page_id) + .await?; utility.slots.remove(index); utility.slots_size -= 1; @@ -282,14 +292,23 @@ where .slots .get(index - 1) .expect("slots always should exist in `size` bounds"); - let node_id = - UnsizedIndexPage::::read_value_with_offset(&mut self.index_file, page_id, offset, len) - .await?; + let node_id = UnsizedIndexPage::::read_value_with_offset::( + &mut self.index_file, + page_id, + offset, + len, + ) + .await?; utility.update_node_id(node_id)?; new_node_id = Some(utility.node_id.clone().into()) } - UnsizedIndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + UnsizedIndexPage::::persist_index_page_utility::( + &mut self.index_file, + page_id, + utility, + ) + .await?; Ok(new_node_id) } @@ -300,7 +319,8 @@ where .get(&(node_id.key.clone(), node_id.value)) .ok_or(eyre!("Node with {:?} id is not found", node_id))?; let mut page = - parse_page::, DATA_LENGTH>(&mut self.index_file, page_id.into()).await?; + parse_page::, DATA_LENGTH, STRIDE>(&mut self.index_file, page_id.into()) + .await?; let splitted_page = page.inner.split(split_index); let new_page_id = if let Some(id) = self.table_of_contents.pop_empty_page_id() { id @@ -324,7 +344,7 @@ where // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. self.add_index_page(splitted_page, new_page_id).await?; - persist_page(&mut page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut page, &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; Ok(()) @@ -334,9 +354,11 @@ where let indexset = BTreeMap::>>::with_maximum_node_size(DATA_LENGTH as usize); let mut nodes = Vec::with_capacity(self.table_of_contents.iter().count()); for (_, page_id) in self.table_of_contents.iter() { - let page = - parse_page::, DATA_LENGTH>(&mut self.index_file, (*page_id).into()) - .await?; + let page = parse_page::, DATA_LENGTH, STRIDE>( + &mut self.index_file, + (*page_id).into(), + ) + .await?; let node = page.inner.get_node(); nodes.push(UnsizedNode::from_inner(node, DATA_LENGTH as usize)); } @@ -353,9 +375,11 @@ where let indexset = BTreeMultiMap::>::with_maximum_node_size(DATA_LENGTH as usize); let mut pages = Vec::with_capacity(self.table_of_contents.iter().count()); for ((key, link), page_id) in self.table_of_contents.iter() { - let page = - parse_page::, DATA_LENGTH>(&mut self.index_file, (*page_id).into()) - .await?; + let page = parse_page::, DATA_LENGTH, STRIDE>( + &mut self.index_file, + (*page_id).into(), + ) + .await?; pages.push(( Pair { key: key.clone(), @@ -373,7 +397,8 @@ where } } -impl SpaceIndexOps for SpaceIndexUnsized +impl SpaceIndexOps + for SpaceIndexUnsized where T: Archive + Ord @@ -412,7 +437,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndex::::bootstrap(file, table_name, version).await + SpaceIndex::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -444,7 +469,7 @@ where } => self.process_split_node(node_id, split_index).await, }?; // The partial page writes above can end with a buffered `write_all` - // that `tokio::fs::File` completes on a background blocking task. + // that the file behind `fsx` completes on the calling thread. // Flush before reporting the event processed so the bytes are visible // to any other handle that opens this file afterwards. self.index_file.flush().await?; @@ -484,7 +509,7 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>( + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( &mut self.index_file, page_index.into(), ) @@ -562,7 +587,7 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>( + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( &mut self.index_file, page_index.into(), ) @@ -633,7 +658,7 @@ where // authority (`parse_indexset` and the strict load audit iterate TOC // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. - persist_pages_batch(pages.values().cloned().collect(), &mut self.index_file).await?; + persist_pages_batch::<_, STRIDE>(pages.values().cloned().collect(), &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; // The batch's last write is buffered; flush so the batch is visible // to other handles once it reports done. diff --git a/src/persistence/space/index/util.rs b/src/persistence/space/index/util.rs index cde98019..b45fb700 100644 --- a/src/persistence/space/index/util.rs +++ b/src/persistence/space/index/util.rs @@ -1,16 +1,17 @@ use crate::prelude::IndexTableOfContents; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::sync::atomic::{AtomicU32, Ordering}; use data_bucket::{ GeneralHeader, GeneralPage, IndexPage, Link, PageType, SizeMeasurable, UnsizedIndexPage, VariableSizeMeasurable, }; -use std::fmt::Debug; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; #[allow(clippy::type_complexity)] -pub fn map_index_pages_to_toc_and_general( +pub fn map_index_pages_to_toc_and_general( pages: Vec>, ) -> ( - IndexTableOfContents<(T, Link), DATA_LENGTH>, + IndexTableOfContents<(T, Link), DATA_LENGTH, STRIDE>, Vec>>, ) where @@ -31,10 +32,10 @@ where } #[allow(clippy::type_complexity)] -pub fn map_unsized_index_pages_to_toc_and_general( +pub fn map_unsized_index_pages_to_toc_and_general( pages: Vec>, ) -> ( - IndexTableOfContents<(T, Link), DATA_LENGTH>, + IndexTableOfContents<(T, Link), DATA_LENGTH, STRIDE>, Vec>>, ) where diff --git a/src/persistence/space/logical_index.rs b/src/persistence/space/logical_index.rs index e2e948fc..7c3faa75 100644 --- a/src/persistence/space/logical_index.rs +++ b/src/persistence/space/logical_index.rs @@ -5,10 +5,12 @@ //! this persistence-worker-owned index derives the structural events required //! by the unchanged WTI disk format. -use std::fmt::Debug; -use std::hash::Hash; +use alloc::{string::String, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; use std::path::{Path, PathBuf}; +use crate::fsx::File; use data_bucket::{Link, SizeMeasurable, SpaceId, VariableSizeMeasurable}; use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; @@ -23,7 +25,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; use crate::UnsizedNode; use crate::convert_multi_change_events; @@ -158,25 +159,25 @@ where /// Sized-key WTI persistence with foreground logical CDC and a background /// structural shadow. The wrapped `SpaceIndex` retains the existing file /// layout byte-for-byte. -pub struct SpaceLogicalIndex +pub struct SpaceLogicalIndex where T: Send + Ord + Eq + Clone + 'static, { index_path: PathBuf, shadow: BTreeMap, - disk: SpaceIndex, + disk: SpaceIndex, } -impl Debug for SpaceLogicalIndex +impl Debug for SpaceLogicalIndex where T: Send + Ord + Eq + Clone + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter.debug_struct("SpaceLogicalIndex").finish_non_exhaustive() } } -impl SpaceLogicalIndex +impl SpaceLogicalIndex where T: Archive + Ord @@ -208,7 +209,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalIndex +impl SpaceIndexOps + for SpaceLogicalIndex where T: Archive + Ord @@ -245,7 +247,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndex::::bootstrap(file, table_name, version).await + SpaceIndex::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -260,27 +262,27 @@ where } /// Variable-sized-key counterpart to [`SpaceLogicalIndex`]. -pub struct SpaceLogicalIndexUnsized +pub struct SpaceLogicalIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { index_path: PathBuf, shadow: BTreeMap>>, - disk: SpaceIndexUnsized, + disk: SpaceIndexUnsized, } -impl Debug for SpaceLogicalIndexUnsized +impl Debug for SpaceLogicalIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_struct("SpaceLogicalIndexUnsized") .finish_non_exhaustive() } } -impl SpaceLogicalIndexUnsized +impl SpaceLogicalIndexUnsized where T: Archive + Ord @@ -313,7 +315,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalIndexUnsized +impl SpaceIndexOps + for SpaceLogicalIndexUnsized where T: Archive + Ord @@ -351,7 +354,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndexUnsized::::bootstrap(file, table_name, version).await + SpaceIndexUnsized::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -368,25 +371,25 @@ where /// Sized-key WTI persistence for a non-unique runtime backend that emits /// logical `(key, link)` mutations. The WTI file layout and its node topology /// remain compatible with earlier WorkTable releases. -pub struct SpaceLogicalMultiIndex +pub struct SpaceLogicalMultiIndex where T: Debug + Send + Ord + Eq + Clone + 'static, { index_path: PathBuf, shadow: BTreeMultiMap, - disk: SpaceIndex, + disk: SpaceIndex, } -impl Debug for SpaceLogicalMultiIndex +impl Debug for SpaceLogicalMultiIndex where T: Debug + Send + Ord + Eq + Clone + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter.debug_struct("SpaceLogicalMultiIndex").finish_non_exhaustive() } } -impl SpaceLogicalMultiIndex +impl SpaceLogicalMultiIndex where T: Archive + Ord @@ -418,7 +421,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalMultiIndex +impl SpaceIndexOps + for SpaceLogicalMultiIndex where T: Archive + Ord @@ -455,7 +459,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndex::::bootstrap(file, table_name, version).await + SpaceIndex::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -470,27 +474,28 @@ where } /// Variable-sized-key counterpart to [`SpaceLogicalMultiIndex`]. -pub struct SpaceLogicalMultiIndexUnsized +pub struct SpaceLogicalMultiIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { index_path: PathBuf, shadow: BTreeMultiMap>>, - disk: SpaceIndexUnsized, + disk: SpaceIndexUnsized, } -impl Debug for SpaceLogicalMultiIndexUnsized +impl Debug + for SpaceLogicalMultiIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_struct("SpaceLogicalMultiIndexUnsized") .finish_non_exhaustive() } } -impl SpaceLogicalMultiIndexUnsized +impl SpaceLogicalMultiIndexUnsized where T: Archive + Ord @@ -523,7 +528,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalMultiIndexUnsized +impl SpaceIndexOps + for SpaceLogicalMultiIndexUnsized where T: Archive + Ord @@ -561,7 +567,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndexUnsized::::bootstrap(file, table_name, version).await + SpaceIndexUnsized::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -577,7 +583,7 @@ where #[cfg(test)] mod tests { - use std::collections::BTreeMap as StdBTreeMap; + use alloc::collections::BTreeMap as StdBTreeMap; use data_bucket::page::PageId; diff --git a/src/persistence/space/mod.rs b/src/persistence/space/mod.rs index 4f9d280b..0ac323af 100644 --- a/src/persistence/space/mod.rs +++ b/src/persistence/space/mod.rs @@ -1,17 +1,18 @@ +use alloc::{string::String, vec::Vec}; mod art_index; mod data; mod index; mod logical_index; -use std::collections::HashMap; -use std::future::Future; +use core::future::Future; +use hashbrown::HashMap; use std::path::Path; +use crate::fsx::File; use data_bucket::page::PageId; use data_bucket::{GeneralPage, Link, SpaceInfoPage}; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; -use tokio::fs::{File, OpenOptions}; pub use art_index::{ ArtPersistenceKey, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, @@ -30,6 +31,8 @@ pub type BatchData = HashMap)>>; pub type BatchChangeEvent = Vec>>; pub trait SpaceDataOps { + const PAGE_STRIDE: u32; + fn from_table_files_path + Send>( path: S, version: u32, @@ -88,10 +91,9 @@ pub trait SpaceSecondaryIndexOps { pub async fn open_or_create_file>(path: S) -> eyre::Result { let path = Path::new(path.as_ref()); - Ok(OpenOptions::new() - .write(true) - .read(true) - .create(!path.exists()) - .open(path) - .await?) + Ok(if path.exists() { + crate::fsx::open(path).await? + } else { + crate::fsx::open_or_create(path).await? + }) } diff --git a/src/persistence/task.rs b/src/persistence/task.rs index fc67f09f..a50ef6f0 100644 --- a/src/persistence/task.rs +++ b/src/persistence/task.rs @@ -1,18 +1,24 @@ -use std::collections::{HashMap, HashSet, VecDeque}; -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::time::Duration; +use alloc::boxed::Box; +use alloc::collections::VecDeque; +use alloc::sync::Arc; +use alloc::{borrow::ToOwned, string::String, string::ToString, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; +use core::panic::Location; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use core::time::Duration; +use hashbrown::{HashMap, HashSet}; use data_bucket::page::PageId; +use nagoya::sync::Notify; use parking_lot::Mutex as ParkingMutex; -use tokio::sync::Notify; -use tokio::task::JoinHandle; + +use nagoya::JoinHandle; use worktable_codegen::worktable; -use crate::persistence::operation::{BatchInnerRow, BatchInnerWorkTable, BatchOperation, OperationId}; +use crate::persistence::event_ledger::{self, EventLedger, EventStream, Stages}; +use crate::persistence::operation::{BatchInnerRow, BatchInnerWorkTable, BatchOperation, OperationId, OperationType}; use crate::persistence::{ PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceResult, PersistenceState, }; @@ -28,16 +34,26 @@ worktable! ( page_id: PageId, link: Link, pos: usize, + event_key: u64, }, indexes: { operation_id_idx: operation_id using worktables_index, page_id_idx: page_id using worktables_index, link_idx: link using worktables_index, + event_key_idx: event_key using worktables_index, }, ); const MAX_PAGE_AMOUNT: usize = 16; +/// Operations one event-ordered collection will take. +/// +/// A batch can only ever apply a contiguous run of the event stream, so this +/// caps the run rather than the page count. It is generous because taking too +/// few costs an extra round trip while taking too many costs nothing: anything +/// past the contiguous prefix is trimmed by validation either way. +const MAX_BATCH_OPERATIONS: usize = 512; + /// Attempts after which batch collection stops grouping by data page and takes /// the whole queue. /// @@ -136,6 +152,17 @@ struct WorkerCompletionGuard { armed: bool, } +/// A private blocking-I/O worker must stop its detached pool on every exit. +struct StopWorkerPool(Option); + +impl Drop for StopWorkerPool { + fn drop(&mut self) { + if let Some(stop) = self.0.take() { + stop(); + } + } +} + impl WorkerCompletionGuard { fn new(lifecycle: Arc) -> Self { Self { lifecycle, armed: true } @@ -174,7 +201,7 @@ impl PersistenceMonitor { pub async fn wait_for_failure(self) -> PersistenceResult { loop { let notified = self.lifecycle.terminal_notify.notified(); - tokio::pin!(notified); + let mut notified = core::pin::pin!(notified); // `notify_waiters` does not retain a permit. Register this waiter // before reading the lifecycle state so a terminal transition // cannot land between the state read and the first poll of @@ -194,6 +221,9 @@ pub struct QueueAnalyzer, last_events_ids: LastEventIds, last_invalid_batch_size: usize, + /// The event key of the last operation pushed, so an operation carrying no + /// primary event keeps its place in the queue instead of sorting to an end. + last_event_key: u64, page_limit: usize, /// Cycles since the engine last declared a batch failed. Drives only the /// give-up condition. @@ -207,12 +237,40 @@ pub struct QueueAnalyzer, } +/// How far each index's event stream has been applied. +/// +/// `None` means nothing has been applied to that stream yet, and it has to be +/// a separate value rather than a reserved id. Event ids start at **0** and +/// `IndexChangeEventId::default()` is also 0, so using the id alone made +/// "nothing applied" indistinguishable from "applied event 0". The gap check +/// had to exempt the first batch to avoid deferring on that ambiguity, and the +/// exemption is what let a first batch of ids 3..=28 be applied ahead of +/// events 0..=2 and corrupt the index file. #[derive(Debug)] pub struct LastEventIds { - pub primary_id: IndexChangeEventId, - pub secondary_ids: HashMap, + pub primary_id: Option, + pub secondary_ids: HashMap>, +} + +impl LastEventIds { + /// Whether `id` is the event this stream is waiting for. + /// + /// The first event of a stream is [`IndexChangeEventId::default`]; every + /// later one must be the immediate successor of the last applied. + pub fn follows(last: Option, id: IndexChangeEventId) -> bool { + match last { + None => id == IndexChangeEventId::default(), + Some(last) => id.is_next_for(last), + } + } } impl Default for LastEventIds @@ -232,11 +290,14 @@ where AvailableIndexes: Debug + Hash + Eq, { pub fn merge(&mut self, another: Self) { - if another.primary_id != IndexChangeEventId::default() { + // `None` is "this batch applied nothing to that stream", which must + // not move the watermark backwards. Previously the same test was + // `!= default`, which also discarded a genuine advance to event 0. + if another.primary_id.is_some() { self.primary_id = another.primary_id } for (index, id) in another.secondary_ids { - if id != IndexChangeEventId::default() || !self.secondary_ids.contains_key(&index) { + if id.is_some() || !self.secondary_ids.contains_key(&index) { self.secondary_ids.insert(index, id); } } @@ -257,20 +318,42 @@ where queue_inner_wt, last_events_ids: Default::default(), last_invalid_batch_size: 0, + last_event_key: 0, page_limit: MAX_PAGE_AMOUNT, attempts: 0, no_progress: 0, + event_ledger: Arc::new(EventLedger::detached()), } } + /// Shares the feeding queue's event bookkeeping with this analyzer. + /// + /// Only the producer side records who pushed an event, so the analyzer has + /// to read the *same* ledger the queue writes for its gap reports to mean + /// anything. + pub fn attach_event_ledger(&mut self, ledger: Arc) { + self.event_ledger = ledger; + } + pub fn push(&mut self, value: Operation) -> eyre::Result<()> { let link = value.link(); + // Where this operation sits in the primary event stream, which is the + // order a batch can actually apply. An operation that changes no + // indexed field carries no event and cannot create a gap, so it takes + // the key of the operation queued before it and stays in place rather + // than sorting to one end. + let event_key = value + .primary_key_events() + .and_then(|events| events.first()) + .map_or(self.last_event_key, |event| event.id().inner()); + self.last_event_key = event_key; let mut row = QueueInnerRow { id: self.queue_inner_wt.get_next_pk().into(), operation_id: value.operation_id(), page_id: link.page_id, link, pos: 0, + event_key, }; let pos = self.operations.push(value); row.pos = pos; @@ -301,6 +384,39 @@ where .map(|(id, _)| id) } + /// Records `stage` against every event id carried by `ops`. + /// + /// Diagnostics only. The whole body is skipped when bookkeeping is off, + /// which keeps the `Debug` formatting of secondary index labels off the + /// path of a release build entirely. + fn record_ops_stage(&self, ops: &[Operation], stage: Stages) + where + SecondaryKeys: TableSecondaryIndexEventsOps, + { + if !event_ledger::enabled() { + return; + } + for op in ops { + if let Some(evs) = op.primary_key_events() { + self.event_ledger + .record_stage_for_events(EventStream::Primary, evs, stage); + } + // `Stages::QUEUED` is unioned in for secondary streams because the + // producer side records primary ids only: an operation reaching + // the analyzer at all proves it was queued, and without this the + // secondary gap report would call every id it knows about leaked. + // The cost is that a secondary record carries no producer call + // site, which the report prints as ``. + for (index, id) in op.secondary_key_events().iter_event_ids() { + self.event_ledger.record_stage( + EventStream::Secondary(format!("{index:?}")), + id.inner(), + stage.union(Stages::QUEUED), + ); + } + } + } + pub async fn collect_batch_from_op_id( &mut self, op_id: OperationId, @@ -323,8 +439,44 @@ where } } + // Event-ordered selection. + // + // A batch can only apply a contiguous run of the event stream, so that + // is the order to select in. Grouping by page instead collected a + // page's worth of operations, let validation trim all but the + // contiguous prefix, and requeued the rest to be collected again next + // round. When the workload writes to scattered pages, which is what + // upserts against random keys do, page order and event order disagree + // and almost nothing in each collection survives the trim: measured at + // 202,000 collections and 198,000 requeues to drain 4,000 operations, + // 15.7s against 0.144s for the same count written sequentially. + // + // Validation stays the authority. Selecting in event order only makes + // the common case gapless by construction, so the trim removes nothing + // and the operation is collected once. + let mut event_ordered_ops = 0usize; + if !took_whole_queue { + let mut last_key: Option = None; + for (key, _) in self.queue_inner_wt.0.indexes.event_key_idx.iter() { + if event_ordered_ops >= MAX_BATCH_OPERATIONS { + break; + } + // The index holds one entry per row, so a multi-row operation + // repeats its key. + if last_key == Some(key) { + continue; + } + last_key = Some(key); + for row in self.queue_inner_wt.select_by_event_key(key).execute()? { + if ops_set.insert(row.operation_id) { + event_ordered_ops += 1; + } + } + } + } + let mut next_op_id = op_id; - let mut no_more_ops = took_whole_queue; + let mut no_more_ops = took_whole_queue || event_ordered_ops > 0; while used_page_ids.len() < self.page_limit && !no_more_ops { let ops_rows = self.queue_inner_wt.select_by_operation_id(next_op_id).execute()?; match next_op_id { @@ -440,13 +592,26 @@ where ops.push(op); } - let mut op = BatchOperation::new(ops, info_wt); + self.record_ops_stage(&ops, Stages::COLLECTED); + let mut op = BatchOperation::new(ops, info_wt).with_event_ledger(self.event_ledger.clone()); let invalid_for_this_batch_ops = op.validate(&self.last_events_ids, self.attempts).await?; if let Some(invalid_for_this_batch_ops) = invalid_for_this_batch_ops { + self.record_ops_stage(&invalid_for_this_batch_ops, Stages::REQUEUED); self.extend_from_iter(invalid_for_this_batch_ops.into_iter())?; let previous_primary = self.last_events_ids.primary_id; let last_ids = op.get_last_event_ids(); let advanced = last_ids.primary_id > previous_primary; + if let Some(id) = last_ids.primary_id { + self.event_ledger.record_applied_upto(EventStream::Primary, id.inner()); + } + if event_ledger::enabled() { + for (index, id) in &last_ids.secondary_ids { + if let Some(id) = id { + self.event_ledger + .record_applied_upto(EventStream::Secondary(format!("{index:?}")), id.inner()); + } + } + } self.last_events_ids.merge(last_ids); self.last_invalid_batch_size = 0; self.page_limit = MAX_PAGE_AMOUNT; @@ -461,6 +626,7 @@ where } else { // can't collect batch for now let ops = op.ops(); + self.record_ops_stage(&ops, Stages::REQUEUED); self.attempts += 1; self.no_progress += 1; if self.last_invalid_batch_size == ops.len() { @@ -473,6 +639,22 @@ where } } + /// Whether the last `None` from collection means "wait for more operations". + /// + /// Collection returns `None` in two situations that look identical to the + /// caller. While it is still escalating it has more to try on its own: a + /// wider page limit, and then `COLLECT_WHOLE_QUEUE_AFTER_ATTEMPTS`, where + /// it stops grouping by page and takes everything queued. Only once it has + /// taken the whole queue and *still* found a hole does the missing event + /// have to arrive from somewhere else. + /// + /// `no_progress` is at least `COLLECT_WHOLE_QUEUE_AFTER_ATTEMPTS` when a + /// call takes the whole queue and is incremented again when that call + /// fails, so strictly greater is exactly "the whole queue was not enough". + fn needs_more_operations(&self) -> bool { + self.no_progress > COLLECT_WHOLE_QUEUE_AFTER_ATTEMPTS + } + pub fn len(&self) -> usize { self.queue_inner_wt.count() } @@ -480,8 +662,8 @@ where #[cfg(test)] mod lifecycle_tests { - use std::collections::HashMap; - use std::sync::atomic::{AtomicUsize, Ordering}; + use core::sync::atomic::{AtomicUsize, Ordering}; + use hashbrown::HashMap; use super::*; @@ -508,7 +690,7 @@ mod lifecycle_tests { } fn iter_event_ids(&self) -> impl Iterator { - std::iter::empty() + core::iter::empty() } fn sort(&mut self) {} @@ -598,6 +780,7 @@ mod lifecycle_tests { fn insert_operation(id: u128) -> Operation<(), u64, TestEvents> { Operation::Insert(InsertOperation { + retired_link: None, id: OperationId::Single(uuid::Uuid::from_u128(id)), pk_gen_state: (), primary_key_events: vec![], @@ -621,6 +804,7 @@ mod lifecycle_tests { length: 1, }; Operation::Insert(InsertOperation { + retired_link: None, id: OperationId::Single(uuid::Uuid::from_u128(id)), pk_gen_state: (), primary_key_events: vec![indexset::cdc::change::ChangeEvent::InsertAt { @@ -660,32 +844,33 @@ mod lifecycle_tests { async fn collection_recovers_when_event_order_and_operation_order_disagree() { let queue_inner_wt = Arc::new(QueueInnerWorkTable::default()); let mut analyzer: QueueAnalyzer<(), u64, TestEvents, TestIndex> = QueueAnalyzer::new(queue_inner_wt); - analyzer.last_events_ids.primary_id = 1.into(); - - // Collecting page 5 from operation 1 also takes operation 3, and - // advances past it. Operation 2 sits between them in operation order, - // on another page, and carries the event the stream needs next, so it - // is skipped and then the walk runs out of operations entirely. The - // page-limit growth that normally widens a stuck collection cannot - // help here: the loop ended because it ran out, not because it was - // full. + analyzer.last_events_ids.primary_id = Some(1.into()); + + // Under page grouping, collecting page 5 from operation 1 also took + // operation 3 and advanced past it. Operation 2 sits between them in + // operation order, on another page, and carries the event the stream + // needs next, so it was skipped and the walk then ran out of + // operations entirely. The page-limit growth that normally widens a + // stuck collection could not help: the loop ended because it ran out, + // not because it was full. analyzer.push(insert_operation_with_event(1, 5, 3)).unwrap(); analyzer.push(insert_operation_with_event(2, 9, 2)).unwrap(); analyzer.push(insert_operation_with_event(3, 5, 4)).unwrap(); + // Selection is event-ordered now, so the inversion costs nothing: the + // operation carrying event 2 is picked first because event 2 comes + // first, and the batch is gapless on the first attempt. The loop and + // its budget stay because what this test guards is that collection + // *recovers*, and a future change to selection order must still + // recover within the budget rather than rebuild a gapped batch. let start = OperationId::Single(uuid::Uuid::from_u128(1)); - for attempt in 0..12 { + for _ in 0..12 { if analyzer .collect_batch_from_op_id(start) .await .expect("collection must not fail the engine over an ordering it can recover from") .is_some() { - assert!( - attempt >= 1, - "the first attempt is expected to defer; progress on attempt 0 would mean \ - the inversion was not reproduced" - ); return; } } @@ -698,6 +883,7 @@ mod lifecycle_tests { fn multi_insert_operation_on(page: u32, id: u128, offset: u32, byte: u8) -> Operation<(), u64, TestEvents> { Operation::Insert(InsertOperation { + retired_link: None, id: OperationId::Multi(uuid::Uuid::from_u128(id)), pk_gen_state: (), primary_key_events: vec![], @@ -727,7 +913,7 @@ mod lifecycle_tests { .unwrap(); assert_eq!( - batch.get(&1.into()).unwrap(), + batch.get(&PageId::from(1u32)).unwrap(), &vec![ ( Link { @@ -765,6 +951,98 @@ mod lifecycle_tests { assert_eq!(batches.load(Ordering::Relaxed), 1); } + /// Draining scattered writes must stay linear in the operation count. + /// + /// A batch applies a contiguous run of the event stream. Collection used to + /// group by page instead, so when a workload wrote to scattered pages it + /// collected a page of operations, had validation trim all but the few + /// whose events happened to be contiguous, and requeued the rest to be + /// collected again. Draining 4,000 operations cost 202,000 collections and + /// 198,000 requeues. + /// + /// Scattered is not a corner case: random-key upserts are exactly this + /// shape, and sequential inserts were the only workload where page order + /// and event order agreed. Measured across the change, for 4,000 + /// operations over 40 pages: 16.104s to 0.140s, and 300 batches to 8. The + /// same count written sequentially took 0.144s both before and after, + /// which is what says this closed a gap rather than skipped work. + /// + /// 2,000 operations here, which took 3.811s before and 0.069s after. The + /// bound is loose on purpose: it is there to catch a return to quadratic, + /// not to police scheduling noise on a loaded machine. + #[tokio::test] + async fn draining_scattered_writes_stays_linear() { + const OPERATIONS: u128 = 2_000; + const PAGES: u128 = 40; + + let batches = Arc::new(AtomicUsize::new(0)); + let task = PersistenceTask::run_engine(TestEngine { + batches: batches.clone(), + events: Arc::new(ParkingMutex::new(Vec::new())), + config: TestConfig, + failure: TestFailure::None, + }); + for i in 1..=OPERATIONS { + let page = (i % PAGES) as u32 + 1; + task.apply_operation(insert_operation_with_event(i, page, (i - 1) as u64)) + .unwrap(); + } + + let draining = std::time::Instant::now(); + task.close().await.unwrap(); + let elapsed = draining.elapsed(); + + assert!(batches.load(Ordering::Relaxed) > 0, "the operations have to apply"); + assert!( + elapsed < Duration::from_millis(1_500), + "draining {OPERATIONS} scattered writes took {elapsed:?}, which is the quadratic collection returning" + ); + } + + /// Which states may wait, stated exactly. + /// + /// This pins the predicate the drain loop asks before sleeping, and it + /// measures no time at all. + /// + /// A wall-clock test stood here first, draining three operations with + /// inverted event ids and asserting under 400 ms against the 2.027s the + /// bug produced. It was deleted rather than kept, for a reason worth + /// recording: once selection became event-ordered that fixture drained on + /// the first attempt and never reached the sleep at all, so replacing the + /// guard with an unconditional sleep left it green. A test that cannot + /// fail is worse than none, because it reads like cover. Only mutation + /// made that visible. + /// + /// Event-ordered selection also means a `None` from collection now only + /// ever means a genuine wait, so the guard below is defence in depth + /// rather than load-bearing. It stays because selection order is exactly + /// the kind of thing that gets changed again. + #[test] + fn only_an_exhausted_collection_waits_for_more_operations() { + let mut analyzer: QueueAnalyzer<(), u64, TestEvents, TestIndex> = + QueueAnalyzer::new(Arc::new(QueueInnerWorkTable::default())); + + // Still escalating. Each retry widens the page limit, and the last of + // these is the one that takes the whole queue, so everything needed is + // already here and waiting only delays reaching it. + for no_progress in 0..=COLLECT_WHOLE_QUEUE_AFTER_ATTEMPTS { + analyzer.no_progress = no_progress; + assert!( + !analyzer.needs_more_operations(), + "a collection with {no_progress} failed attempts has not exhausted its own \ + escalation, so sleeping delays the fallback rather than waiting for anything" + ); + } + + // The whole queue was taken and the stream still had a hole, so the + // missing event is genuinely not here yet. + analyzer.no_progress = COLLECT_WHOLE_QUEUE_AFTER_ATTEMPTS + 1; + assert!( + analyzer.needs_more_operations(), + "once the whole queue was not enough, the missing event has to arrive from elsewhere" + ); + } + /// Regression: the blocker filter kept the operations *after* a blocking /// multi operation instead of the complete ones before it. /// @@ -791,34 +1069,33 @@ mod lifecycle_tests { .get_batch_data_op() .unwrap(); - let page_one_writes = batch.get(&1.into()).unwrap(); + // These operations carry no primary events, so nothing constrains + // their order and event-ordered selection takes both groups in one + // collection. Group A is no longer applied *instead of* group B. + // + // What must still hold is the invariant the blocker logic existed to + // protect: a multi operation is never split across batches, because + // applying half of one ships a stream whose remaining event ids never + // arrive. Assert that directly rather than asserting the particular + // split the page walk used to produce. + let page_one_writes = batch.get(&PageId::from(1u32)).expect("group A lives on page 1"); + for offset in [0u32, 8] { + assert!( + page_one_writes.iter().any(|(link, _)| link.offset == offset), + "group A must be applied whole, missing its write at offset {offset}" + ); + } + let group_b_on_page_one = page_one_writes.iter().any(|(link, _)| link.offset == 16); + let group_b_on_page_two = batch.get(&PageId::from(2u32)).is_some_and(|writes| !writes.is_empty()); assert_eq!( - page_one_writes, - &vec![ - ( - Link { - page_id: 1.into(), - offset: 0, - length: 8, - }, - vec![1; 8], - ), - ( - Link { - page_id: 1.into(), - offset: 8, - length: 8, - }, - vec![2; 8], - ), - ], - "the complete earlier group must be applied" + group_b_on_page_one, group_b_on_page_two, + "the multi operation spanning both pages must be applied whole or not at all" ); - assert!( - !batch.contains_key(&2.into()), - "the blocking group must stay queued, not be applied without its earlier events" + assert_eq!( + analyzer.len(), + 0, + "one event-ordered collection takes every queued operation" ); - assert_eq!(analyzer.len(), 2, "both rows of the blocked group remain queued"); } #[tokio::test] @@ -888,7 +1165,7 @@ mod lifecycle_tests { }); // Drive the worker to its idle poll, where it parks inside the window. - tokio::task::yield_now().await; + nagoya::yield_now().await; task.apply_operation(insert_operation(1)).unwrap(); task.close().await.unwrap(); @@ -918,7 +1195,7 @@ mod lifecycle_tests { #[tokio::test] async fn wake_landing_inside_the_pop_race_window_is_not_lost() { let lifecycle = Arc::new(PersistenceLifecycle::new()); - let mut queue = Queue::<(), u64, TestEvents>::new(lifecycle.clone()); + let mut queue = Queue::<(), u64, TestEvents>::new(lifecycle.clone(), "tests/queue"); let gate = Arc::new(PopRaceWindowGate::new()); queue.pop_race_window_gate = Some(gate.clone()); let queue = Arc::new(queue); @@ -981,41 +1258,62 @@ mod lifecycle_tests { assert!(Arc::ptr_eq(&wait_error, &intake_error)); } - #[test] - fn runtime_shutdown_is_terminal_and_rejects_later_operations() { - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build() - .unwrap(); - let task = runtime.block_on(async { - let task = PersistenceTask::run_engine(TestEngine { - batches: Arc::new(AtomicUsize::new(0)), - events: Arc::new(ParkingMutex::new(Vec::new())), - config: TestConfig, - failure: TestFailure::None, - }); - tokio::task::yield_now().await; - task + /// A worker that stops publishes a terminal state instead of leaving its + /// waiters parked, and refuses operations afterwards. + /// + /// This used to build a tokio runtime, spawn the engine onto it and drop + /// the runtime out from under the worker. That worked because + /// `tokio::spawn` picked up whatever runtime the caller happened to be on, + /// and it is no longer how the worker is scheduled: it runs on the + /// engine's own pool, so tearing down a caller's runtime leaves it + /// running. Deliberately. A flush loop that dies because its caller + /// dropped an unrelated runtime loses writes it had already accepted. + /// + /// So the shutdown under test is the one that still exists: `Drop` on an + /// idle task, with `monitor()` for the waiter that has to outlive it. + /// + /// Both terminal outcomes are accepted, because which one happens is a + /// genuine race rather than a fact about the engine. `Drop` wakes the + /// queue and then cancels; if a pool thread polls the worker inside that + /// window it sees `Closing` and closes cleanly, otherwise the cancellation + /// lands first and the completion guard reports it. Asserting one of them + /// would be asserting who won. + #[tokio::test] + async fn a_stopped_worker_is_terminal_and_rejects_later_operations() { + let task = PersistenceTask::run_engine(TestEngine { + batches: Arc::new(AtomicUsize::new(0)), + events: Arc::new(ParkingMutex::new(Vec::new())), + config: TestConfig, + failure: TestFailure::None, }); + nagoya::yield_now().await; - drop(runtime); + let monitor = task.monitor(); + let sink = task.vacuum_sink(); + drop(task); - let verifier = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let wait_error = verifier - .block_on(async { tokio::time::timeout(Duration::from_secs(1), task.wait_for_failure()).await }) - .expect("cancelled worker must notify terminal waiters") - .unwrap_err(); - assert_eq!( - wait_error.to_string(), - "persistence engine failed: persistence worker was cancelled" - ); + let outcome = tokio::time::timeout(Duration::from_secs(1), monitor.wait_for_failure()) + .await + .expect("a stopped worker must notify terminal waiters"); + + match &outcome { + Ok(()) => {} + Err(error) => assert_eq!( + error.to_string(), + "persistence engine failed: persistence worker was cancelled" + ), + } - let intake_error = task.apply_operation(insert_operation(1)).unwrap_err(); - assert!(Arc::ptr_eq(&wait_error, &intake_error)); + // Terminal either way means no further operation is accepted. The + // queue outlives the task through `vacuum_sink`, which is exactly the + // path that made this worth asserting: a push accepted here would be + // acknowledged to a caller and then never written. + let intake_error = sink + .reclaim_pages(vec![1.into()]) + .expect_err("a terminal engine must refuse operations"); + if let Err(wait_error) = &outcome { + assert!(Arc::ptr_eq(wait_error, &intake_error)); + } } /// Regression: an operation pushed while `Drop` ran was accepted, then @@ -1036,7 +1334,7 @@ mod lifecycle_tests { failure: TestFailure::None, }); // Let the worker reach its idle poll so `Drop` takes the abort path. - tokio::task::yield_now().await; + nagoya::yield_now().await; let sink = task.vacuum_sink(); drop(task); @@ -1117,16 +1415,16 @@ enum PersistenceMessage { #[cfg(test)] #[derive(Debug)] struct PopRaceWindowGate { - entered: tokio::sync::Semaphore, - proceed: tokio::sync::Semaphore, + entered: nagoya::sync::Semaphore, + proceed: nagoya::sync::Semaphore, } #[cfg(test)] impl PopRaceWindowGate { fn new() -> Self { Self { - entered: tokio::sync::Semaphore::new(0), - proceed: tokio::sync::Semaphore::new(0), + entered: nagoya::sync::Semaphore::new(0), + proceed: nagoya::sync::Semaphore::new(0), } } @@ -1134,12 +1432,15 @@ impl PopRaceWindowGate { /// blocks until [`Self::release`]. async fn pause(&self) { self.entered.add_permits(1); - self.proceed.acquire().await.expect("gate semaphore closed").forget(); + // No `expect` here any more: nagoya's semaphore has no closed state, + // so `acquire` yields the permit rather than a `Result`. The panic + // this used to carry was for a case that cannot arise. + self.proceed.acquire().await.forget(); } /// Waits until the popping task is parked inside the window. async fn wait_entered(&self) { - self.entered.acquire().await.expect("gate semaphore closed").forget(); + self.entered.acquire().await.forget(); } /// Lets the popping task run on from the window. @@ -1148,6 +1449,41 @@ impl PopRaceWindowGate { } } +/// Primary index event ids lifted off an operation before it is moved into the +/// queue, so they can be recorded once the push is known to have been accepted. +/// +/// Primary only: `Queue` is generic over the secondary event type with no bound +/// that could iterate it, and the primary stream is the one whose gap guard +/// stalls the engine. The analyzer records secondary ids at collection time, +/// where that bound does exist. +/// +/// Cheap when bookkeeping is off: `EventLedger::event_ids` returns an empty +/// `Vec`, which allocates nothing, and the rest is two `Copy` field reads. +struct QueuedEventIds { + ids: Vec, + op_id: OperationId, + op_type: OperationType, +} + +impl QueuedEventIds { + fn of( + value: &Operation, + ) -> Self { + Self { + ids: value + .primary_key_events() + .map(|evs| EventLedger::event_ids(evs.as_slice())) + .unwrap_or_default(), + op_id: value.operation_id(), + op_type: value.operation_type(), + } + } + + fn record(&self, ledger: &EventLedger, site: &'static Location<'static>) { + ledger.record_queued(EventStream::Primary, &self.ids, self.op_id, self.op_type, site); + } +} + #[derive(Debug)] pub struct Queue { // Not `lockfree::queue::Queue`: its `Removable::empty` materializes the @@ -1161,38 +1497,75 @@ pub struct Queue { // queue that still holds work. len: Arc, lifecycle: Arc, + /// Producer-side half of the event-gap bookkeeping: every operation that + /// reaches persistence passes through this queue, so an event id the + /// engine is waiting for that never appears here was assigned by the index + /// and dropped before it was queued. Shared with the analyzer, which reads + /// it when the gap guard fires. Diagnostics only, and inert unless + /// [`event_ledger::enabled`]. + event_ledger: Arc, #[cfg(test)] - pop_race_window_gate: Option>, + pop_race_window_gate: Option>, } impl Queue { - fn new(lifecycle: Arc) -> Self { + fn new(lifecycle: Arc, table_path: &str) -> Self { Self { queue: ParkingMutex::new(VecDeque::new()), notify: Notify::new(), len: Arc::new(AtomicUsize::new(0)), lifecycle, + event_ledger: Arc::new(EventLedger::new(table_path)), #[cfg(test)] pop_race_window_gate: None, } } - pub fn push(&self, value: Operation) -> PersistenceResult { - self.push_message(PersistenceMessage::Operation(value)) + /// The event bookkeeping this queue writes, for sharing with the analyzer. + pub fn event_ledger(&self) -> Arc { + self.event_ledger.clone() } - /// Enqueues a whole batch of operations under one lifecycle check, one - /// queue lock acquisition and one worker wake-up, so callers producing - /// many operations at once (`insert_many`) pay the intake overhead once - /// instead of per row. All-or-nothing: either every operation is accepted - /// or none is. - pub fn push_many( + /// Enqueues one operation, naming the producer's call site. + /// + /// The site is passed explicitly rather than taken with `#[track_caller]`, + /// because that only reaches one frame up: a wrapper that wants its own + /// caller named in a gap report has to forward a location through here. + /// through here rather than call `push` and lose it. + pub fn push_at( + &self, + value: Operation, + site: &'static Location<'static>, + ) -> PersistenceResult { + // The ids have to be lifted out before the operation is moved into the + // queue, but they are only recorded once the push is accepted: a + // refused push (the engine is closing or already failed) genuinely + // does not queue its events, and recording it as queued would hide + // exactly that leak mode. + let queued = QueuedEventIds::of(&value); + self.push_message(PersistenceMessage::Operation(value))?; + queued.record(&self.event_ledger, site); + Ok(()) + } + + /// Enqueues a whole batch under one lifecycle check, one queue lock and one + /// worker wake-up, so a caller producing many operations at once + /// (`insert_many`) pays the intake overhead once instead of per row. + /// All-or-nothing: either every operation is accepted or none is. Takes the + /// producer's call site for the same reason as [`Queue::push_at`]. + pub fn push_many_at( &self, values: Vec>, + site: &'static Location<'static>, ) -> PersistenceResult { if values.is_empty() { return Ok(()); } + let queued = if crate::persistence::event_ledger::enabled() { + values.iter().map(QueuedEventIds::of).collect::>() + } else { + Vec::new() + }; let state = self.lifecycle.state.lock(); match &*state { PersistenceState::Running => {} @@ -1205,6 +1578,11 @@ impl Queue Queue Option> { loop { let notified = self.notify.notified(); - tokio::pin!(notified); + let mut notified = core::pin::pin!(notified); // `wake()` uses `notify_waiters`, which stores no permit: only a // waiter that already exists observes it. Register this waiter // before draining the queue and reading the lifecycle state, so a @@ -1305,17 +1683,27 @@ where fn apply_move( &self, bytes: Vec, + old_link: Link, new_link: Link, primary_key_events: Vec>>, secondary_keys_events: SecondaryKeys, ) -> PersistenceResult { - self.push(Operation::Update(UpdateOperation { - id: OperationId::Single(uuid::Uuid::now_v7()), - primary_key_events, - secondary_keys_events, - bytes, - link: new_link, - })) + // `Location::caller()` without `#[track_caller]` resolves to this line + // rather than to vacuum's call site. That is deliberate: the trait + // declaration lives outside this module and cannot be annotated, and + // this line is already a unique producer label, because `apply_move` + // is only ever reached from a vacuum row move. + self.push_at( + Operation::Update(UpdateOperation { + retired_link: Some(old_link), + id: OperationId::Single(uuid::Uuid::now_v7()), + primary_key_events, + secondary_keys_events, + bytes, + link: new_link, + }), + Location::caller(), + ) } fn reclaim_pages(&self, page_ids: Vec) -> PersistenceResult { @@ -1353,11 +1741,10 @@ impl Drop /// `close()` lifecycle (drain, join, surface terminal errors) is the /// long-term replacement for this heuristic. fn drop(&mut self) { - let Some(handle) = self.engine_task_handle.as_ref() else { - return; - }; - if handle.is_finished() { - return; + match self.engine_task_handle.as_ref() { + None => return, + Some(handle) if handle.is_finished() => return, + Some(_) => {} } if matches!( self.lifecycle.state(), @@ -1379,7 +1766,12 @@ impl Drop } self.queue.wake(); if self.check_wait_triggers() { - handle.abort(); + // `cancel` consumes the handle, where `abort` took `&self`. Taking + // the field is the whole difference, and it is safe here because + // this task is being dropped and nothing reads the handle again. + if let Some(handle) = self.engine_task_handle.take() { + handle.cancel(); + } } else { tracing::error!( "PersistenceTask dropped with work in flight; the engine task keeps draining detached and then stops, but its errors can no longer be observed. Call close() (or wait_for_ops() before dropping) to guarantee a clean shutdown." @@ -1391,17 +1783,21 @@ impl Drop impl PersistenceTask { + /// `#[track_caller]` so an event-gap report names the producer that + /// pushed the operation rather than this forwarding line. + #[track_caller] pub fn apply_operation(&self, op: Operation) -> PersistenceResult { - self.queue.push(op) + self.queue.push_at(op, Location::caller()) } /// Enqueues a batch of operations atomically with a single worker /// wake-up. See [`Queue::push_many`]. + #[track_caller] pub fn apply_operations( &self, ops: Vec>, ) -> PersistenceResult { - self.queue.push_many(ops) + self.queue.push_many_at(ops, Location::caller()) } pub fn ensure_running(&self) -> PersistenceResult { @@ -1417,14 +1813,15 @@ impl /// This is intentionally separate from `VacuumStats`: online vacuum makes /// freed pages durably reusable, but does not truncate `.wt.data`. /// Operators can sample this value to observe physical growth and reuse. - pub async fn persisted_data_file_size_bytes(&self) -> std::io::Result { - tokio::fs::metadata(format!( + pub async fn persisted_data_file_size_bytes(&self) -> Result { + // `metadata` answers with the length, which is the only thing anything + // here ever asked a metadata handle for. + crate::fsx::metadata(format!( "{}/{}", self.table_path.trim_end_matches('/'), WT_DATA_EXTENSION )) .await - .map(|metadata| metadata.len()) } /// Returns a sink that lets vacuum queue persistence operations for row @@ -1448,12 +1845,15 @@ impl { let table_path = engine.config().table_path().to_owned(); let lifecycle = Arc::new(PersistenceLifecycle::new()); - let queue = Arc::new(Queue::new(lifecycle.clone())); + let queue = Arc::new(Queue::new(lifecycle.clone(), &table_path)); let engine_queue = queue.clone(); let engine_lifecycle = lifecycle.clone(); let analyzer_inner_wt: Arc = Default::default(); let mut analyzer = QueueAnalyzer::new(analyzer_inner_wt.clone()); + // Producer and consumer must share one ledger: the queue records who + // pushed an event, the analyzer's gap guard reads it back. + analyzer.attach_event_ledger(queue.event_ledger()); let analyzer_in_progress = Arc::new(AtomicBool::new(true)); let task_analyzer_in_progress = analyzer_in_progress.clone(); @@ -1476,7 +1876,7 @@ impl // luck; this yield holds it open. Unit tests only, and it // changes scheduling rather than behaviour. #[cfg(test)] - tokio::task::yield_now().await; + nagoya::yield_now().await; if matches!(engine_lifecycle.state(), PersistenceState::Closing) { // Re-check the queue before giving up on it. An // operation can be enqueued between the poll above and @@ -1545,8 +1945,14 @@ impl engine_lifecycle.fail(e); return; } - } else { - tokio::time::sleep(Duration::from_millis(500)).await; + } else if analyzer.needs_more_operations() { + // Only here is waiting the right thing: collection has + // already taken the whole queue and the stream still + // has a hole, so the event it needs is not yet queued. + // Sleeping on the escalating retries instead charged + // 500 ms for each step towards the fallback that fixes + // them, which is how a 0.03s drain became 290s. + nagoya::sleep(Duration::from_millis(500)).await; } } else if let Some(page_ids) = pending_reclaim.take() { // `get_first_op_id_available() == None` is only sufficient @@ -1571,11 +1977,21 @@ impl // Constructed outside the async block so cancellation before its first // poll still drops the guard and publishes terminal failure. let completion_guard = WorkerCompletionGuard::new(lifecycle.clone()); + // HostFile performs blocking I/O. Do not occupy a compute-pool worker + // with writes or fsync. Only this persistence task uses the private pool. + let engine_runtime = nagoya::runtime::Runtime::new(1); + let weak_pool = Arc::downgrade(engine_runtime.pool()); + let stop_pool = StopWorkerPool(Some(move || { + if let Some(pool) = weak_pool.upgrade() { + pool.shut_down(); + } + })); let task = async move { + let _stop_pool = stop_pool; worker.await; completion_guard.disarm(); }; - let engine_task_handle = tokio::spawn(task); + let engine_task_handle = engine_runtime.spawn(task); Self { queue, engine_task_handle: Some(engine_task_handle), @@ -1635,10 +2051,10 @@ impl tracing::info!("Waiting for {} operations", count); } - tokio::select! { - _ = self.lifecycle.progress_notify.notified() => {}, - _ = tokio::time::sleep(Duration::from_secs(1)) => {} - } + // A `tokio::select!` racing the notify against a sleep, which is + // what a timeout is. The second arm exists so a wake lost to a + // race still gets re-checked, not to measure anything. + let _ = nagoya::timeout(Duration::from_secs(1), self.lifecycle.progress_notify.notified()).await; } } @@ -1661,12 +2077,17 @@ impl let begin_result = self.lifecycle.begin_close(); self.queue.wake(); + // `None` is cancellation. Note what this no longer catches: tokio's + // `JoinError` also reported a *panic* in the worker, and nagoya's + // handle does not, because the panic propagates out of the await + // instead. A panicking worker therefore unwinds through this call + // rather than arriving as a terminal error. if let Some(handle) = self.engine_task_handle.take() - && let Err(error) = handle.await + && handle.await.is_none() { return Err(self .lifecycle - .fail(eyre::eyre!("persistence engine task failed to join: {error}"))); + .fail(eyre::eyre!("persistence engine task was cancelled before it closed"))); } match self.lifecycle.state() { diff --git a/src/primary_key.rs b/src/primary_key.rs index ff73bbec..ab4469bd 100644 --- a/src/primary_key.rs +++ b/src/primary_key.rs @@ -1,4 +1,4 @@ -use std::sync::atomic::{ +use core::sync::atomic::{ AtomicI8, AtomicI16, AtomicI32, AtomicI64, AtomicU8, AtomicU16, AtomicU32, AtomicU64, Ordering, }; @@ -21,7 +21,7 @@ pub trait PrimaryKeyGeneratorRange { /// /// Concurrent `reserve` and [`PrimaryKeyGenerator::next`] calls never /// observe overlapping keys. - fn reserve(&self, count: usize) -> std::ops::Range; + fn reserve(&self, count: usize) -> core::ops::Range; } pub trait PrimaryKeyGeneratorState { @@ -52,7 +52,7 @@ macro_rules! atomic_primary_key { } impl PrimaryKeyGeneratorRange<$ty> for $atomic_ty { - fn reserve(&self, count: usize) -> std::ops::Range<$ty> { + fn reserve(&self, count: usize) -> core::ops::Range<$ty> { let count = <$ty>::try_from(count).unwrap_or_else(|_| { panic!( "autoincrement primary key space exhausted: cannot reserve {count} {} keys", @@ -142,7 +142,7 @@ mod tests { #[test] fn concurrent_reservations_never_overlap() { - use std::sync::Arc; + use alloc::sync::Arc; let generator = Arc::new(AtomicU64::from_state(0)); let mut handles = vec![]; diff --git a/src/runtime/dispatch.rs b/src/runtime/dispatch.rs new file mode 100644 index 00000000..9e281cb7 --- /dev/null +++ b/src/runtime/dispatch.rs @@ -0,0 +1,73 @@ +//! Owned query dispatch without borrowed tasks or blocking a pool worker. + +use super::{Profile, Runtime, RuntimeJoinHandle}; +use crate::WorkTableError; +use alloc::{boxed::Box, sync::Arc}; +use core::{ + future::{Future, poll_fn}, + pin::Pin, +}; +use parking_lot::Mutex; + +/// A type-erased submission function retained by a select plan. +pub type Dispatch = fn(Box) -> Pin> + Send>>; + +struct CancelOnDrop(Option>); +impl Drop for CancelOnDrop { + fn drop(&mut self) { + if let Some(handle) = self.0.take() { + handle.cancel(); + } + } +} + +/// Await an owned task. Dropping the wait cancels the task at its next suspension. +/// Synchronous work already running is allowed to finish. +pub async fn run_on(future: F) -> Result +where + R: Runtime, + F: Future + Send + 'static, + F::Output: Send + 'static, + R::JoinHandle: Unpin, +{ + let mut guard = CancelOnDrop::(Some(R::spawn(future))); + let result = poll_fn(|cx| Pin::new(guard.0.as_mut().expect("task present until completion")).poll(cx)).await; + guard.0.take(); + result.ok_or(WorkTableError::RuntimeCancelled) +} + +/// Dispatch through a named profile, preserving its existing backend identity. +pub async fn run_profile(future: F) -> Result +where + P: Profile, + F: Future + Send + 'static, + F::Output: Send + 'static, + ::JoinHandle: Unpin, +{ + run_on::(future).await +} + +/// The standard dispatcher for a concrete backend. +pub fn dispatcher(work: Box) -> Pin> + Send>> +where + R: Runtime, + R::JoinHandle<()>: Unpin, +{ + Box::pin(run_on::(async move { + work(); + })) +} + +/// Execute owned CPU work through a saved profile dispatcher. +pub async fn run_owned( + dispatch: Dispatch, + work: impl FnOnce() -> T + Send + 'static, +) -> Result { + let result = Arc::new(Mutex::new(None)); + let output = result.clone(); + dispatch(Box::new(move || { + *output.lock() = Some(work()); + })) + .await?; + result.lock().take().ok_or(WorkTableError::RuntimeCancelled) +} diff --git a/src/runtime/flavor.rs b/src/runtime/flavor.rs new file mode 100644 index 00000000..26085727 --- /dev/null +++ b/src/runtime/flavor.rs @@ -0,0 +1,537 @@ +//! The WorkTable runtime registry: every pool a table can dispatch to. +//! +//! # One table, one byte, one spelling +//! +//! [`Flavor`] is the definition the `runtime:` DSL key, the +//! `WT_DEFAULT_RUNTIME` environment variable, the results tables and the docs +//! all refer to. Adding a flavor means adding a row to the enum, an arm to +//! [`Flavor::tuning`] and an arm to [`Flavor::from_name`], and nothing else. +//! +//! # Why a byte, and not a type parameter +//! +//! The hot path reads it. `spawn` resolves a flavor to a pool on every call, +//! so the representation of a flavor is a cost the engine pays per spawned +//! task. One `#[repr(u8)]` discriminant makes the comparison a single `cmp` +//! and the pool lookup an array index; see [`crate::runtime::NagoyaRt`] for +//! what it replaced, which was a process-wide mutex and a linear scan over +//! `Tuning` structs compared field by field. +//! +//! A type parameter cannot do the job on its own regardless. +//! `WT_DEFAULT_RUNTIME` is read at run time, so the selection has to exist as +//! a value; a generic on top of that would be a second mechanism for the same +//! thing. The type parameter stays because a schema names its flavor at +//! compile time and `F::FLAVOR` then folds to a constant, but the value is +//! what everything downstream carries. +//! +//! # The discriminants are stable +//! +//! They appear in results tables and in `WT_DEFAULT_RUNTIME`. Renumbering +//! them silently rewrites history, so they are written out rather than left +//! to the compiler. + +use crate::runtime::Tuning; + +/// Every runtime WorkTable can dispatch to. +/// +/// One byte, `Copy`, so a comparison is a single `cmp` and a lookup is an +/// array index. +/// +/// | # | flavor | spelling | what it does | +/// |---|---|---|---| +/// | 0 | [`Locality`](Flavor::Locality) | `nagoya(locality)` | keeps a woken task on the worker that woke it | +/// | 1 | [`Spread`](Flavor::Spread) | `nagoya(spread)` | forwards every wake to the injector | +/// | 2 | [`Throughput`](Flavor::Throughput) | `nagoya(throughput)` | spread, plus a fatter injector trip | +/// | 3 | [`LowLatency`](Flavor::LowLatency) | `nagoya(low_latency)` | spins longer before parking | +/// | 4 | [`WideInjector`](Flavor::WideInjector) | `nagoya(wide_injector)` | one long intake trip, for chunky submissions | +/// | 5 | [`SharedSlot`](Flavor::SharedSlot) | `nagoya(shared_slot)` | locality with overflow sharing | +/// +/// Discriminants 6 to 9 are reserved for the flavors that need a scheduler +/// mechanism ps-st3 does not expose yet, so that adding one later does not +/// renumber the six above. See [`RESERVED`]. +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)] +pub enum Flavor { + /// The default: keep wake handoffs local with four empty search rounds + /// and 128 spin hints per round before host parking. + /// + /// This balances table throughput with CPU use between bursts. Independent + /// reads can still favor another scheduler; compare actual workloads. + #[default] + Locality = 0, + /// Send every wake to the shared injector for independent work. + Spread = 1, + /// Spread routing with an injector batch of eight. + Throughput = 2, + /// Locality routing with 512 rounds of 128 spin hints before parking. + /// This spends more CPU to keep workers responsive between arrivals. + LowLatency = 3, + /// Spread routing with an injector batch of 32. + WideInjector = 4, + /// Keep the warm slot and the first displaced inbox job private. + /// + /// Further displacement while that inbox is occupied enters the shared + /// injector, serviced and announced at local fairness boundaries. + /// This is an overflow policy, not a guarantee that only one task is private. + SharedSlot = 5, +} +/// How many flavors there are, and the length of the executor table. +pub const FLAVOR_COUNT: usize = 6; + +/// Discriminants held back for the flavors that need ps-st3 to grow a +/// mechanism first, recorded so a later release does not renumber the ones +/// above. +/// +/// | # | name | needs | +/// |---|---|---| +/// | 6 | `stealable_deque` | local wakes onto the stealable deque, not a private slot | +/// | 7 | `self_wake` | only a task's *own* wake stays local | +/// | 8 | `idle_gated` | stay local only when no worker is idle | +/// | 9 | `bounded_steal` | a cap on how many workers sweep for work at once | +/// +/// All five are value-level policy in principle and scheduler code in +/// practice, so each one is a ps-st3 release rather than a `Tuning` field this +/// crate can set. +pub const RESERVED: &[(u8, &str)] = &[ + (6, "stealable_deque"), + (7, "self_wake"), + (8, "idle_gated"), + (9, "bounded_steal"), +]; + +impl Flavor { + /// Every flavor, in discriminant order. + /// + /// The one place that enumerates them, so a `match` that gains an arm and + /// a loop that does not cannot disagree. + pub const ALL: [Flavor; FLAVOR_COUNT] = [ + Flavor::Locality, + Flavor::Spread, + Flavor::Throughput, + Flavor::LowLatency, + Flavor::WideInjector, + Flavor::SharedSlot, + ]; + + /// The spelling that selects this flavor. + /// + /// The same word in `runtime: nagoya(spread)`, in + /// `WT_DEFAULT_RUNTIME=nagoya(spread)` and in a results row, so the three + /// cannot drift apart. + #[must_use] + pub const fn name(self) -> &'static str { + match self { + Flavor::Locality => "locality", + Flavor::Spread => "spread", + Flavor::Throughput => "throughput", + Flavor::LowLatency => "low_latency", + Flavor::WideInjector => "wide_injector", + Flavor::SharedSlot => "shared_slot", + } + } + + /// The flavor a spelling selects. + /// + /// A name held back in [`RESERVED`] is rejected with what it is waiting + /// for rather than as an unknown word, because those are different + /// mistakes and want different next steps from the reader. + /// + /// # Errors + /// + /// The unrecognised name, and what was expected. + pub fn from_name(name: &str) -> Result { + use alloc::string::ToString as _; + + for flavor in Flavor::ALL { + if flavor.name() == name { + return Ok(flavor); + } + } + for (discriminant, reserved) in RESERVED { + if *reserved == name { + return Err(alloc::format!( + "nagoya flavor `{name}` is reserved as discriminant {discriminant} but not implemented: it \ + needs a scheduler mechanism ps-st3 does not expose yet" + )); + } + } + let known = Flavor::ALL.map(Flavor::name).join("`, `"); + Err(alloc::format!("unknown nagoya flavor `{name}`; expected one of `{known}`").to_string()) + } + + /// The idle policy the pool for this flavor runs with, after any + /// environment overrides. + /// + /// See [`tuning_overrides`] for the four knobs and why they exist. + #[cfg(feature = "std")] + #[must_use] + pub fn tuned(self) -> Tuning { + tuning_overrides(self.tuning()) + } + + /// The idle policy the pool for this flavor runs with. + /// + /// Built from a preset and then overridden rather than written as a + /// literal: [`Tuning`] is `#[non_exhaustive]` from ps-st3 0.6, so a field + /// it gains later is not a breaking change and this function does not have + /// to be edited again. + #[must_use] + pub fn tuning(self) -> Tuning { + match self { + Flavor::Locality => Tuning::locality(), + Flavor::Spread => Tuning::spread(), + Flavor::Throughput => Tuning::throughput(), + // Keep this explicit aggressive idle budget separate from the baseline. + Flavor::LowLatency => Tuning::locality().with_backoff_spins(128).with_rounds_before_park(512), + // Spread's wake routing, because a wide intake is pointless if a + // wake never reaches the injector to be batched with anything. + Flavor::WideInjector => Tuning::spread().with_injector_batch(32), + // Locality's routing exactly, with only the overflow policy + // changed. Changing the wake routing too would make it a second + // spelling of `spread` rather than a third point between them. + Flavor::SharedSlot => Tuning::locality().with_share_displaced(true).with_lifo_run_limit(4), + } + } +} + +impl core::fmt::Display for Flavor { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(formatter, "nagoya({})", self.name()) + } +} + +/// The process-level flavor override, or `None` if `WT_DEFAULT_RUNTIME` is +/// unset. +/// +/// Read and parsed **once**, on the first call, and never again: +/// `std::env::var` allocates and must not appear below the setup path. Every +/// later call is one acquire load and a branch. +/// +/// The variable takes the same spelling as the DSL key, with the backend +/// optional: `nagoya(spread)` and `spread` both select +/// [`Flavor::Spread`](Flavor::Spread). +/// +/// # Resolution order +/// +/// 1. a flavor passed at construction +/// 2. `WT_DEFAULT_RUNTIME` +/// 3. the table's declared `runtime:` +/// 4. [`Flavor::Locality`] +/// +/// An env override outranks the declared flavor deliberately: it is what lets +/// one benchmark binary sweep every flavor with no rebuild, which is the only +/// way to interleave arms inside a single process. A silent override is a +/// debugging trap, so anything that resolves a flavor should print what it +/// resolved. +/// +/// # Panics +/// +/// On an unparseable value. A benchmark arm that silently fell back to the +/// default is the easiest possible way to publish a wrong table, and it has +/// happened on this project already, so a typo fails loudly at startup rather +/// than quietly at the top of a results column. +#[cfg(feature = "std")] +#[inline] +pub fn env_override() -> Option { + static SELECTED: std::sync::OnceLock> = std::sync::OnceLock::new(); + *SELECTED.get_or_init(|| { + let raw = std::env::var("WT_DEFAULT_RUNTIME").ok()?; + Some(parse_selection(raw.trim()).unwrap_or_else(|error| { + panic!("WT_DEFAULT_RUNTIME={raw:?} is not a runtime selection: {error}"); + })) + }) +} + +/// `nagoya(spread)`, or the bare `spread`. +/// +/// # Errors +/// +/// What was wrong with the spelling. +#[cfg(feature = "std")] +pub fn parse_selection(source: &str) -> Result { + let inner = match source.split_once('(') { + Some((backend, rest)) => { + let backend = backend.trim(); + if backend != "nagoya" { + return Err(alloc::format!( + "`{backend}` is not a flavored backend; only `nagoya` takes a flavor" + )); + } + rest.strip_suffix(')') + .ok_or_else(|| alloc::string::String::from("missing the closing parenthesis"))? + } + None => source, + }; + Flavor::from_name(inner.trim()) +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::{FLAVOR_COUNT, Flavor, RESERVED, parse_selection}; + + #[test] + fn discriminants_are_the_ones_written_down() { + // These appear in results tables. Renumbering them rewrites history, + // so the values are asserted rather than left to the compiler. + assert_eq!(Flavor::Locality as u8, 0); + assert_eq!(Flavor::Spread as u8, 1); + assert_eq!(Flavor::Throughput as u8, 2); + assert_eq!(Flavor::LowLatency as u8, 3); + assert_eq!(Flavor::WideInjector as u8, 4); + assert_eq!(Flavor::SharedSlot as u8, 5); + } + + #[test] + fn a_flavor_is_one_byte() { + assert_eq!(size_of::(), 1); + assert_eq!( + size_of::>(), + 1, + "the niche is worth having on the hot path" + ); + } + + #[test] + fn all_is_every_variant_in_discriminant_order() { + assert_eq!(Flavor::ALL.len(), FLAVOR_COUNT); + for (index, flavor) in Flavor::ALL.into_iter().enumerate() { + assert_eq!( + flavor as usize, index, + "{flavor} is out of order, so the array lookup would miss" + ); + } + } + + #[test] + fn every_name_round_trips() { + for flavor in Flavor::ALL { + assert_eq!(Flavor::from_name(flavor.name()).unwrap(), flavor); + } + } + + #[test] + fn names_are_distinct() { + let mut names = Flavor::ALL.map(Flavor::name).to_vec(); + names.sort_unstable(); + let before = names.len(); + names.dedup(); + assert_eq!(names.len(), before, "two flavors share a spelling"); + } + + #[test] + fn a_reserved_name_says_what_it_is_waiting_for() { + for (_, reserved) in RESERVED { + let error = Flavor::from_name(reserved).unwrap_err(); + assert!(error.contains("reserved"), "{error}"); + assert!(error.contains("ps-st3"), "{error}"); + } + } + + #[test] + fn a_reserved_discriminant_is_not_in_use() { + for (discriminant, _) in RESERVED { + assert!( + Flavor::ALL.iter().all(|flavor| *flavor as u8 != *discriminant), + "discriminant {discriminant} is both reserved and in use" + ); + } + } + + #[test] + fn an_unknown_name_lists_what_would_have_worked() { + let error = Flavor::from_name("banana").unwrap_err(); + for flavor in Flavor::ALL { + assert!(error.contains(flavor.name()), "{} missing from: {error}", flavor.name()); + } + } + + #[test] + fn the_env_spelling_is_the_dsl_spelling() { + for flavor in Flavor::ALL { + assert_eq!( + parse_selection(&alloc::format!("nagoya({})", flavor.name())).unwrap(), + flavor + ); + assert_eq!(parse_selection(flavor.name()).unwrap(), flavor); + } + } + + #[test] + fn the_env_spelling_tolerates_whitespace() { + assert_eq!(parse_selection(" nagoya( spread ) ".trim()).unwrap(), Flavor::Spread); + } + + #[test] + fn a_non_nagoya_backend_is_rejected_as_one() { + let error = parse_selection("tokio(spread)").unwrap_err(); + assert!(error.contains("only `nagoya` takes a flavor"), "{error}"); + } + + #[test] + fn an_unclosed_parenthesis_says_so() { + let error = parse_selection("nagoya(spread").unwrap_err(); + assert!(error.contains("closing parenthesis"), "{error}"); + } + + /// Keep the selected baseline and its idle budget aligned with the DSL. + #[test] + fn the_default_uses_locality_with_a_short_idle_budget() { + assert_eq!(Flavor::default(), Flavor::Locality); + assert_eq!(Flavor::default().tuning().rounds_before_park, 4); + assert_eq!(Flavor::default().tuning().backoff_spins, 128); + } + + #[test] + fn display_is_the_dsl_form() { + assert_eq!(Flavor::Spread.to_string(), "nagoya(spread)"); + } + + /// Each flavor has to select a genuinely different pool, or the sweep is + /// measuring the same executor under several names. This is the check + /// that would have caught an arm that fell through to a preset. + #[test] + fn no_two_flavors_share_a_tuning() { + for (index, flavor) in Flavor::ALL.into_iter().enumerate() { + for other in Flavor::ALL.into_iter().skip(index + 1) { + assert_ne!( + flavor.tuning(), + other.tuning(), + "{flavor} and {other} are the same pool under two names" + ); + } + } + } + + /// LowLatency deliberately spends a larger idle CPU budget than the baseline. + #[test] + fn low_latency_preserves_its_explicit_aggressive_idle_budget() { + let base = Flavor::Locality.tuning(); + let fast = Flavor::LowLatency.tuning(); + assert_eq!(fast.backoff_spins, 128); + assert_eq!(fast.rounds_before_park, 512); + assert!(fast.rounds_before_park > base.rounds_before_park); + assert_eq!(fast.local_wakes, base.local_wakes); + assert_eq!(fast.injector_batch, base.injector_batch); + } + + /// Both of `shared_slot`'s changes are about the same thing: letting go of + /// work a worker cannot run soon. Neither touches the wake routing, which + /// is what would make it a second spelling of `spread`. + #[test] + fn shared_slot_changes_only_how_a_worker_lets_go() { + let base = Flavor::Locality.tuning(); + let shared = Flavor::SharedSlot.tuning(); + assert!(shared.share_displaced); + assert!(!base.share_displaced, "locality keeps its overflow private"); + assert!( + shared.lifo_run_limit < base.lifo_run_limit, + "it has to reach its own queue sooner, or the work it let go of is never promoted" + ); + assert_eq!(shared.local_wakes, base.local_wakes); + assert_eq!(shared.backoff_spins, base.backoff_spins); + assert_eq!(shared.injector_batch, base.injector_batch); + } + + #[test] + fn wide_injector_changes_only_the_intake() { + let base = Flavor::Spread.tuning(); + let wide = Flavor::WideInjector.tuning(); + assert_eq!(wide.injector_batch, 32); + assert_eq!(wide.local_wakes, base.local_wakes); + assert_eq!(wide.backoff_spins, base.backoff_spins); + } +} + +/// The four free parameters, overridden per process. +/// +/// # Why these are knobs and not flavors +/// +/// A flavor is a named point somebody has measured and can recommend. These +/// are the axes those points sit on, and sweeping an axis is how a new point +/// gets found. Turning every value of every axis into a flavor would be four +/// nested loops of names nobody chose. +/// +/// So they exist for the sweep, they are read once each, and a run that sets +/// one is not running a flavor any more: it is running an unnamed tuning that +/// happens to start from one. Anything reporting a number from such a run has +/// to say so, which is why [`describe_tuning`] exists. +/// +/// | variable | field | default | +/// |---|---|---| +/// | `WT_ROUNDS` | `rounds_before_park` | 64 | +/// | `WT_BACKOFF` | `backoff_spins` | 1024, or 128 under `low_latency` | +/// | `WT_PROMOTE` | `promote_every` | 64 | +/// | `WT_BATCH` | `injector_batch` | per flavor | +/// | `WT_LIFO` | `lifo_run_limit` | 32 | +/// +/// The names match the `ROUNDS` / `BACKOFF` / `PROMOTE` / `BATCH` variables +/// the `perf-benchmarks` examples already take, prefixed so they cannot +/// collide with a host's own environment. +#[cfg(feature = "std")] +#[must_use] +pub fn tuning_overrides(base: Tuning) -> Tuning { + fn read(name: &str) -> Option { + std::env::var(name).ok()?.trim().parse().ok() + } + /// The five knobs, each present only if its variable was set. + /// + /// A struct rather than a tuple so the fields are named at the one place + /// that reads them; getting `promote` and `batch` the wrong way round in a + /// destructuring would be silent and would show up as a tuning nobody + /// asked for. + struct Overrides { + rounds: Option, + backoff: Option, + promote: Option, + batch: Option, + lifo: Option, + } + + static OVERRIDES: std::sync::OnceLock = std::sync::OnceLock::new(); + let overrides = OVERRIDES.get_or_init(|| Overrides { + rounds: read("WT_ROUNDS"), + backoff: read("WT_BACKOFF"), + promote: read("WT_PROMOTE"), + batch: read("WT_BATCH"), + lifo: read("WT_LIFO"), + }); + + let mut tuning = base; + if let Some(rounds) = overrides.rounds { + tuning = tuning.with_rounds_before_park(rounds); + } + if let Some(backoff) = overrides.backoff { + tuning = tuning.with_backoff_spins(backoff); + } + if let Some(promote) = overrides.promote { + tuning = tuning.with_promote_every(promote); + } + if let Some(batch) = overrides.batch { + tuning = tuning.with_injector_batch(batch); + } + if let Some(lifo) = overrides.lifo { + tuning = tuning.with_lifo_run_limit(lifo); + } + tuning +} + +/// One line naming the pool a run actually used. +/// +/// A results row that says only `spread` when `WT_BACKOFF` was set describes a +/// tuning nobody can reproduce from the flavor name, so this prints the fields +/// rather than the label. +#[cfg(feature = "std")] +#[must_use] +pub fn describe_tuning(flavor: Flavor) -> alloc::string::String { + let tuning = flavor.tuned(); + alloc::format!( + "nagoya({}) rounds={} backoff={} promote={} batch={} lifo={} local_wakes={} share_displaced={}", + flavor.name(), + tuning.rounds_before_park, + tuning.backoff_spins, + tuning.promote_every, + tuning.injector_batch, + tuning.lifo_run_limit, + tuning.local_wakes, + tuning.share_displaced, + ) +} diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs new file mode 100644 index 00000000..d479cdce --- /dev/null +++ b/src/runtime/mod.rs @@ -0,0 +1,233 @@ +//! Runtime traits, backend adapters and owned query dispatch. +//! +//! Hosted generated selects use the declared backend for owned execution. +//! Runtime-annotated mutations dispatch owned Arc table handles. Ordinary +//! borrowed operations retain caller execution and portable Nagoya locks. +//! Persistence I/O has its own worker pool; schema selection does not replace it. +//! +//! The primitive traits expose backend integration to callers. Their existence +//! does not make every table primitive generic over a backend. Hosted backends +//! require std; trait definitions and the inline owned select path remain +//! available without default features. + +use alloc::sync::Arc; +use core::future::Future; +use core::ops::{Deref, DerefMut}; +use core::pin::Pin; +use core::time::Duration; + +/// What a [`Runtime::timeout`] returns when the future did not finish in time. +/// +/// Normalised on nagoya's, which is a unit struct. See [`TokioRt`] for what +/// that costs the other impl. +pub use nagoya::Elapsed; +/// The idle policy a nagoya pool runs with. +/// +/// Through nagoya rather than from `ps-st3` directly. Selecting an idle +/// policy is a nagoya-level decision, and naming the type through the crate +/// that owns that decision is what lets this crate stop depending on the +/// queues underneath it for one struct. +pub use nagoya::Tuning; + +/// The backends themselves need `std`, because the only reason a backend +/// exists is to spawn and spawning needs threads. The trait, the flavor +/// markers' contract and the profile machinery do not, so they stay available +/// to a `no_std` build: a table that never spawns still names its runtime in +/// types that have to resolve. +#[cfg(feature = "std")] +mod nagoya_rt; + +mod dispatch; +mod flavor; +mod profile; +pub use dispatch::{Dispatch, dispatcher, run_on, run_owned, run_profile}; +#[cfg(all(feature = "std", feature = "tokio-runtime"))] +mod tokio_rt; + +pub use flavor::{FLAVOR_COUNT, Flavor, RESERVED}; +#[cfg(feature = "std")] +pub use flavor::{describe_tuning, env_override, parse_selection, tuning_overrides}; +#[cfg(feature = "std")] +pub use nagoya_rt::{ + Locality, LowLatency, NagoyaRt, SharedSlot, Spread, Throughput, WideInjector, engine_executor, engine_flavor, + executor_for_flavor, +}; + +pub use profile::{Profile, RuntimeCompatibleWith, RuntimeUnpinned, TableRuntime}; +#[cfg(all(feature = "std", feature = "tokio-runtime"))] +pub use tokio_rt::{TokioJoinHandle, TokioRt}; + +#[cfg(all(test, feature = "std"))] +mod tests; + +/// An async runtime, named by a table rather than assumed. +/// +/// Every method is associated rather than taken on `&self`, because a backend +/// is a type in a schema and never a value anyone holds. The associated types +/// carry their own helper trait, since an associated type with no bound is a +/// type nothing can be called on. +pub trait Runtime: Send + Sync + 'static { + /// The async reader-writer lock guarding one row. + type RwLock: RuntimeRwLock; + /// The wake primitive the persistence worker and the vacuum share. + type Notify: RuntimeNotify; + /// The counting gate the persistence tests step the worker with. + type Semaphore: RuntimeSemaphore; + /// A handle to a spawned task. + type JoinHandle: RuntimeJoinHandle; + + /// Run `future` on this runtime's threads. + fn spawn(future: F) -> Self::JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static; + + /// A future that is ready once `duration` has passed. + fn sleep(duration: Duration) -> impl Future + Send; + + /// Run `future`, giving up after `duration`. + fn timeout(duration: Duration, future: F) -> impl Future> + Send + where + F: Future + Send; + + /// Hand the scheduler a chance to run something else. + fn yield_now() -> impl Future + Send; +} + +/// The async reader-writer lock surface this crate uses. +pub trait RuntimeRwLock: Send + Sync + 'static +where + T: Send + Sync + 'static, +{ + /// A read guard borrowed from the lock. + type ReadGuard<'a>: Deref + Send + where + Self: 'a; + /// A write guard borrowed from the lock. + type WriteGuard<'a>: DerefMut + Send + where + Self: 'a; + /// A read guard that owns its `Arc` instead of borrowing. + /// + /// Public API depends on this one: `EmptyLinkRegistry`'s `PoppedLink` is a + /// pair whose second element is an owned read guard held across awaits. + type OwnedReadGuard: Deref + Send + 'static; + + /// A lock holding `value`, unlocked. + fn new(value: T) -> Self + where + Self: Sized; + + /// Wait for exclusive access. + fn write(&self) -> impl Future> + Send; + + /// Take shared access if it is free, without waiting. + /// + /// The lock-map cleanup path probes with this while holding a synchronous + /// map guard, which is why it must not be a future. + fn try_read(&self) -> Option>; + + /// Take shared access if it is free, keeping the `Arc` alive. + fn try_read_owned(self: Arc) -> Option; +} + +/// The wake primitive this crate uses. +pub trait RuntimeNotify: Default + Send + Sync + 'static { + /// The future [`RuntimeNotify::notified`] returns. + type Notified<'a>: RuntimeNotified + where + Self: 'a; + + /// A notify with no stored permit. + fn new() -> Self + where + Self: Sized; + + /// Wake one waiter, storing a permit if there is none. + fn notify_one(&self); + + /// Wake every current waiter, storing no permit. + fn notify_waiters(&self); + + /// A future that resolves on the next notification. + fn notified(&self) -> Self::Notified<'_>; +} + +/// The future a [`RuntimeNotify`] hands out. +/// +/// `enable` is here because `notify_waiters` stores no permit: a waiter that +/// reads state before registering can lose a transition that lands between the +/// read and the first poll. Both backends spell the fix the same way. +pub trait RuntimeNotified: Future + Send { + /// Register this waiter now, and report whether a notification is already + /// waiting for it. + fn enable(self: Pin<&mut Self>) -> bool; +} + +/// The counting semaphore this crate uses. +pub trait RuntimeSemaphore: Send + Sync + 'static { + /// A held permit. + type Permit<'a>: RuntimeSemaphorePermit + where + Self: 'a; + + /// A semaphore starting with `permits` available. + fn new(permits: usize) -> Self + where + Self: Sized; + + /// Hand the semaphore `permits` more than it was created with. + fn add_permits(&self, permits: usize); + + /// Wait for a permit. + /// + /// Normalised on nagoya's shape, which has no closed state and so returns + /// the permit rather than a `Result`. See [`TokioRt`] for the adaptation. + fn acquire(&self) -> impl Future> + Send; +} + +/// A permit taken from a [`RuntimeSemaphore`]. +pub trait RuntimeSemaphorePermit { + /// Drop the permit without returning it, shrinking the semaphore by one. + fn forget(self); +} + +/// A handle to a spawned task. +/// +/// Two deltas between the backends are normalised here, both on nagoya's +/// shape. Cancellation is `cancel(self)`, not tokio's `abort(&self)`, so a +/// cancelled handle cannot be awaited afterwards. Awaiting yields `Option`, +/// not tokio's `Result`, so `None` means cancelled and a panic in +/// the task unwinds through the await rather than arriving as a value. +pub trait RuntimeJoinHandle: Future> + Send + Sized + 'static { + /// Stop the task at its next suspension point and throw away its output. + fn cancel(self); + + /// Whether the task has finished, without waiting for it. + fn is_finished(&self) -> bool; +} + +/// A nagoya pool tuning, named as a type so a schema can select one. +/// +/// The three markers below are the whole set. See [`Tuning`] for what each one +/// trades, and note that the numbers behind them were measured on one machine +/// against one workload shape. +pub trait FlavorMarker: Send + Sync + 'static { + /// Which pool this flavor selects, as one byte. + /// + /// **This is what the hot path reads.** `spawn` resolves a flavor to a + /// pool on every call, so the flavor's representation is a per-task cost. + /// Because `F` is a type parameter the discriminant is a compile-time + /// constant, the array index folds, and a warm lookup is one acquire load + /// and a branch. + const FLAVOR: Flavor; + + /// The idle policy the pool for this flavor runs with. + /// + /// Called once, to build the pool, and never on the hot path. Defaulted + /// through [`Flavor::tuning`] so the registry is the only place a flavor's + /// numbers are written down. + fn tuning() -> Tuning { + Self::FLAVOR.tuning() + } +} diff --git a/src/runtime/nagoya_rt.rs b/src/runtime/nagoya_rt.rs new file mode 100644 index 00000000..13a5d300 --- /dev/null +++ b/src/runtime/nagoya_rt.rs @@ -0,0 +1,340 @@ +//! The Nagoya backend and its six selectable pool flavors. + +use alloc::boxed::Box; +use alloc::sync::Arc; +use core::future::Future; +use core::marker::PhantomData; +use core::pin::Pin; +use core::time::Duration; +use std::sync::OnceLock; + +use nagoya::Executor; + +use super::flavor::{FLAVOR_COUNT, Flavor, env_override}; +use super::{ + Elapsed, FlavorMarker, Runtime, RuntimeJoinHandle, RuntimeNotified, RuntimeNotify, RuntimeRwLock, RuntimeSemaphore, + RuntimeSemaphorePermit, +}; + +/// Keep a woken task on the worker that woke it. +/// +/// For work whose wakes are a chain: an update path handing a row lock to its +/// successor wants the lines the releasing worker just touched. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Locality; + +/// Send every wake to the injector, where any worker can take it. +/// +/// For work whose wakes are independent, which is what read-mostly and +/// insert-mostly tables look like. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Spread; + +/// Fewer, larger trips to the injector. +/// +/// For a firehose of short independent operations submitted from outside the +/// pool, where the trip to the shared queue is the cost. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Throughput; + +/// Locality routing with a longer idle-spin budget before parking. +/// +/// Runs 512 empty search rounds of 128 spin hints rather than the default four. +/// Compare CPU use between arrivals alongside latency; see [`Flavor::LowLatency`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct LowLatency; + +/// Spread's wake routing, with one long trip to the injector. +/// +/// `injector_batch: 32`, for work submitted in chunks. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct WideInjector; + +impl FlavorMarker for Locality { + const FLAVOR: Flavor = Flavor::Locality; +} + +impl FlavorMarker for Spread { + const FLAVOR: Flavor = Flavor::Spread; +} + +impl FlavorMarker for Throughput { + const FLAVOR: Flavor = Flavor::Throughput; +} + +impl FlavorMarker for LowLatency { + const FLAVOR: Flavor = Flavor::LowLatency; +} + +impl FlavorMarker for WideInjector { + const FLAVOR: Flavor = Flavor::WideInjector; +} + +/// Locality's routing, sharing displaced work after the first private inbox job. +/// +/// See [`Flavor::SharedSlot`] for the two failure modes this sits between. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SharedSlot; + +impl FlavorMarker for SharedSlot { + const FLAVOR: Flavor = Flavor::SharedSlot; +} + +/// The nagoya backend, at one of the [`FlavorMarker`] tunings. +/// +/// This is a type-level selection and never a value: every [`Runtime`] method +/// is associated, so `NagoyaRt` appears in a signature and nowhere +/// else. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct NagoyaRt(PhantomData F>); + +/// Threads for a flavor's pool. +/// +/// The machine's parallelism, which is what `nagoya::runtime::background` uses +/// and what `tokio::spawn` gave the callers this replaces. Changing the count +/// and the tuning in one step makes any measurement of the tuning unreadable. +fn workers() -> usize { + // `WT_RUNTIME_WORKERS` overrides it, read once for the same reason + // `WT_DEFAULT_RUNTIME` is: this is on the pool-construction path, and a + // sweep over worker counts should not need a rebuild per arm. + // + // The default is the machine's parallelism, which is what + // `nagoya::runtime::background` uses and what `tokio::spawn` gave the + // callers this replaces. Changing the count and the tuning in one step + // makes any measurement of the tuning unreadable, so the count is a knob + // rather than something a flavor sets. + // + // Worth sweeping on a heterogeneous machine: `available_parallelism` + // counts efficiency cores, so on a 12P + 4E part it starts four workers + // that drain their queues substantially slower than the other twelve. + static WORKERS: OnceLock = OnceLock::new(); + *WORKERS.get_or_init(|| { + std::env::var("WT_RUNTIME_WORKERS") + .ok() + .and_then(|raw| raw.trim().parse::().ok()) + .filter(|count| *count > 0) + .unwrap_or_else(|| std::thread::available_parallelism().map_or(2, core::num::NonZeroUsize::get)) + }) +} + +/// The pool for a flavor, started on first use and shared thereafter. +/// +/// # Why an array and not a registry +/// +/// This is called from `spawn`, so it runs once per spawned task. It used to +/// build a `Tuning` struct, compare it field by field against +/// `Tuning::locality()`, and then, for anything that was not locality, take a +/// **process-wide mutex and linear-scan a `Vec` comparing `Tuning` structs by +/// value**. Locality returned before the lock and paid none of it. +/// +/// That is not a small constant, it is a serialization point, and it fell on +/// precisely the flavors that are supposed to win. Any A/B run through it +/// would have measured the incumbent running free against every challenger +/// through a contended lock, and the conclusion would have come out backwards. +/// +/// A fixed array indexed by the discriminant has no lock, no allocation and +/// nothing to compare. When the caller is `NagoyaRt` the index is a +/// compile-time constant, so a warm lookup is one acquire load and a branch, +/// and every flavor pays the same, which is the property the A/B depends on. +/// +/// Entries are leaked. There is one per flavor a process actually uses, and a +/// pool whose threads are detached has nothing useful to do with a `Drop`. +static EXECUTORS: [OnceLock<&'static Executor>; FLAVOR_COUNT] = [const { OnceLock::new() }; FLAVOR_COUNT]; + +#[inline] +fn executor_for(flavor: Flavor) -> &'static Executor { + EXECUTORS[flavor as usize].get_or_init(|| start_pool(flavor)) +} + +/// The flavor a spawn actually runs on, given the one its type names. +/// +/// `WT_DEFAULT_RUNTIME` outranks the declared flavor, which is what lets one +/// benchmark binary sweep every flavor with no rebuild. One acquire load and +/// a branch; see [`env_override`] for why the variable is read exactly once. +#[inline] +pub(crate) fn resolved(declared: Flavor) -> Flavor { + env_override().unwrap_or(declared) +} + +/// The pool the **engine's own** background work runs on. +/// +/// The persistence worker and the vacuum sweep are the whole of the engine's +/// async spawning; everything else runs inline on the caller's executor. They +/// are not generic over a runtime, so they cannot read a table's declared +/// flavor and instead take the process-level selection: `WT_DEFAULT_RUNTIME`, +/// or locality. +/// +/// Routing them matters more than their two call sites suggest. A benchmark +/// that moved only its client tasks to a flavor would leave the engine's +/// flush loop and vacuum sweep on the locality pool, so the two halves of the +/// stack would be on different schedulers contending for the same cores, and +/// the arm would describe a configuration nobody would ship. +#[must_use] +pub fn engine_executor() -> &'static Executor { + executor_for(engine_flavor()) +} + +/// The shared executor for one named flavor, started on first use. +/// +/// This Rust callsite lets a caller submit owned work to a selected pool. A +/// cross-pool submission uses the injector and may wake another worker; include +/// dispatch cost when comparing it with inline work. Generated query profiles +/// and runtime-annotated mutations expose their own owned execution contracts. +#[must_use] +pub fn executor_for_flavor(flavor: Flavor) -> &'static Executor { + executor_for(flavor) +} + +/// The flavor the engine's background work resolved to, for a benchmark to +/// print and record next to its numbers. +/// +/// An A/B where one arm silently fell back to the default is the easiest way +/// to publish a wrong table, and it has happened on this project already. +#[must_use] +pub fn engine_flavor() -> Flavor { + resolved(Flavor::default()) +} + +/// A pool at this flavor's tuning, with its threads marked as pool workers. +/// +/// # Why every flavor gets its own pool, including the default +/// +/// It would be cheaper for locality to take `nagoya::runtime::background()`, +/// which is already at that tuning, and that is what this did. It is also +/// what made a flavor comparison unreadable: the shared pool is started by +/// nagoya, which sizes it from `available_parallelism`, while every other +/// flavor got a pool started here. Two arms that differ in who started the +/// threads are not two tunings, they are two configurations, and the tuning +/// is only one of the differences between them. +/// +/// Building all of them the same way costs one extra pool in a process that +/// also calls `nagoya::spawn` directly, and buys arms that differ in exactly +/// the thing being measured. +fn start_pool(flavor: Flavor) -> &'static Executor { + // `Runtime::with_tuning` rather than a hand-built `Pool`, and this is the + // whole reason nagoya grew that constructor. `nagoya::task::mark_current` + // is private, and that marker is the only thing that makes `local_wakes` + // do anything: without it a wake takes the injector whatever the tuning + // says. A pool built here by hand therefore ran every locality-flavored + // tuning as if it were spread, silently, which is how `low_latency` would + // have been measured as a backoff change with its routing quietly + // disabled. + let runtime = Box::leak(Box::new(nagoya::runtime::Runtime::with_tuning( + workers(), + flavor.tuned(), + "wt", + ))); + runtime.executor() +} + +impl Runtime for NagoyaRt { + type RwLock = nagoya::sync::RwLock; + type Notify = nagoya::sync::Notify; + type Semaphore = nagoya::sync::Semaphore; + type JoinHandle = nagoya::JoinHandle; + + fn spawn(future: Fut) -> Self::JoinHandle + where + Fut: Future + Send + 'static, + Fut::Output: Send + 'static, + { + executor_for(resolved(F::FLAVOR)).spawn(future) + } + + fn sleep(duration: Duration) -> impl Future + Send { + nagoya::sleep(duration) + } + + fn timeout(duration: Duration, future: Fut) -> impl Future> + Send + where + Fut: Future + Send, + { + nagoya::timeout(duration, future) + } + + fn yield_now() -> impl Future + Send { + nagoya::yield_now() + } +} + +impl RuntimeRwLock for nagoya::sync::RwLock { + type ReadGuard<'a> = nagoya::sync::RwLockReadGuard<'a, T>; + type WriteGuard<'a> = nagoya::sync::RwLockWriteGuard<'a, T>; + type OwnedReadGuard = nagoya::sync::OwnedRwLockReadGuard; + + fn new(value: T) -> Self { + nagoya::sync::RwLock::new(value) + } + + fn write(&self) -> impl Future> + Send { + nagoya::sync::RwLock::write(self) + } + + fn try_read(&self) -> Option> { + nagoya::sync::RwLock::try_read(self) + } + + fn try_read_owned(self: Arc) -> Option { + nagoya::sync::RwLock::try_read_owned(self) + } +} + +impl RuntimeNotify for nagoya::sync::Notify { + type Notified<'a> = nagoya::sync::Notified<'a>; + + fn new() -> Self { + nagoya::sync::Notify::new() + } + + fn notify_one(&self) { + nagoya::sync::Notify::notify_one(self); + } + + fn notify_waiters(&self) { + nagoya::sync::Notify::notify_waiters(self); + } + + fn notified(&self) -> Self::Notified<'_> { + nagoya::sync::Notify::notified(self) + } +} + +impl RuntimeNotified for nagoya::sync::Notified<'_> { + fn enable(self: Pin<&mut Self>) -> bool { + nagoya::sync::Notified::enable(self) + } +} + +impl RuntimeSemaphore for nagoya::sync::Semaphore { + type Permit<'a> = nagoya::sync::SemaphorePermit<'a>; + + fn new(permits: usize) -> Self { + nagoya::sync::Semaphore::new(permits) + } + + fn add_permits(&self, permits: usize) { + nagoya::sync::Semaphore::add_permits(self, permits); + } + + fn acquire(&self) -> impl Future> + Send { + nagoya::sync::Semaphore::acquire(self) + } +} + +impl RuntimeSemaphorePermit for nagoya::sync::SemaphorePermit<'_> { + fn forget(self) { + nagoya::sync::SemaphorePermit::forget(self); + } +} + +impl RuntimeJoinHandle for nagoya::JoinHandle { + fn cancel(self) { + nagoya::JoinHandle::cancel(self); + } + + fn is_finished(&self) -> bool { + nagoya::JoinHandle::is_finished(self) + } +} + +impl super::RuntimeCompatibleWith> for NagoyaRt {} diff --git a/src/runtime/profile.rs b/src/runtime/profile.rs new file mode 100644 index 00000000..00f2a810 --- /dev/null +++ b/src/runtime/profile.rs @@ -0,0 +1,51 @@ +//! Named profiles used by owned query execution. +use crate::runtime::{Runtime, Tuning}; + +/// A named executor identity. Its backend family must match the table declaration; +/// a Nagoya profile may select a different flavor at a callsite. It selects owned task submission, not the +/// table's portable lock implementation or its private persistence I/O pool. +/// Profiles emitted by `runtimes!` describe their backend with `tuning()`; +/// overriding tuning metadata alone does not reconfigure that backend. +pub trait Profile: 'static { + /// The runtime this profile runs on. Must be compatible with the table backend. + type Backend: Runtime; + + /// The pool settings this profile asks for. + fn tuning() -> Tuning; + + /// Submission used by owned asynchronous select execution. + fn dispatcher() -> super::Dispatch + where + ::JoinHandle<()>: Unpin, + { + super::dispatcher:: + } +} + +/// Executor identity carried by generated hosted paged row types. +#[diagnostic::on_unimplemented( + message = "{Self} has no hosted WorkTable runtime", + label = "runtime selection needs a generated paged row with the std feature" +)] +pub trait TableRuntime { + /// The declared backend and flavor, defaulting to Nagoya locality. + type Backend: Runtime; +} + +/// A row whose select builders admit an explicit matching profile. +/// Generated hosted paged rows implement this. Mutation section annotations +/// govern their own methods, not selects, and therefore do not suppress it. +#[diagnostic::on_unimplemented( + message = "{Self} does not permit select runtime selection", + label = "this row must implement RuntimeUnpinned" +)] +pub trait RuntimeUnpinned {} + +/// Backend compatibility for query profiles. Built-in Nagoya flavors can be +/// mixed at a callsite; Tokio profiles require a Tokio table. This selects +/// scheduling policy without changing portable table lock implementations. +#[diagnostic::on_unimplemented( + message = "profile runtime {Self} is incompatible with table runtime {Other}", + label = "select a profile from the same runtime backend family" +)] +pub trait RuntimeCompatibleWith: Runtime {} diff --git a/src/runtime/tests.rs b/src/runtime/tests.rs new file mode 100644 index 00000000..e41c2a9e --- /dev/null +++ b/src/runtime/tests.rs @@ -0,0 +1,167 @@ +//! One conformance body, run against every [`Runtime`] impl. +//! +//! The macro is the point. Two impls that pass different tests prove nothing +//! about a schema being able to swap them, and the integration lane reuses this +//! same macro rather than writing a third copy. + +use alloc::sync::Arc; +/// Only `tokio_block_on` names it, and that is behind the feature. +#[cfg(feature = "tokio-runtime")] +use core::future::Future; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::time::Duration; + +use super::{Runtime, RuntimeJoinHandle, RuntimeNotify, RuntimeRwLock, RuntimeSemaphore, RuntimeSemaphorePermit}; + +/// Exercises the associated types through the trait only, so a backend that +/// compiles here is one a generic call site can name. +async fn primitives() { + let lock: Arc> = Arc::new(RuntimeRwLock::new(7)); + { + let mut guard = lock.write().await; + *guard += 1; + } + let owned = lock.clone().try_read_owned().expect("nobody holds the lock"); + assert_eq!(*owned, 8); + // The probe the lock map does while holding its synchronous map guard. + assert!(lock.try_read().is_some()); + drop(owned); + + let notify = ::new(); + notify.notify_one(); + // A stored permit, so this resolves without a second task. + notify.notified().await; + notify.notify_waiters(); + + let semaphore = ::new(0); + semaphore.add_permits(1); + semaphore.acquire().await.forget(); +} + +/// A task that finishes returns its output. +async fn spawn_and_await() { + let handle = R::spawn(async { 41 + 1 }); + assert_eq!(handle.await, Some(42)); +} + +/// A cancelled task stops, and its handle is consumed rather than left +/// awaitable. +async fn cancel() { + let ran = Arc::new(AtomicBool::new(false)); + let flag = ran.clone(); + let handle = R::spawn(async move { + R::sleep(Duration::from_millis(300)).await; + flag.store(true, Ordering::Release); + }); + handle.cancel(); + R::sleep(Duration::from_millis(600)).await; + assert!(!ran.load(Ordering::Acquire), "the cancelled task ran to completion"); +} + +/// Sleeping waits at least as long as it was asked to. +async fn sleep() { + let started = std::time::Instant::now(); + R::sleep(Duration::from_millis(50)).await; + assert!(started.elapsed() >= Duration::from_millis(50)); +} + +/// A future that never completes times out. +async fn timeout_elapses() { + let result = R::timeout(Duration::from_millis(50), core::future::pending::<()>()).await; + assert!(result.is_err()); +} + +/// A future that completes inside its budget is not punished for it. +async fn timeout_returns() { + let result = R::timeout(Duration::from_secs(30), async { 7 }).await; + assert_eq!(result.ok(), Some(7)); +} + +/// Yielding resumes. +async fn yields() { + R::yield_now().await; +} + +/// Runs every conformance body above against `$runtime`, driving each with +/// `$block_on`. +/// +/// `$block_on` is a parameter because entering a runtime is the one thing a +/// runtime cannot abstract over: nagoya has a free `block_on` and tokio needs a +/// `Runtime` value built first. +macro_rules! runtime_conformance_tests { + ($module:ident, $runtime:ty, $block_on:path) => { + mod $module { + #[test] + fn primitives() { + $block_on(super::primitives::<$runtime>()); + } + + #[test] + fn spawn_and_await() { + $block_on(super::spawn_and_await::<$runtime>()); + } + + #[test] + fn cancel() { + $block_on(super::cancel::<$runtime>()); + } + + #[test] + fn sleep() { + $block_on(super::sleep::<$runtime>()); + } + + #[test] + fn timeout_elapses() { + $block_on(super::timeout_elapses::<$runtime>()); + } + + #[test] + fn timeout_returns() { + $block_on(super::timeout_returns::<$runtime>()); + } + + #[test] + fn yields() { + $block_on(super::yields::<$runtime>()); + } + } + }; +} + +/// Exported for the integration lane, which runs the same bodies against the +/// backend a generated table selected. Unused inside this module, which is why +/// the allow: the invocations below reach the macro directly. +#[allow(unused_imports)] +pub(crate) use runtime_conformance_tests; + +runtime_conformance_tests!( + nagoya_locality, + crate::runtime::NagoyaRt, + nagoya::block_on +); +runtime_conformance_tests!( + nagoya_spread, + crate::runtime::NagoyaRt, + nagoya::block_on +); +runtime_conformance_tests!( + nagoya_throughput, + crate::runtime::NagoyaRt, + nagoya::block_on +); + +#[cfg(feature = "tokio-runtime")] +runtime_conformance_tests!(tokio_backend, crate::runtime::TokioRt, super::tokio_block_on); + +/// Tokio has no free `block_on`: a runtime has to exist first, and `spawn` +/// needs it to be the ambient one. +#[cfg(feature = "tokio-runtime")] +fn tokio_block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .expect("a tokio runtime") + .block_on(future) +} diff --git a/src/runtime/tokio_rt.rs b/src/runtime/tokio_rt.rs new file mode 100644 index 00000000..337619a0 --- /dev/null +++ b/src/runtime/tokio_rt.rs @@ -0,0 +1,176 @@ +//! The tokio backend, behind the `tokio-runtime` feature. +//! +//! # Why this is optional and off +//! +//! Getting tokio out of the normal dependency graph was the whole of the work +//! this builds on: it used to arrive through six `tokio::` paths that +//! `worktable!` emitted into consumer crates, which made a runtime part of the +//! macro's contract for every consumer whether they ran one or not. Adding it +//! back unconditionally would undo that. `cargo tree -e normal -i tokio` prints +//! nothing in the default feature set, and that is the check. +//! +//! # What selecting it costs +//! +//! [`TokioRt::spawn`] needs an ambient tokio runtime and panics without one, +//! where the nagoya backend starts its own threads on first use. That is +//! tokio's shape, not something this wrapper can paper over. + +use alloc::sync::Arc; +use core::future::Future; +use core::pin::Pin; +use core::task::{Context, Poll}; +use core::time::Duration; + +use super::{ + Elapsed, Runtime, RuntimeJoinHandle, RuntimeNotified, RuntimeNotify, RuntimeRwLock, RuntimeSemaphore, + RuntimeSemaphorePermit, +}; + +/// The tokio backend. +/// +/// Unflavored: tokio's scheduler exposes no equivalent of `ps-st3`'s +/// [`Tuning`](super::Tuning), which is why the schema grammar accepts +/// `runtime: tokio` and rejects `runtime: tokio(spread)`. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TokioRt; + +impl Runtime for TokioRt { + type RwLock = tokio::sync::RwLock; + type Notify = tokio::sync::Notify; + type Semaphore = tokio::sync::Semaphore; + type JoinHandle = TokioJoinHandle; + + fn spawn(future: F) -> Self::JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + TokioJoinHandle(tokio::spawn(future)) + } + + fn sleep(duration: Duration) -> impl Future + Send { + tokio::time::sleep(duration) + } + + async fn timeout(duration: Duration, future: F) -> Result + where + F: Future + Send, + { + tokio::time::timeout(duration, future).await.map_err(|_| Elapsed) + } + + fn yield_now() -> impl Future + Send { + tokio::task::yield_now() + } +} + +/// A `tokio::task::JoinHandle` wearing nagoya's shape. +/// +/// Two things change. Cancellation consumes the handle, because nagoya's does +/// and because `abort(&self)` invites awaiting a handle that will never +/// produce a value. And the output is `Option`, so `None` is cancellation; +/// a panic in the task is resumed here rather than handed back as a value, +/// which is what nagoya does by letting it unwind through the await. +#[derive(Debug)] +pub struct TokioJoinHandle(tokio::task::JoinHandle); + +impl Future for TokioJoinHandle { + type Output = Option; + + fn poll(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll { + match Pin::new(&mut self.0).poll(context) { + Poll::Pending => Poll::Pending, + Poll::Ready(Ok(output)) => Poll::Ready(Some(output)), + Poll::Ready(Err(error)) if error.is_cancelled() => Poll::Ready(None), + Poll::Ready(Err(error)) => std::panic::resume_unwind(error.into_panic()), + } + } +} + +impl RuntimeJoinHandle for TokioJoinHandle { + fn cancel(self) { + self.0.abort(); + } + + fn is_finished(&self) -> bool { + self.0.is_finished() + } +} + +impl RuntimeRwLock for tokio::sync::RwLock { + type ReadGuard<'a> = tokio::sync::RwLockReadGuard<'a, T>; + type WriteGuard<'a> = tokio::sync::RwLockWriteGuard<'a, T>; + type OwnedReadGuard = tokio::sync::OwnedRwLockReadGuard; + + fn new(value: T) -> Self { + tokio::sync::RwLock::new(value) + } + + fn write(&self) -> impl Future> + Send { + tokio::sync::RwLock::write(self) + } + + fn try_read(&self) -> Option> { + tokio::sync::RwLock::try_read(self).ok() + } + + fn try_read_owned(self: Arc) -> Option { + tokio::sync::RwLock::try_read_owned(self).ok() + } +} + +impl RuntimeNotify for tokio::sync::Notify { + type Notified<'a> = tokio::sync::futures::Notified<'a>; + + fn new() -> Self { + tokio::sync::Notify::new() + } + + fn notify_one(&self) { + tokio::sync::Notify::notify_one(self); + } + + fn notify_waiters(&self) { + tokio::sync::Notify::notify_waiters(self); + } + + fn notified(&self) -> Self::Notified<'_> { + tokio::sync::Notify::notified(self) + } +} + +impl RuntimeNotified for tokio::sync::futures::Notified<'_> { + fn enable(self: Pin<&mut Self>) -> bool { + tokio::sync::futures::Notified::enable(self) + } +} + +impl RuntimeSemaphore for tokio::sync::Semaphore { + type Permit<'a> = tokio::sync::SemaphorePermit<'a>; + + fn new(permits: usize) -> Self { + tokio::sync::Semaphore::new(permits) + } + + fn add_permits(&self, permits: usize) { + tokio::sync::Semaphore::add_permits(self, permits); + } + + /// nagoya's semaphore has no closed state, so the normalised signature has + /// no error to carry. Nothing in this crate closes a semaphore, and `close` + /// is not on [`RuntimeSemaphore`], so the only way to reach the panic is a + /// caller going past the trait to the concrete tokio type. + async fn acquire(&self) -> Self::Permit<'_> { + tokio::sync::Semaphore::acquire(self) + .await + .expect("nothing closes a worktable semaphore") + } +} + +impl RuntimeSemaphorePermit for tokio::sync::SemaphorePermit<'_> { + fn forget(self) { + tokio::sync::SemaphorePermit::forget(self); + } +} + +impl super::RuntimeCompatibleWith for TokioRt {} diff --git a/src/storage_catalog.rs b/src/storage_catalog.rs new file mode 100644 index 00000000..f882427c --- /dev/null +++ b/src/storage_catalog.rs @@ -0,0 +1,613 @@ +//! The database-wide system catalog backed by a generated WorkTable. + +use data_bucket::storage::{ + CatalogError, CatalogKey, CatalogMutation, CatalogName, CatalogRecord, CatalogRecordKind, CatalogWritePermit, + CommittedGeneration, DomainError, GenerationBuilder, GenerationPlan, PageAddress, PageStore, PreparedSystemCatalog, + ReplicaState, ReplicationErrorCode, StorageDomain, StorageDomainId, SystemCatalog, SystemIndexRecord, + SystemPageRecord, SystemReplicationRecord, SystemTableRecord, TableId, WriterEpoch, +}; +use data_bucket::{PageId, SpaceId}; +use parking_lot::RwLock; + +use crate::prelude::*; +use crate::worktable; + +type CatalogKeyBytes = [u8; 32]; +type CatalogNameBytes = [u8; 96]; +type ObjectBytes = [u8; 32]; + +worktable!( + name: StorageCatalog, + vec: true, + columns: { + key: CatalogKeyBytes primary_key using fxhash, + kind: u8, + table_id: u32, + space_id: u32, + page_id: u32, + page_kind: u8, + generation: u64, + index_id: u32, + name_length: u8, + name_bytes: CatalogNameBytes, + schema_version: u32, + data_space_id: u32, + page_stride: u32, + row_count: u64, + live_row_bytes: u64, + allocated_data_pages: u64, + live_data_pages: u64, + primary_index_entries: u64, + secondary_index_entries: u64, + tombstones: u64, + applied_generation: u64, + durable_generation: u64, + object: ObjectBytes, + object_offset: u64, + encoded_length: u32, + decoded_length: u32, + checksum: ObjectBytes, + live_rows: u32, + live_bytes: u32, + entries: u64, + index_space_id: u32, + index_primary: bool, + upstash: u8, + tigris: u8, + last_error: u16, + }, +); + +/// DataBucket owns writes through its private permit; applications receive a +/// query-only view over the same generated table. +pub struct GeneratedSystemCatalog { + table: RwLock, +} + +impl Default for GeneratedSystemCatalog { + fn default() -> Self { + Self { + table: RwLock::new(StorageCatalogWorkTable::new()), + } + } +} + +impl GeneratedSystemCatalog { + #[must_use] + pub fn view(&self) -> SystemCatalogView<'_> { + SystemCatalogView { catalog: self } + } +} + +/// Read access to catalog rows. Mutations remain part of DataBucket's commit. +#[derive(Clone, Copy)] +pub struct SystemCatalogView<'a> { + catalog: &'a GeneratedSystemCatalog, +} + +impl SystemCatalogView<'_> { + #[must_use] + pub fn record(&self, key: &CatalogKey) -> Option { + self.catalog.record(key) + } + + #[must_use] + pub fn tables(&self) -> Vec { + self.catalog + .records(CatalogRecordKind::Table) + .into_iter() + .filter_map(|record| match record { + CatalogRecord::Table(row) => Some(row), + _ => None, + }) + .collect() + } + + #[must_use] + pub fn system_tables(&self) -> Vec { + self.tables() + } + + #[must_use] + pub fn pages(&self) -> Vec { + self.catalog + .records(CatalogRecordKind::Page) + .into_iter() + .filter_map(|record| match record { + CatalogRecord::Page(row) => Some(row), + _ => None, + }) + .collect() + } + + #[must_use] + pub fn system_pages(&self) -> Vec { + self.pages() + } + + #[must_use] + pub fn indexes(&self) -> Vec { + self.catalog + .records(CatalogRecordKind::Index) + .into_iter() + .filter_map(|record| match record { + CatalogRecord::Index(row) => Some(row), + _ => None, + }) + .collect() + } + + #[must_use] + pub fn system_indexes(&self) -> Vec { + self.indexes() + } + + #[must_use] + pub fn replication(&self) -> Vec { + self.catalog + .records(CatalogRecordKind::Replication) + .into_iter() + .filter_map(|record| match record { + CatalogRecord::Replication(row) => Some(row), + _ => None, + }) + .collect() + } + + #[must_use] + pub fn system_replication(&self) -> Vec { + self.replication() + } +} + +/// One database-wide DataBucket domain with one generated system catalog. +pub struct Database { + domain: Arc>>, +} + +impl Clone for Database { + fn clone(&self) -> Self { + Self { + domain: self.domain.clone(), + } + } +} + +impl Database { + #[must_use] + pub fn new(id: StorageDomainId, writer_epoch: WriterEpoch, store: S) -> Self { + Self { + domain: Arc::new(RwLock::new(StorageDomain::new( + id, + writer_epoch, + GeneratedSystemCatalog::default(), + store, + ))), + } + } + + pub fn open(id: StorageDomainId, writer_epoch: WriterEpoch, store: S) -> Result> { + let domain = StorageDomain::open(id, writer_epoch, GeneratedSystemCatalog::default(), store)?; + Ok(Self { + domain: Arc::new(RwLock::new(domain)), + }) + } + + pub fn begin_generation(&self) -> Result> { + self.domain.read().begin_generation() + } + + #[must_use] + pub fn id(&self) -> StorageDomainId { + self.domain.read().id() + } + + #[must_use] + pub fn generation(&self) -> u64 { + self.domain.read().generation() + } + + pub fn register_table(&self, name: &str, schema_version: u32) -> Result> { + self.register_table_with_stride(name, schema_version, data_bucket::PAGE_SIZE as u32) + } + + pub fn register_table_with_stride( + &self, + name: &str, + schema_version: u32, + page_stride: u32, + ) -> Result> { + let name = CatalogName::new(name).map_err(DomainError::Catalog)?; + let mut domain = self.domain.write(); + let tables = domain.catalog().records(CatalogRecordKind::Table); + let existing = tables.iter().find_map(|record| match record { + CatalogRecord::Table(table) if table.name == name => Some(table.clone()), + _ => None, + }); + let mut table = if let Some(table) = existing { + if table.schema_version == schema_version && table.page_stride == page_stride { + return Ok(table.table_id); + } + table + } else { + let next = tables + .iter() + .filter_map(|record| match record { + CatalogRecord::Table(table) => Some(table.table_id.0), + _ => None, + }) + .max() + .unwrap_or(0) + .checked_add(1) + .ok_or(DomainError::Catalog(CatalogError::InvalidMutation))?; + SystemTableRecord { + table_id: TableId(next), + name, + schema_version, + data_space_id: SpaceId(0), + page_stride, + row_count: 0, + live_row_bytes: 0, + allocated_data_pages: 0, + live_data_pages: 0, + primary_index_entries: 0, + secondary_index_entries: 0, + tombstones: 0, + applied_generation: domain.generation(), + durable_generation: domain.generation(), + } + }; + table.schema_version = schema_version; + table.page_stride = page_stride; + let table_id = table.table_id; + let mut generation = domain.begin_generation()?; + generation.update_catalog(CatalogMutation::Upsert(CatalogRecord::Table(table))); + domain.commit_generation(generation.finish())?; + Ok(table_id) + } + + pub fn commit_generation(&self, plan: GenerationPlan) -> Result> { + self.domain.write().commit_generation(plan) + } + + pub fn read_page(&self, address: PageAddress) -> Result>, DomainError> { + self.domain.read().read_page(address) + } + + #[must_use] + pub fn catalog(&self) -> DatabaseCatalog { + DatabaseCatalog { + domain: self.domain.clone(), + } + } +} + +#[cfg(feature = "s3-support")] +impl Database { + pub fn open_s3( + id: StorageDomainId, + writer_epoch: WriterEpoch, + config: data_bucket::storage::s3::S3Config, + ) -> Result> { + let store = data_bucket::storage::s3::S3PageStore::new(config).map_err(DomainError::Store)?; + Self::open(id, writer_epoch, store) + } +} + +/// Cloneable read-only access to the database's generated catalog. +pub struct DatabaseCatalog { + domain: Arc>>, +} + +impl Clone for DatabaseCatalog { + fn clone(&self) -> Self { + Self { + domain: self.domain.clone(), + } + } +} + +impl DatabaseCatalog { + #[must_use] + pub fn record(&self, key: &CatalogKey) -> Option { + self.domain.read().catalog().record(key) + } + + #[must_use] + pub fn system_tables(&self) -> Vec { + records_of(&self.domain, CatalogRecordKind::Table) + .into_iter() + .filter_map(|record| match record { + CatalogRecord::Table(row) => Some(row), + _ => None, + }) + .collect() + } + + #[must_use] + pub fn system_pages(&self) -> Vec { + records_of(&self.domain, CatalogRecordKind::Page) + .into_iter() + .filter_map(|record| match record { + CatalogRecord::Page(row) => Some(row), + _ => None, + }) + .collect() + } + + #[must_use] + pub fn system_indexes(&self) -> Vec { + records_of(&self.domain, CatalogRecordKind::Index) + .into_iter() + .filter_map(|record| match record { + CatalogRecord::Index(row) => Some(row), + _ => None, + }) + .collect() + } + + #[must_use] + pub fn system_replication(&self) -> Vec { + records_of(&self.domain, CatalogRecordKind::Replication) + .into_iter() + .filter_map(|record| match record { + CatalogRecord::Replication(row) => Some(row), + _ => None, + }) + .collect() + } +} + +fn records_of( + domain: &RwLock>, + kind: CatalogRecordKind, +) -> Vec { + domain.read().catalog().records(kind) +} + +pub struct PreparedGeneratedCatalog { + table: StorageCatalogWorkTable, + checkpoint: Vec, +} + +impl PreparedSystemCatalog for PreparedGeneratedCatalog { + fn checkpoint(&self) -> &[u8] { + &self.checkpoint + } +} + +impl SystemCatalog for GeneratedSystemCatalog { + type Prepared = PreparedGeneratedCatalog; + + fn prepare( + &self, + _permit: &CatalogWritePermit, + mutations: &[CatalogMutation], + ) -> Result { + let bytes = self.table.read().unload().map_err(|_| CatalogError::Codec)?; + let mut table = StorageCatalogWorkTable::load(&bytes).map_err(|_| CatalogError::Codec)?; + for mutation in mutations { + match mutation { + CatalogMutation::Upsert(record) => table.upsert(record_to_row(record)), + CatalogMutation::Delete(key) => { + if let Some(existing) = table.select(key) { + let mut tombstone = existing.clone(); + tombstone.kind = 0; + table.upsert(tombstone); + } + } + } + } + let checkpoint = table.unload().map_err(|_| CatalogError::Codec)?; + Ok(PreparedGeneratedCatalog { table, checkpoint }) + } + + fn prepare_restore(&self, _permit: &CatalogWritePermit, checkpoint: &[u8]) -> Result { + let table = StorageCatalogWorkTable::load(checkpoint).map_err(|_| CatalogError::Codec)?; + Ok(PreparedGeneratedCatalog { + table, + checkpoint: checkpoint.to_vec(), + }) + } + + fn publish(&self, _permit: &CatalogWritePermit, prepared: Self::Prepared) { + *self.table.write() = prepared.table; + } + + fn record(&self, key: &CatalogKey) -> Option { + self.table.read().select(key).and_then(row_to_record) + } + + fn records(&self, kind: CatalogRecordKind) -> Vec { + self.table + .read() + .select_all() + .filter(|row| row.kind == kind as u8) + .filter_map(row_to_record) + .collect() + } +} + +fn record_to_row(record: &CatalogRecord) -> StorageCatalogRow { + let mut row = empty_row(record.key(), record.kind()); + match record { + CatalogRecord::Table(value) => { + row.table_id = value.table_id.0; + row.name_length = value.name.length(); + row.name_bytes = *value.name.bytes(); + row.schema_version = value.schema_version; + row.data_space_id = value.data_space_id.0; + row.page_stride = value.page_stride; + row.row_count = value.row_count; + row.live_row_bytes = value.live_row_bytes; + row.allocated_data_pages = value.allocated_data_pages; + row.live_data_pages = value.live_data_pages; + row.primary_index_entries = value.primary_index_entries; + row.secondary_index_entries = value.secondary_index_entries; + row.tombstones = value.tombstones; + row.applied_generation = value.applied_generation; + row.durable_generation = value.durable_generation; + } + CatalogRecord::Page(value) => { + row.table_id = value.table_id.0; + row.space_id = value.space_id.0; + row.page_id = usize::from(value.page_id) as u32; + row.page_kind = value.page_kind as u8; + row.generation = value.generation; + row.object = value.object; + row.object_offset = value.object_offset; + row.encoded_length = value.encoded_length; + row.decoded_length = value.decoded_length; + row.checksum = value.checksum; + row.live_rows = value.live_rows; + row.live_bytes = value.live_bytes; + } + CatalogRecord::Index(value) => { + row.table_id = value.table_id.0; + row.index_id = value.index_id; + row.index_space_id = value.space_id.0; + row.index_primary = value.primary; + row.name_length = value.name.length(); + row.name_bytes = *value.name.bytes(); + row.entries = value.entries; + row.generation = value.generation; + } + CatalogRecord::Replication(value) => { + row.generation = value.generation; + row.upstash = value.upstash as u8; + row.tigris = value.tigris as u8; + row.last_error = value.last_error.map_or(0, |error| error as u16); + } + } + row +} + +fn empty_row(key: CatalogKey, kind: CatalogRecordKind) -> StorageCatalogRow { + StorageCatalogRow { + key, + kind: kind as u8, + table_id: 0, + space_id: 0, + page_id: 0, + page_kind: 0, + generation: 0, + index_id: 0, + name_length: 0, + name_bytes: [0; 96], + schema_version: 0, + data_space_id: 0, + page_stride: 0, + row_count: 0, + live_row_bytes: 0, + allocated_data_pages: 0, + live_data_pages: 0, + primary_index_entries: 0, + secondary_index_entries: 0, + tombstones: 0, + applied_generation: 0, + durable_generation: 0, + object: [0; 32], + object_offset: 0, + encoded_length: 0, + decoded_length: 0, + checksum: [0; 32], + live_rows: 0, + live_bytes: 0, + entries: 0, + index_space_id: 0, + index_primary: false, + upstash: 0, + tigris: 0, + last_error: 0, + } +} + +fn row_to_record(row: &StorageCatalogRow) -> Option { + match row.kind { + value if value == CatalogRecordKind::Table as u8 => Some(CatalogRecord::Table(SystemTableRecord { + table_id: data_bucket::storage::TableId(row.table_id), + name: catalog_name(row)?, + schema_version: row.schema_version, + data_space_id: SpaceId(row.data_space_id), + page_stride: row.page_stride, + row_count: row.row_count, + live_row_bytes: row.live_row_bytes, + allocated_data_pages: row.allocated_data_pages, + live_data_pages: row.live_data_pages, + primary_index_entries: row.primary_index_entries, + secondary_index_entries: row.secondary_index_entries, + tombstones: row.tombstones, + applied_generation: row.applied_generation, + durable_generation: row.durable_generation, + })), + value if value == CatalogRecordKind::Page as u8 => Some(CatalogRecord::Page(SystemPageRecord { + table_id: data_bucket::storage::TableId(row.table_id), + space_id: SpaceId(row.space_id), + page_id: PageId::from(row.page_id), + page_kind: page_kind(row.page_kind)?, + generation: row.generation, + object: row.object, + object_offset: row.object_offset, + encoded_length: row.encoded_length, + decoded_length: row.decoded_length, + checksum: row.checksum, + live_rows: row.live_rows, + live_bytes: row.live_bytes, + })), + value if value == CatalogRecordKind::Index as u8 => Some(CatalogRecord::Index(SystemIndexRecord { + table_id: data_bucket::storage::TableId(row.table_id), + index_id: row.index_id, + space_id: SpaceId(row.index_space_id), + primary: row.index_primary, + name: catalog_name(row)?, + entries: row.entries, + generation: row.generation, + })), + value if value == CatalogRecordKind::Replication as u8 => { + Some(CatalogRecord::Replication(SystemReplicationRecord { + generation: row.generation, + upstash: replica_state(row.upstash)?, + tigris: replica_state(row.tigris)?, + last_error: replication_error(row.last_error)?, + })) + } + _ => None, + } +} + +fn catalog_name(row: &StorageCatalogRow) -> Option { + let length = usize::from(row.name_length); + let name = core::str::from_utf8(row.name_bytes.get(..length)?).ok()?; + data_bucket::storage::CatalogName::new(name).ok() +} + +fn page_kind(value: u8) -> Option { + match value { + 1 => Some(data_bucket::storage::PageKind::Data), + 2 => Some(data_bucket::storage::PageKind::PrimaryIndex), + 3 => Some(data_bucket::storage::PageKind::SecondaryIndex), + 4 => Some(data_bucket::storage::PageKind::Metadata), + _ => None, + } +} + +fn replica_state(value: u8) -> Option { + match value { + 0 => Some(ReplicaState::Absent), + 1 => Some(ReplicaState::Staged), + 2 => Some(ReplicaState::Durable), + 3 => Some(ReplicaState::Failed), + _ => None, + } +} + +fn replication_error(value: u16) -> Option> { + match value { + 0 => Some(None), + 1 => Some(Some(ReplicationErrorCode::Transport)), + 2 => Some(Some(ReplicationErrorCode::Conflict)), + 3 => Some(Some(ReplicationErrorCode::Corrupt)), + 4 => Some(Some(ReplicationErrorCode::Unauthorized)), + _ => None, + } +} diff --git a/src/table/mod.rs b/src/table/mod.rs index eeba4d01..8a76f110 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -1,9 +1,14 @@ +use alloc::{string::String, vec::Vec}; pub mod select; pub mod system_info; +#[cfg(feature = "std")] pub mod vacuum; use crate::in_memory::{ArchivedRowWrapper, DataPages, RowWrapper, StorableRow}; -use crate::persistence::{AcknowledgeOperation, InsertOperation, Operation, PersistenceLoadError}; +#[cfg(feature = "std")] +use crate::persistence::PersistenceLoadError; +use crate::persistence::operation::new_operation_uuid; +use crate::persistence::{AcknowledgeOperation, InsertOperation, Operation}; use crate::prelude::{Link, LockMap, OperationId, PrimaryKeyGeneratorState}; use crate::primary_key::{PrimaryKeyGenerator, TablePrimaryKey}; use crate::util::OffsetEqLink; @@ -11,8 +16,13 @@ use crate::{ AvailableIndex, IndexError, IndexMap, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, convert_change_events, in_memory, }; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::marker::PhantomData; use data_bucket::INNER_PAGE_SIZE; use derive_more::{Display, Error, From}; +#[cfg(feature = "std")] +use hashbrown::HashSet; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; #[cfg(feature = "perf_measurements")] @@ -24,12 +34,8 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Portable, Serialize}; -use std::collections::HashSet; -use std::fmt::Debug; -use std::marker::PhantomData; +#[cfg(feature = "std")] use std::path::Path; -use std::sync::Arc; -use uuid::Uuid; /// Keys per chunk when a bulk delete takes its mutation guards. /// /// Guards are striped 64 ways, so any batch wider than that holds every stripe @@ -51,7 +57,7 @@ pub struct WorkTable< const DATA_LENGTH: usize = INNER_PAGE_SIZE, PkMap = IndexMap>, > where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static, PkMap: crate::UniqueIndex>, { @@ -94,7 +100,7 @@ impl< PkMap, > where - PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + core::hash::Hash, SecondaryIndexes: Default, PkGen: Default, PkMap: crate::UniqueIndex>, @@ -127,7 +133,7 @@ impl< > WorkTable where Row: TableRow, - PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + core::hash::Hash, PkMap: crate::UniqueIndex>, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, @@ -138,6 +144,7 @@ where /// This load-only scan prevents a torn index link from turning zeroed or /// unrelated bytes into a plausible row. It deliberately does not run on /// steady-state operations. + #[cfg(feature = "std")] pub fn validate_persisted_state(&self, path: impl AsRef) -> Result<(), PersistenceLoadError> where <::WrappedRow as Archive>::Archived: Portable @@ -146,7 +153,7 @@ where { let path = path.as_ref(); let mut links = HashSet::with_capacity(self.primary_index.pk_map.len()); - let mut cells_by_page = std::collections::HashMap::::new(); + let mut cells_by_page = hashbrown::HashMap::::new(); for (primary_key, offset_link) in self.primary_index.pk_map.iter_values() { if !links.insert(offset_link) { @@ -211,7 +218,7 @@ where /// caller can iterate it directly while pre-assigning contiguous keys to a /// batch of rows for `insert_many`. Interleaved [`Self::get_next_pk`] /// calls keep working and never overlap a reservation. - pub fn reserve_pks(&self, count: usize) -> std::ops::Range + pub fn reserve_pks(&self, count: usize) -> core::ops::Range where PkGen: crate::primary_key::PrimaryKeyGeneratorRange, { @@ -238,7 +245,7 @@ where if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None } @@ -302,6 +309,8 @@ where SecondaryIndexes: TableSecondaryIndex, LockType: 'static, { + let _publication = + TableSecondaryIndex::::row_publication(&*self.indexes); let pk = row.get_primary_key().clone(); let link = self.data.insert(row.clone()).map_err(WorkTableError::PagesError)?; if self.primary_index.insert_checked(pk.clone(), link).is_none() { @@ -317,6 +326,13 @@ where Err(WorkTableError::AlreadyExists(at.to_string_value())) } + IndexError::ColumnSlotIdExhausted { bits, inserted_already } => { + self.primary_index.remove(&pk, link); + self.indexes.delete_from_indexes(row, link, inserted_already)?; + self.data.delete(link).map_err(WorkTableError::PagesError)?; + + Err(WorkTableError::ColumnSlotIdExhausted(bits)) + } IndexError::NotFound => { // Mirror the AlreadyExists arm. Returning without rollback // left the primary key permanently bound to a ghosted row @@ -401,6 +417,8 @@ where // delete and a batch insert cannot deadlock against each other. Chunks // release before the next is taken, so that ordering holds across them. let _mutation_guards = self.lock_manager.mutation_guards(chunk.iter()); + let _publication = + TableSecondaryIndex::::row_publication(&*self.indexes); let mut links: Vec = Vec::with_capacity(chunk.len()); @@ -475,7 +493,7 @@ where /// Returns the keys actually deleted, in key order. pub fn delete_range(&self, range: R) -> Result, BatchDeleteError> where - R: std::ops::RangeBounds, + R: core::ops::RangeBounds, Row: Archive + Clone + for<'a> Serialize, Share>, rkyv::rancor::Error>>, @@ -516,6 +534,8 @@ where // takes, so it is the one that most needs not to hold every stripe. for chunk in keys.chunks(DELETE_CHUNK_KEYS) { let _mutation_guards = self.lock_manager.mutation_guards(chunk.iter()); + let _publication = + TableSecondaryIndex::::row_publication(&*self.indexes); // Second walk, under the guards. Links for guarded keys cannot move // and are used directly; keys that appeared since the first walk are @@ -531,8 +551,8 @@ where .primary_index .pk_map .range_values(( - std::ops::Bound::Included(chunk[0].clone()), - std::ops::Bound::Included(chunk[chunk.len() - 1].clone()), + core::ops::Bound::Included(chunk[0].clone()), + core::ops::Bound::Included(chunk[chunk.len() - 1].clone()), )) .map(|(key, link)| (key, link.into())) .collect(); @@ -613,6 +633,8 @@ where } let pks: Vec = rows.iter().map(|row| row.get_primary_key().clone()).collect(); let _mutation_guards = self.lock_manager.mutation_guards(pks.iter()); + let _publication = + TableSecondaryIndex::::row_publication(&*self.indexes); let mut links: Vec = Vec::with_capacity(rows.len()); for (row_index, row) in rows.iter().enumerate() { @@ -676,6 +698,12 @@ where self.data.delete(link).map_err(WorkTableError::PagesError)?; Err(WorkTableError::AlreadyExists(at.to_string_value())) } + IndexError::ColumnSlotIdExhausted { bits, inserted_already } => { + self.primary_index.remove(pk, link); + self.indexes.delete_from_indexes(row.clone(), link, inserted_already)?; + self.data.delete(link).map_err(WorkTableError::PagesError)?; + Err(WorkTableError::ColumnSlotIdExhausted(bits)) + } IndexError::NotFound => { self.primary_index.remove(pk, link); self.indexes.delete_row(row.clone(), link)?; @@ -756,6 +784,8 @@ where { let pk = row.get_primary_key().clone(); let _mutation_guard = self.lock_manager.mutation_guard(&pk); + let _publication = + TableSecondaryIndex::::row_publication(&*self.indexes); let (link, _) = match self.data.insert_cdc(row.clone()) { Ok(result) => result, @@ -789,7 +819,7 @@ where merged_secondary_events.extend(rollback_secondary_events); let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(Uuid::now_v7()), + id: OperationId::Single(new_operation_uuid()), primary_key_events: merged_primary_events, secondary_keys_events: merged_secondary_events, }); @@ -800,6 +830,32 @@ where (ack_op, WorkTableError::AlreadyExists(at.to_string_value())) } } + IndexError::ColumnSlotIdExhausted { bits, inserted_already } => { + let (_, rollback_pk_events) = self.primary_index.remove_cdc(pk.clone(), link); + let rollback_pk_events = convert_change_events(rollback_pk_events); + + let (rollback_secondary_events, _) = + self.indexes + .delete_from_indexes_cdc(row.clone(), link, inserted_already); + + let mut merged_primary_events = primary_key_events.clone(); + merged_primary_events.extend(rollback_pk_events); + + let mut merged_secondary_events = secondary_events.clone(); + merged_secondary_events.extend(rollback_secondary_events); + + let ack_op = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(new_operation_uuid()), + primary_key_events: merged_primary_events, + secondary_keys_events: merged_secondary_events, + }); + + if let Err(e) = self.data.delete(link) { + (ack_op, WorkTableError::PagesError(e)) + } else { + (ack_op, WorkTableError::ColumnSlotIdExhausted(bits)) + } + } IndexError::NotFound => { // Mirror the AlreadyExists arm: roll the primary index and // the row's secondary entries back and release the data @@ -816,7 +872,7 @@ where merged_secondary_events.extend(rollback_secondary_events); let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(Uuid::now_v7()), + id: OperationId::Single(new_operation_uuid()), primary_key_events: merged_primary_events, secondary_keys_events: merged_secondary_events, }); @@ -834,7 +890,7 @@ where unsafe { if let Err(e) = self.data.with_mut_ref(link, |r| r.unghost()) { let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(Uuid::now_v7()), + id: OperationId::Single(new_operation_uuid()), primary_key_events: primary_key_events.clone(), secondary_keys_events: secondary_events.clone(), }); @@ -846,7 +902,7 @@ where Ok(bytes) => bytes, Err(e) => { let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(Uuid::now_v7()), + id: OperationId::Single(new_operation_uuid()), primary_key_events: primary_key_events.clone(), secondary_keys_events: secondary_events.clone(), }); @@ -855,7 +911,8 @@ where }; let op = Operation::Insert(InsertOperation { - id: OperationId::Single(Uuid::now_v7()), + retired_link: None, + id: OperationId::Single(new_operation_uuid()), pk_gen_state: self.pk_gen.get_state(), primary_key_events, secondary_keys_events: secondary_events, @@ -907,6 +964,8 @@ where } let pks: Vec = rows.iter().map(|row| row.get_primary_key().clone()).collect(); let _mutation_guards = self.lock_manager.mutation_guards(pks.iter()); + let _publication = + TableSecondaryIndex::::row_publication(&*self.indexes); let mut links: Vec = Vec::with_capacity(rows.len()); let mut forward_primary: Vec>>> = Vec::with_capacity(rows.len()); @@ -948,7 +1007,7 @@ where } let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(Uuid::now_v7()), + id: OperationId::Single(new_operation_uuid()), primary_key_events: merged_primary, secondary_keys_events: merged_secondary, }); @@ -1011,6 +1070,18 @@ where Err(e) => WorkTableError::PagesError(e), } } + IndexError::ColumnSlotIdExhausted { bits, inserted_already } => { + let (_, rollback_primary) = self.primary_index.remove_cdc(pks[row_index].clone(), link); + primary_key_events.extend(convert_change_events(rollback_primary)); + let (rollback_secondary, _) = + self.indexes + .delete_from_indexes_cdc(row.clone(), link, inserted_already); + secondary_events.extend(rollback_secondary); + match self.data.delete(link) { + Ok(()) => WorkTableError::ColumnSlotIdExhausted(bits), + Err(e) => WorkTableError::PagesError(e), + } + } IndexError::NotFound => { let (_, rollback_primary) = self.primary_index.remove_cdc(pks[row_index].clone(), link); primary_key_events.extend(convert_change_events(rollback_primary)); @@ -1049,11 +1120,11 @@ where // creation-ordered, so cross-chunk event order survives the // analyzer's operation-id sort. const PERSIST_GROUP_ROWS: usize = 1024; - let mut batch_id = Uuid::now_v7(); + let mut batch_id = new_operation_uuid(); let mut ops = Vec::with_capacity(links.len()); for (row_index, link) in links.iter().enumerate() { if row_index != 0 && row_index % PERSIST_GROUP_ROWS == 0 { - batch_id = Uuid::now_v7(); + batch_id = new_operation_uuid(); } let published = unsafe { self.data.with_mut_ref(*link, |r| r.unghost()) }; let bytes = match published @@ -1087,10 +1158,11 @@ where } }; ops.push(Operation::Insert(InsertOperation { + retired_link: None, id: OperationId::Multi(batch_id), pk_gen_state: self.pk_gen.get_state(), - primary_key_events: std::mem::take(&mut forward_primary[row_index]), - secondary_keys_events: std::mem::take(&mut forward_secondary[row_index]), + primary_key_events: core::mem::take(&mut forward_primary[row_index]), + secondary_keys_events: core::mem::take(&mut forward_secondary[row_index]), bytes, link: *link, })); @@ -1125,6 +1197,8 @@ where SecondaryIndexes: TableSecondaryIndex, LockType: 'static, { + let _publication = + TableSecondaryIndex::::row_publication(&*self.indexes); let pk = row_new.get_primary_key().clone(); if pk != row_old.get_primary_key() { return Err(WorkTableError::PrimaryUpdateTry); @@ -1157,6 +1231,16 @@ where Err(WorkTableError::AlreadyExists(at.to_string_value())) } + IndexError::ColumnSlotIdExhausted { bits, inserted_already } => { + // The primary index still points at old_link here (it is + // swung only after every index check passes), so the + // unwind only has to drop what reinsert_row published on + // the new link and release the new slot. + self.indexes.delete_from_indexes(row_new, new_link, inserted_already)?; + self.data.delete(new_link).map_err(WorkTableError::PagesError)?; + + Err(WorkTableError::ColumnSlotIdExhausted(bits)) + } IndexError::NotFound => { // The primary index was never swung and the new row is // still ghosted, so no reader can observe it; release the @@ -1203,6 +1287,8 @@ where AvailableIndexes: Debug + AvailableIndex, PrimaryIndex: TableIndexCdc, { + let _publication = + TableSecondaryIndex::::row_publication(&*self.indexes); let pk = row_new.get_primary_key().clone(); if pk != row_old.get_primary_key() { return (None, Err(WorkTableError::PrimaryUpdateTry)); @@ -1250,7 +1336,7 @@ where merged_secondary_events.extend(rollback_secondary_events); let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(Uuid::now_v7()), + id: OperationId::Single(new_operation_uuid()), primary_key_events: vec![], secondary_keys_events: merged_secondary_events, }); @@ -1261,6 +1347,29 @@ where (ack_op, WorkTableError::AlreadyExists(at.to_string_value())) } } + IndexError::ColumnSlotIdExhausted { bits, inserted_already } => { + // Same shape as the AlreadyExists arm: the primary index + // was never swung, so only the secondary entries this + // reinsert published have to be taken back out. + let (rollback_secondary_events, _) = + self.indexes + .delete_from_indexes_cdc(row_new, new_link, inserted_already); + + let mut merged_secondary_events = secondary_events.clone(); + merged_secondary_events.extend(rollback_secondary_events); + + let ack_op = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(new_operation_uuid()), + primary_key_events: vec![], + secondary_keys_events: merged_secondary_events, + }); + + if let Err(e) = self.data.delete(new_link) { + (ack_op, WorkTableError::PagesError(e)) + } else { + (ack_op, WorkTableError::ColumnSlotIdExhausted(bits)) + } + } IndexError::NotFound => { // As in `reinsert`: the primary index was never swung and // the new row is still ghosted, so releasing the new data @@ -1269,7 +1378,7 @@ where // secondary entries cannot be unwound precisely; no // current index implementation returns it. let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(Uuid::now_v7()), + id: OperationId::Single(new_operation_uuid()), primary_key_events: vec![], secondary_keys_events: secondary_events.clone(), }); @@ -1290,7 +1399,7 @@ where // Delete old data if let Err(e) = self.data.delete(old_link) { let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(Uuid::now_v7()), + id: OperationId::Single(new_operation_uuid()), primary_key_events: primary_key_events.clone(), secondary_keys_events: secondary_events.clone(), }); @@ -1302,7 +1411,7 @@ where Ok(bytes) => bytes, Err(e) => { let ack_op = Operation::Acknowledge(AcknowledgeOperation { - id: OperationId::Single(Uuid::now_v7()), + id: OperationId::Single(new_operation_uuid()), primary_key_events: primary_key_events.clone(), secondary_keys_events: secondary_events.clone(), }); @@ -1311,7 +1420,8 @@ where }; let op = Operation::Insert(InsertOperation { - id: OperationId::Single(Uuid::now_v7()), + retired_link: Some(old_link), + id: OperationId::Single(new_operation_uuid()), pk_gen_state: self.pk_gen.get_state(), primary_key_events, secondary_keys_events: secondary_events, @@ -1349,7 +1459,7 @@ pub enum BatchInsertError { /// batch needs to know the prefix already succeeded rather than assume nothing /// happened. #[derive(Debug, Display, Error)] -pub enum BatchDeleteError { +pub enum BatchDeleteError { /// One key could not be deleted. Everything before it was. #[display("batch delete stopped at {key:?} after {deleted} deleted: {source}")] Key { @@ -1366,15 +1476,22 @@ pub enum BatchDeleteError { #[derive(Debug, Display, Error, From)] pub enum WorkTableError { + #[display("A runtime-selected query requires execute_async().await")] + RuntimeRequiresAsync, + #[display("The query runtime cancelled execution")] + RuntimeCancelled, NotFound, #[display("Value already exists for `{}` index", _0)] AlreadyExists(#[error(not(source))] String), #[display("Row with this primary key already exists")] PrimaryAlreadyExists, + #[display("ColumnSlotId{} capacity is exhausted", _0)] + ColumnSlotIdExhausted(#[error(not(source))] u8), SerializeError, SecondaryIndexError, PrimaryUpdateTry, PagesError(in_memory::PagesExecutionError), #[display("{}", _0)] - PersistenceError(#[error(not(source))] std::sync::Arc), + #[cfg(feature = "std")] + PersistenceError(#[error(not(source))] alloc::sync::Arc), } diff --git a/src/table/select/mod.rs b/src/table/select/mod.rs index 5b9fc6e6..65c9090b 100644 --- a/src/table/select/mod.rs +++ b/src/table/select/mod.rs @@ -1,8 +1,10 @@ -use std::collections::VecDeque; +use alloc::collections::VecDeque; + +use crate::runtime::Tuning; mod query; -pub use query::{SelectQueryBuilder, SelectQueryExecutor}; +pub use query::{SelectQueryAsyncExecutor, SelectQueryBuilder, SelectQueryExecutor, SelectQueryFuture}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Order { @@ -17,4 +19,10 @@ pub struct QueryParams { pub order: VecDeque<(Order, RowFields)>, pub range: VecDeque<(ColumnRange, RowFields)>, pub sorted_by: Option, + /// The pool settings the profile named at the call site asks for, `None` + /// when no `.runtime()` was written. Descriptive metadata; submission uses + /// the concrete profile dispatcher below, not a runtime lookup by tuning. + pub tuning: Option, + /// Submission chosen by the explicit runtime callsite. + pub dispatch: Option, } diff --git a/src/table/select/query.rs b/src/table/select/query.rs index 2b2f3f66..c46f7682 100644 --- a/src/table/select/query.rs +++ b/src/table/select/query.rs @@ -1,7 +1,9 @@ use crate::WorkTableError; +use crate::runtime::{Profile, RuntimeUnpinned, TableRuntime}; use crate::select::{Order, QueryParams}; +use alloc::vec::Vec; -use std::collections::VecDeque; +use alloc::collections::VecDeque; pub struct SelectQueryBuilder where @@ -23,6 +25,8 @@ where order: VecDeque::new(), range: VecDeque::new(), sorted_by: None, + tuning: None, + dispatch: None, }, iter, } @@ -36,6 +40,8 @@ where order: VecDeque::new(), range: VecDeque::new(), sorted_by: Some(sorted_by), + tuning: None, + dispatch: None, }, iter, } @@ -67,6 +73,32 @@ where self.params.range.push_back((range.into(), column)); self } + + /// Select the executor for an owned asynchronous query. + /// + /// Finish with `execute_async().await`. Calling synchronous `execute()` + /// after this link returns `RuntimeRequiresAsync` instead of ignoring it. + /// Borrowed iteration and `where_by` predicates run on the caller while + /// constructing the future. Range filters, ordering, offset and limit run + /// on the selected executor over owned rows. This materializes all input + /// rows, so use synchronous execution for short or streaming selections. + /// + /// The profile backend family must match `TableRuntime::Backend`; Nagoya + /// flavors may differ from the table default. A mutation section profile + /// applies to its own methods and does not pin unrelated select builders. + /// Hosted paged tables implement these markers; Vec tables stay synchronous. + pub fn runtime

(mut self, profile: P) -> Self + where + Row: TableRuntime + RuntimeUnpinned, + P: Profile, + P::Backend: crate::runtime::RuntimeCompatibleWith<::Backend>, + ::JoinHandle<()>: Unpin, + { + let _ = profile; + self.params.tuning = Some(P::tuning()); + self.params.dispatch = Some(P::dispatcher()); + self + } } pub trait SelectQueryExecutor @@ -82,3 +114,14 @@ where where F: FnMut(&Row) -> bool; } + +/// Owned asynchronous select execution. Borrowed iteration and predicates are +/// materialized by the caller; the owned filtering/sorting plan can be dispatched. +/// A future containing owned rows only, independent of the source iterator's lifetime. +pub type SelectQueryFuture = core::pin::Pin< + alloc::boxed::Box, WorkTableError>> + Send + 'static>, +>; + +pub trait SelectQueryAsyncExecutor { + fn execute_async(self) -> SelectQueryFuture; +} diff --git a/src/table/system_info.rs b/src/table/system_info.rs index 49e95761..bb1c6010 100644 --- a/src/table/system_info.rs +++ b/src/table/system_info.rs @@ -1,5 +1,7 @@ -use prettytable::{Table, format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR, row}; -use std::fmt::{self, Debug, Display, Formatter}; +use alloc::{string::String, string::ToString, vec::Vec}; +use core::fmt::{self, Debug, Display, Formatter}; +#[cfg(not(feature = "std"))] +use ordered_float::FloatCore; use crate::in_memory::{RowWrapper, StorableRow}; use crate::mem_stat::MemStat; @@ -55,7 +57,7 @@ impl< PkMap, > WorkTable where - PrimaryKey: Debug + Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, PkMap: UniqueIndex>, @@ -118,31 +120,76 @@ impl Display for SystemInfo { "Allocated Memory: {mem_fmt} (data) + {idx_fmt} (indexes) = {total_fmt} total\n" )?; - let mut table = Table::new(); - table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR); - table.add_row(row!["Index", "Type", "Keys", "Capacity", "Node Count", "Heap", "Used"]); - + // **Padded by hand rather than by a table crate.** + // + // This used to be `prettytable-rs`, which reaches `csv` and then + // `memchr` and fails to build without `std` in 505 places. Every + // alternative measured puts its usable API behind `std` too: `tabled` + // compiles without `std` but exposes no `Table` at all in that mode, + // and `comfy-table` and `ascii_table` do not compile. Seven columns of + // short strings are not worth a dependency that decides whether this + // crate can be embedded. + let mut rows: Vec<[String; COLUMNS]> = Vec::with_capacity(self.indexes_info.len() + 1); + rows.push([ + "Index".to_string(), + "Type".to_string(), + "Keys".to_string(), + "Capacity".to_string(), + "Node Count".to_string(), + "Heap".to_string(), + "Used".to_string(), + ]); for idx in &self.indexes_info { - table.add_row(row![ - idx.name, + rows.push([ + idx.name.to_string(), idx.index_type.to_string(), - idx.key_count, - idx.capacity, - idx.node_count, + idx.key_count.to_string(), + idx.capacity.to_string(), + idx.node_count.to_string(), fmt_bytes(idx.heap_size), fmt_bytes(idx.used_size), ]); } - let mut buffer = Vec::new(); - table.print(&mut buffer).unwrap(); - let table_str = String::from_utf8(buffer).unwrap(); - writeln!(f, "{}", table_str.trim_end())?; + // Width by character count, not byte length: an index named with any + // multi-byte character would otherwise pad short and skew every column + // after it. + let mut widths = [0usize; COLUMNS]; + for row in &rows { + for (width, cell) in widths.iter_mut().zip(row) { + *width = (*width).max(cell.chars().count()); + } + } + + for (index, row) in rows.iter().enumerate() { + for (column, (cell, width)) in row.iter().zip(&widths).enumerate() { + if column + 1 == COLUMNS { + write!(f, "{cell}")?; + } else { + let padding = width - cell.chars().count(); + write!(f, "{cell}{:padding$}{COLUMN_GAP}", "")?; + } + } + writeln!(f)?; + // A rule under the header, and nothing between the rows: the same + // shape `FORMAT_NO_BORDER_LINE_SEPARATOR` produced. + if index == 0 { + let rule: usize = widths.iter().sum::() + COLUMN_GAP.len() * (COLUMNS - 1); + writeln!(f, "{:- String { const KB: f64 = 1024.0; const MB: f64 = 1024.0 * KB; diff --git a/src/table/vacuum/fragmentation_info.rs b/src/table/vacuum/fragmentation_info.rs index 6541e014..68d0fb92 100644 --- a/src/table/vacuum/fragmentation_info.rs +++ b/src/table/vacuum/fragmentation_info.rs @@ -12,7 +12,8 @@ //! //! [`WorkTable`]: crate::table::WorkTable -use std::collections::HashMap; +use alloc::vec::Vec; +use hashbrown::HashMap; use data_bucket::page::PageId; use data_bucket::{INNER_PAGE_SIZE, Link}; diff --git a/src/table/vacuum/manager.rs b/src/table/vacuum/manager.rs index 566ed452..01e7cee3 100644 --- a/src/table/vacuum/manager.rs +++ b/src/table/vacuum/manager.rs @@ -1,8 +1,11 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; -use tokio::task::AbortHandle; +use alloc::sync::Arc; +use alloc::{string::ToString, vec::Vec}; +use core::sync::atomic::{AtomicU64, Ordering}; +use core::time::Duration; +use hashbrown::HashMap; +// The task handle is nagoya's now: dropping it detaches, `cancel` stops it, +// which is the same contract `AbortHandle` had here. +use nagoya::JoinHandle; use parking_lot::RwLock; use smart_default::SmartDefault; @@ -123,12 +126,26 @@ impl VacuumManager { ) } - /// Starts a background task that periodically checks fragmentation and runs - /// vacuum. + /// Starts the sweep task. Returns a handle whose `cancel` stops it. /// - /// Returns an `AbortHandle` that can be used to cancel the task. - pub fn run_vacuum_task(self: Arc) -> AbortHandle { - let handle = tokio::spawn(async move { + /// It does not poll: it parks on the registered tables until one of them + /// frees enough space to be worth a sweep, with [`FALLBACK_INTERVAL`] only + /// bounding how long a table that never reaches its threshold goes + /// unlooked-at. + /// + /// A background task even so, and not folded into whichever mutation freed + /// the space, because a sweep waits for the table to go *quiet* before it + /// takes the registry exclusion (see [`VacuumPacing::wait_until_quiet`]). + /// A foreground mutation running its own sweep would be waiting on its own + /// quiescence. + /// + /// [`VacuumPacing::wait_until_quiet`]: crate::vacuum::VacuumPacing + #[cfg(feature = "std")] + pub fn run_vacuum_task(self: Arc) -> JoinHandle<()> { + // The engine's pool, not nagoya's process-wide one: see + // `runtime::engine_executor` for why the sweep has to follow whatever + // the client tasks were put on. + crate::runtime::engine_executor().spawn(async move { loop { self.wait_for_work().await; @@ -230,7 +247,7 @@ impl VacuumManager { } // The persistence worker's turn. See the // note above the loop. - tokio::time::sleep(BETWEEN_PASSES).await; + nagoya::sleep(BETWEEN_PASSES).await; } Err(e) => { // println!("Vacuum failed for table '{}': {}", table_name, e); @@ -243,9 +260,7 @@ impl VacuumManager { } } } - }); - - handle.abort_handle() + }) } /// Blocks until some registered table has freed enough space to be worth @@ -256,14 +271,15 @@ impl VacuumManager { vacuums.values().cloned().collect() }; if registered.is_empty() { - tokio::time::sleep(FALLBACK_INTERVAL).await; + nagoya::sleep(FALLBACK_INTERVAL).await; return; } let waits: Vec<_> = registered.iter().map(|v| v.wait_until_worth_running()).collect(); - tokio::select! { - _ = futures::future::select_all(waits) => {} - _ = tokio::time::sleep(FALLBACK_INTERVAL) => {} - } + // This was a two-armed `tokio::select!` racing the waits against a + // sleep, which is what a timeout is. Saying `timeout` says the intent + // and costs no combinator: the fallback exists so a table that never + // becomes worth vacuuming is still looked at eventually. + let _ = nagoya::timeout(FALLBACK_INTERVAL, futures::future::select_all(waits)).await; } } diff --git a/src/table/vacuum/mod.rs b/src/table/vacuum/mod.rs index 74fc55db..efc4855d 100644 --- a/src/table/vacuum/mod.rs +++ b/src/table/vacuum/mod.rs @@ -1,3 +1,5 @@ +use alloc::boxed::Box; +use alloc::vec::Vec; use async_trait::async_trait; use data_bucket::Link; @@ -37,6 +39,7 @@ pub trait VacuumPersistence: Send + Sync { fn apply_move( &self, bytes: Vec, + old_link: Link, new_link: Link, primary_key_events: Vec>>, secondary_keys_events: SecondaryEvents, diff --git a/src/table/vacuum/pacing.rs b/src/table/vacuum/pacing.rs index 1ff64c9d..80427e97 100644 --- a/src/table/vacuum/pacing.rs +++ b/src/table/vacuum/pacing.rs @@ -15,8 +15,8 @@ //! the preceding check. Every insert, delete and upsert passes through those //! stripes, including mutations that never ask for reclaimable space. -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::time::Duration; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use core::time::Duration; use smart_default::SmartDefault; @@ -114,7 +114,7 @@ pub trait ForegroundActivity { impl ForegroundActivity for crate::lock::LockMap where - PrimaryKey: Clone + std::fmt::Debug + Eq + std::hash::Hash, + PrimaryKey: Clone + core::fmt::Debug + Eq + core::hash::Hash, { fn mutations_in_flight(&self) -> usize { crate::lock::LockMap::mutations_in_flight(self) @@ -132,7 +132,7 @@ impl VacuumPacing { /// once even on an idle table, so a waiting insert gets the registry /// before vacuum asks for it back. pub(crate) async fn wait_until_quiet(&self, activity: &impl ForegroundActivity, gate: &VacuumGate) { - tokio::task::yield_now().await; + nagoya::yield_now().await; let mut backoff = self.backoff; let mut quiet = 0; @@ -143,7 +143,7 @@ impl VacuumPacing { gate.note_stand_down(); quiet = 0; observed_epoch = current_epoch; - tokio::time::sleep(backoff).await; + nagoya::sleep(backoff).await; // Doubling, so a table busy for a long time is asked about // cheaply rather than every couple of milliseconds. backoff = backoff.saturating_mul(2).min(self.max_backoff); @@ -156,15 +156,15 @@ impl VacuumPacing { } // Idle once is a gap between two writes. Look again, close // together, before believing it. - tokio::time::sleep(self.backoff).await; + nagoya::sleep(self.backoff).await; } } } #[cfg(test)] mod tests { - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use super::*; @@ -208,7 +208,7 @@ mod tests { }) }; - tokio::time::sleep(Duration::from_millis(10)).await; + nagoya::sleep(Duration::from_millis(10)).await; assert!( !waiting.is_finished(), "activity between snapshots must keep vacuum out" diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index 47030bca..91294e1a 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -1,9 +1,12 @@ -use std::collections::VecDeque; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; +use alloc::boxed::Box; +use alloc::collections::VecDeque; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicU64, Ordering}; +use core::time::Duration; +use std::time::Instant; /// How long retirements have to stop arriving for a delete burst to count as /// over. Short enough that a sweep still follows a delete promptly, long @@ -85,7 +88,7 @@ pub struct EmptyDataVacuum< const DATA_LENGTH: usize, SecondaryEvents = (), > where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static + Debug, PkMap: UniqueIndex>, { @@ -139,7 +142,7 @@ impl< > where Row: TableRow + StorableRow + Send + Clone + 'static, - PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + core::hash::Hash, PkMap: UniqueIndex>, ::WrappedRow: RowWrapper, Row: Archive @@ -217,7 +220,7 @@ where let deadline = Instant::now() + MAX_SETTLE; loop { let before = self.data_pages.pending_retirements(); - tokio::time::sleep(SETTLE_INTERVAL).await; + nagoya::sleep(SETTLE_INTERVAL).await; if self.data_pages.pending_retirements() == before || Instant::now() >= deadline { return; } @@ -648,7 +651,7 @@ where .reinsert_row_cdc(row.clone(), old_link, row, new_link); res.expect("should be ok as index were no violated"); let (_, primary_key_events) = self.primary_index.insert_cdc(pk.clone(), new_link); - persistence.apply_move(raw_data, new_link, primary_key_events, secondary_keys_events)?; + persistence.apply_move(raw_data, old_link, new_link, primary_key_events, secondary_keys_events)?; } else { self.secondary_indexes .reinsert_row(row.clone(), old_link, row, new_link) @@ -684,7 +687,7 @@ impl< > where Row: TableRow + StorableRow + Send + Sync + Clone + 'static, - PrimaryKey: Debug + Clone + Ord + Send + Sync + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + Sync + TablePrimaryKey + core::hash::Hash, PkMap: UniqueIndex> + Send + Sync + 'static, ::WrappedRow: RowWrapper, Row: Archive @@ -737,9 +740,10 @@ where #[cfg(test)] mod tests { - use std::collections::{HashMap, VecDeque}; - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use alloc::collections::VecDeque; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use hashbrown::HashMap; use data_bucket::Link; use data_bucket::page::PageId; @@ -747,7 +751,7 @@ mod tests { use crate::in_memory::{ArchivedRowWrapper, RowWrapper, StorableRow}; use crate::prelude::*; - use std::time::Duration; + use core::time::Duration; use crate::vacuum::vacuum::{CandidateMove, EmptyDataVacuum}; use crate::vacuum::{VacuumGate, VacuumPacing, WorkTableVacuum}; diff --git a/src/util/mod.rs b/src/util/mod.rs index 7b1adf10..ac5cc7a4 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,8 +1,22 @@ pub(crate) mod epoch; mod offset_eq_link; +#[cfg(feature = "std")] mod optimized_vec; mod ordered_float; pub use offset_eq_link::OffsetEqLink; +#[cfg(feature = "std")] pub use optimized_vec::OptimizedVec; pub use ordered_float::{OrderedF32Def, OrderedF64Def}; + +/// Give up the rest of the timeslice after a spin has stopped paying. +/// +/// Under `std` that is the operating system's yield. Without one there is no +/// scheduler to yield to, so the spin hint is the whole of what can be done. +#[inline] +pub(crate) fn yield_now() { + #[cfg(feature = "std")] + std::thread::yield_now(); + #[cfg(not(feature = "std"))] + core::hint::spin_loop(); +} diff --git a/src/util/offset_eq_link.rs b/src/util/offset_eq_link.rs index 0b39d0db..a1c3c3c5 100644 --- a/src/util/offset_eq_link.rs +++ b/src/util/offset_eq_link.rs @@ -22,20 +22,20 @@ impl OffsetEqLink { } } -impl std::hash::Hash for OffsetEqLink { - fn hash(&self, state: &mut H) { +impl core::hash::Hash for OffsetEqLink { + fn hash(&self, state: &mut H) { self.absolute_index().hash(state); } } impl PartialOrd for OffsetEqLink { - fn partial_cmp(&self, other: &Self) -> Option { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for OffsetEqLink { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { self.absolute_index().cmp(&other.absolute_index()) } } @@ -48,7 +48,7 @@ impl PartialEq for OffsetEqLink { impl Eq for OffsetEqLink {} -impl std::ops::Deref for OffsetEqLink { +impl core::ops::Deref for OffsetEqLink { type Target = Link; fn deref(&self) -> &Self::Target { @@ -85,7 +85,7 @@ impl SizeMeasurable for OffsetEqLink { mod tests { use super::*; use data_bucket::page::PageId; - use std::collections::HashSet; + use hashbrown::HashSet; const TEST_DATA_LENGTH: usize = 4096; diff --git a/src/util/optimized_vec.rs b/src/util/optimized_vec.rs index c26114d0..9570ec9b 100644 --- a/src/util/optimized_vec.rs +++ b/src/util/optimized_vec.rs @@ -1,3 +1,4 @@ +use alloc::vec::Vec; /// Struct for storing data in a vector with stable indexes and slot reuse. /// Slots are `Option`: `remove` is `Option::take`, so the value moves out /// without a `Clone` bound and the slot is freed immediately. The previous @@ -201,7 +202,7 @@ mod tests { /// count proves the removed value is the only remaining owner. #[test] fn test_optimized_vec_remove_moves_without_clone() { - use std::rc::Rc; + use alloc::rc::Rc; struct NotClone(#[allow(dead_code)] Rc<()>); diff --git a/src/util/ordered_float.rs b/src/util/ordered_float.rs index 598be510..29ff13e3 100644 --- a/src/util/ordered_float.rs +++ b/src/util/ordered_float.rs @@ -4,7 +4,7 @@ use rkyv::{Archive, Deserialize, Serialize}; #[rkyv(remote = ordered_float::OrderedFloat, archived = ArchivedF64)] #[rkyv(derive(Debug))] pub struct OrderedF64Def { - #[rkyv(getter = std::ops::Deref::deref)] + #[rkyv(getter = core::ops::Deref::deref)] value: f64, } @@ -18,7 +18,7 @@ impl From for ordered_float::OrderedFloat { #[rkyv(remote = ordered_float::OrderedFloat, archived = ArchivedF32)] #[rkyv(derive(Debug))] pub struct OrderedF32Def { - #[rkyv(getter = std::ops::Deref::deref)] + #[rkyv(getter = core::ops::Deref::deref)] value: f32, } diff --git a/src/vec_hydrate.rs b/src/vec_hydrate.rs new file mode 100644 index 00000000..4e42b54e --- /dev/null +++ b/src/vec_hydrate.rs @@ -0,0 +1,831 @@ +//! Load and unload a `worktable_vec!` table as pages. +//! +//! # What this is +//! +//! `worktable_vec!` drops paging because a `Vec` does not need it while the +//! table is in use. It still needs a way to put rows on a disk and get them +//! back, and that is a codec rather than a storage engine: rows live in a +//! `Vec` and are pages only at rest. Everything between a load and an unload +//! runs at `Vec` speed because it *is* a `Vec`. +//! +//! This is ported from `worktable-vec`'s `hydrate` module. Corruption and append +//! tests live here alongside generated-table integration tests. It is reproduced rather +//! than depended on because a dependency would invert the direction this is +//! meant to travel: WorkTable is meant to absorb that crate, not require it. +//! +//! # Page based, and each page stands alone +//! +//! A page is 16 KiB: a 28 byte header, then an rkyv archive of **the rows that +//! fit in that page**, then a 12 byte directory at the tail. Nothing spans a +//! boundary. +//! +//! That is the whole design. An archive split across pages means one damaged +//! page destroys every row in the file, and it means appending a row rewrites +//! everything. Self-contained pages make damage local and appends O(new rows), +//! and cost only the few bytes of archive overhead repeated per page. +//! +//! # What is checked +//! +//! Every page carries a CRC-32 covering all bytes except the checksum, and every header field is +//! validated rather than merely written. rkyv's own validation checks that an +//! archive is structurally sound, which is not the same as checking that these +//! are the bytes that were written: a flipped bit inside a `u64` passes +//! structural validation and reads back as a different number. The checksum is +//! what catches that. +//! +//! # These files are not interchangeable with `worktable-vec`'s +//! +//! That crate stores `Vec<(K, V)>`, because its value type has no key in it. A +//! `worktable_vec!` row is a named struct that already carries its primary key +//! as a column, so this stores `Vec` and does not write the key twice. +//! Different types, different rkyv archives, different fingerprints. +//! +//! The fingerprint is what makes that safe rather than merely true: a foreign +//! file is refused by [`LoadError::ForeignRows`] instead of being read as +//! debris. Do not expect a file written by one to open in the other. +//! +//! # These are not WorkTable space files either +//! +//! A WorkTable space opens with a page carrying a name, a schema and a primary +//! key list. These pages use a distinct page type (4, archived rows), a zero +//! space id and a row-type fingerprint in the trailer. Neither reader accepts +//! the other container as its own. + +use alloc::vec::Vec; + +use rkyv::api::high::{HighDeserializer, HighValidator}; +use rkyv::bytecheck::CheckBytes; +use rkyv::rancor::{Error as RkyvError, Strategy}; +use rkyv::ser::Serializer; +use rkyv::ser::allocator::ArenaHandle; +use rkyv::ser::sharing::Share; +use rkyv::util::AlignedVec; +use rkyv::{Archive, Deserialize, Serialize}; + +/// One page, header included. +pub const PAGE_SIZE: usize = 4096 * 4; + +/// DataBucket's `GENERAL_HEADER_SIZE`, which this page opens with. +pub const HEADER_SIZE: usize = 28; + +/// The page trailer: row count, row-type fingerprint and CRC-32, all little endian. +pub const DIRECTORY_SIZE: usize = 12; + +/// How much of a page is body, between the header and the directory. +pub const BODY_SIZE: usize = PAGE_SIZE - HEADER_SIZE - DIRECTORY_SIZE; + +/// `DATA_VERSION` 3: DataBucket's page framing, plus a row directory. +/// +/// This identifies the Vec snapshot framing. Ordinary WorkTable spaces also +/// use version 3, but start with SpaceInfo metadata and use a different row +/// directory layout. The two containers are not interchangeable. +pub const PAGE_VERSION: u32 = 3; + +/// Archived row batches, distinct from DataBucket's ordinary Data pages (2). +const PAGE_TYPE_ARCHIVED_ROWS: u32 = 4; + +/// What a load can refuse on. +/// +/// Every variant is a statement about the bytes rather than about the caller, +/// and every one names the page, because a file that will not load is a +/// question about which page went wrong. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LoadError { + /// The byte length is not a whole number of pages. + NotWholePages { + /// How many bytes arrived. + found: usize, + }, + /// A page carries a version this build does not write. + /// + /// Also what a page of zeroes looks like, which is the shape a torn write + /// leaves behind. + ForeignPages { + /// Which page, counting from zero. + page: usize, + /// The version that page claims. + version: u32, + }, + /// A page belongs to a different container. + ForeignPageType { + /// Position in the supplied byte slice. + page: usize, + /// Type carried by the header. + page_type: u32, + }, + /// The space id, page number or chain links are invalid. + PageIdentity { + /// Position in the supplied byte slice. + page: usize, + }, + /// A header claimed a body longer than a page holds. + Overlong { + /// Which page, counting from zero. + page: usize, + /// What its header claimed. + claimed: usize, + }, + /// The body does not match the checksum written with it. + /// + /// This is the one rkyv cannot find. A flipped bit inside an integer is a + /// structurally perfect archive of the wrong number. + Corrupt { + /// Which page, counting from zero. + page: usize, + /// The checksum written with the body. + expected: u32, + /// The checksum of the bytes actually there. + found: u32, + }, + /// The pages disagree with each other about the row type. + Inconsistent { + /// Which page disagreed. + page: usize, + }, + /// These are a different row type's bytes. + /// + /// Caught by a fingerprint rather than by deserialization, because + /// deserialization does not catch it: rkyv validates a `(u64, String)` + /// archive as a perfectly good `(u64, u64)` and hands back a `String`'s + /// relative pointer as an integer. Keys look right, values are debris, and + /// nothing errors. + ForeignRows { + /// The fingerprint these bytes were written with. + found: u32, + /// The fingerprint this row type expects. + expected: u32, + }, + /// A page's rows did not deserialize. + Rows { + /// Which page, counting from zero. + page: usize, + }, + /// A page's directory promised a row count its body did not contain. + RowCount { + /// Which page, counting from zero. + page: usize, + /// What the directory promised. + expected: usize, + /// What the body held. + found: usize, + }, +} + +impl core::fmt::Display for LoadError { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::NotWholePages { found } => { + write!( + formatter, + "{found} bytes is not a whole number of {PAGE_SIZE} byte pages" + ) + } + Self::ForeignPages { page, version } => { + write!( + formatter, + "page {page} claims format version {version}, not {PAGE_VERSION}" + ) + } + Self::Overlong { page, claimed } => { + write!( + formatter, + "page {page} claims a {claimed} byte body, over the {BODY_SIZE} byte limit" + ) + } + Self::ForeignPageType { page, page_type } => write!( + formatter, + "page {page} has type {page_type}, not archived rows ({PAGE_TYPE_ARCHIVED_ROWS})" + ), + Self::PageIdentity { page } => write!(formatter, "page {page} has invalid identity or chain links"), + Self::Corrupt { page, expected, found } => { + write!( + formatter, + "page {page} checksums to {found:#010x}, not the {expected:#010x} written with it" + ) + } + Self::Inconsistent { page } => { + write!( + formatter, + "page {page} names a different row type than the pages before it" + ) + } + Self::ForeignRows { found, expected } => { + write!( + formatter, + "these pages hold row type {found:#010x}, not {expected:#010x}" + ) + } + Self::Rows { page } => write!(formatter, "page {page} did not deserialize into rows"), + Self::RowCount { page, expected, found } => { + write!(formatter, "page {page} promised {expected} rows and held {found}") + } + } + } +} + +impl core::error::Error for LoadError {} + +/// One row does not fit in a page, so the file was not written. +/// +/// Refused rather than written, because the writer used to produce a file +/// `load` then refused: `unload` reported success and the rows were gone. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RowTooLarge { + /// Which row, counting from zero. + pub row: usize, + /// How many bytes its archive needed. + pub bytes: usize, + /// How many a page body holds. + pub limit: usize, +} + +impl core::fmt::Display for RowTooLarge { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + formatter, + "row {} needs {} bytes and a page body holds {}", + self.row, self.bytes, self.limit + ) + } +} + +impl core::error::Error for RowTooLarge {} + +/// Failure to encode a snapshot segment with an explicit starting page number. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum UnloadError { + /// An individual row cannot fit. + RowTooLarge(RowTooLarge), + /// A page number would exceed the format's u32 range. + PageIndexOverflow, +} + +impl core::fmt::Display for UnloadError { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::RowTooLarge(error) => error.fmt(formatter), + Self::PageIndexOverflow => formatter.write_str("snapshot page number exceeds u32"), + } + } +} + +impl core::error::Error for UnloadError {} + +/// The one thing [`Codec::decode`] can say. +/// +/// Which page it happened on is the caller's to add, because a codec does not +/// know it is reading a page. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct NotAnArchive; + +/// Rows to bytes and back. +/// +/// Blanket-implemented, so anything deriving rkyv's traits satisfies it and +/// the generated row needs no separate impl. +pub trait Codec: Sized { + /// Rows to bytes. + fn encode(&self) -> AlignedVec<16>; + /// Bytes back to rows. + /// + /// # Errors + /// + /// Fails when the bytes are not this type's archive. + fn decode(bytes: &[u8]) -> Result; +} + +impl Codec for T +where + T: Archive + for<'a> Serialize, ArenaHandle<'a>, Share>, RkyvError>>, + ::Archived: + Deserialize> + for<'a> CheckBytes>, +{ + fn encode(&self) -> AlignedVec<16> { + // Infallible in practice: the only failure rkyv reports here is an + // allocator refusing, which on this path means the process is already + // out of memory. + rkyv::to_bytes::(self).expect("rows serialize") + } + + fn decode(bytes: &[u8]) -> Result { + rkyv::from_bytes::(bytes).map_err(|_| NotAnArchive) + } +} + +/// What row type wrote these bytes. +/// +/// FNV-1a over `core::any::type_name`, which is neither stable across compiler +/// versions nor guaranteed unique. That is fine for what it is for: refusing +/// an obvious mismatch, not authenticating a schema. A false match is possible +/// and a false mismatch is a rebuild, so it fails toward refusing to load +/// rather than toward reinterpreting. +pub fn fingerprint() -> u32 { + let mut hash: u32 = 0x811c_9dc5; + for byte in core::any::type_name::().as_bytes() { + hash ^= u32::from(*byte); + hash = hash.wrapping_mul(0x0100_0193); + } + hash +} + +/// CRC-32 using the existing no-default-features checksum dependency. +fn crc32(bytes: &[u8]) -> u32 { + crc32fast::hash(bytes) +} + +/// DataBucket's `GeneralHeader`, byte for byte. +/// +/// Seven little-endian `u32`s in declaration order, which is what +/// `rkyv::to_bytes` of that struct produces: no relative pointers, and +/// `page_type` padded from `u16` to four bytes. For a `Data` page of space 3, +/// id 7, previous 6, next 8, length `0x11223344`: +/// +/// ```text +/// 02000000 03000000 07000000 06000000 08000000 02000000 44332211 +/// version space page previous next type length +/// ``` +/// +/// Written out here rather than imported from `data_bucket`, which is `std`. +/// The bytes above document the shared framing. The archived-rows page type +/// and trailer distinguish this container from ordinary DataBucket spaces. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Header { + /// `DATA_VERSION`. See [`PAGE_VERSION`]. + version: u32, + /// Zero: standalone snapshots do not belong to a WorkTable space. + space: u32, + page: u32, + previous: u32, + next: u32, + /// Archived row batches, which is 4. + page_type: u32, + /// Bytes of row archive in this page, before the directory. + body: u32, +} + +impl Header { + fn write(self, out: &mut Vec) { + for field in [ + self.version, + self.space, + self.page, + self.previous, + self.next, + self.page_type, + self.body, + ] { + out.extend_from_slice(&field.to_le_bytes()); + } + } + + fn read(raw: &[u8]) -> Self { + let at = |n: usize| { + let mut word = [0u8; 4]; + word.copy_from_slice(&raw[n * 4..n * 4 + 4]); + u32::from_le_bytes(word) + }; + Self { + version: at(0), + space: at(1), + page: at(2), + previous: at(3), + next: at(4), + page_type: at(5), + body: at(6), + } + } +} + +/// The row directory, at the tail of every page. +/// +/// This count describes the archived row batch. Ordinary WorkTable v3 data +/// pages instead use an offset/length directory per live row. Both containers +/// describe their rows locally, but their directory layouts are different. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Directory { + rows: u32, + schema: u32, + crc: u32, +} + +impl Directory { + fn write(self, page: &mut [u8]) { + let at = page.len() - DIRECTORY_SIZE; + page[at..at + 4].copy_from_slice(&self.rows.to_le_bytes()); + page[at + 4..at + 8].copy_from_slice(&self.schema.to_le_bytes()); + page[at + 8..].copy_from_slice(&self.crc.to_le_bytes()); + } + + fn read(page: &[u8]) -> Self { + let at = page.len() - DIRECTORY_SIZE; + let word = |n: usize| { + let mut bytes = [0u8; 4]; + bytes.copy_from_slice(&page[n..n + 4]); + u32::from_le_bytes(bytes) + }; + Self { + rows: word(at), + schema: word(at + 4), + crc: word(at + 8), + } + } +} + +/// The most rows of `rows` whose archive fits one page body. +/// +/// **Bounded probes.** The obvious version binary searches over the whole +/// remaining slice, which re-serializes every row still to be written on every +/// probe, for every page. In `worktable-vec` that measured 2.3 seconds to +/// write what rkyv alone encodes in 6.7 ms, because the work is quadratic in +/// the row count. +/// +/// So the search is bounded to roughly two pages of rows: one sample encode +/// gives bytes per row, the estimate from that sets the ceiling, and the +/// binary search runs under it. Every probe serializes about a page, never a +/// file. Uniform rows land in a probe or two and wildly variable rows still +/// terminate, because the ceiling is only a ceiling. +/// +/// Always returns at least one for a non-empty slice, so the caller always +/// makes progress. A single row too large for a page is caught by the writer +/// rather than looping here forever. +fn rows_per_page(rows: &[R], hint: usize) -> (usize, AlignedVec<16>) +where + Vec: Codec, + R: Clone, +{ + if rows.is_empty() { + return (0, rows.to_vec().encode()); + } + + let mut best = None; + let mut fits = |take: usize| { + let archive = rows[..take].to_vec().encode(); + if archive.len() <= BODY_SIZE { + best = Some((take, archive)); + true + } else { + false + } + }; + + // A page holds about what the last one held, so start there and walk. + // Uniform rows settle in a probe or two; only the first page, or a run + // whose rows change size, pays for a search. + if hint > 0 && hint <= rows.len() && fits(hint) { + let mut take = hint; + while take < rows.len() && fits(take + 1) { + take += 1; + } + return best.expect("the successful hint supplied an archive"); + } + + // No usable hint, or the rows grew. One sample gives bytes per row, and + // the estimate from it bounds the search to about two pages of rows. + let sample = rows.len().min(64); + let sampled = rows[..sample].to_vec().encode().len(); + let estimate = (BODY_SIZE * sample) + .checked_div(sampled) + .map_or(rows.len(), |estimate| estimate.max(1)); + let mut low = 1usize; + let mut high = rows.len().min(estimate.saturating_mul(2)).max(1); + while low < high { + let mid = low + (high - low).div_ceil(2); + if fits(mid) { + low = mid; + } else { + high = mid - 1; + } + } + best.filter(|(take, _)| *take == low) + .unwrap_or_else(|| (low, rows[..low].to_vec().encode())) +} + +/// Rows to pages, each page standing alone. +/// +/// # Errors +/// +/// [`RowTooLarge`] when one row's archive does not fit a page body. Nothing is +/// written in that case. +pub fn to_pages(rows: &[R]) -> Result, RowTooLarge> +where + Vec: Codec, + R: Clone, +{ + match to_pages_at(rows, 0) { + Ok(bytes) => Ok(bytes), + Err(UnloadError::RowTooLarge(error)) => Err(error), + // A Vec holding over u32::MAX 16 KiB pages would require over 64 TiB. + Err(UnloadError::PageIndexOverflow) => panic!("snapshot exceeds u32 page count"), + } +} + +/// Encode an append segment, numbering its pages from `first_page`. +/// +/// Pass the existing file length divided by [`PAGE_SIZE`]. The preceding +/// segment remains terminal; appending does not rewrite its last page. +/// Only new rows belong in this segment. Updates and deletes require a full +/// snapshot. A standalone segment can be inspected with [`from_pages`]. +/// +/// # Errors +/// +/// Refuses oversized rows and page-number overflow without producing bytes. +pub fn to_pages_at(rows: &[R], first_page: u32) -> Result, UnloadError> +where + Vec: Codec, + R: Clone, +{ + let schema = fingerprint::>(); + let mut out = Vec::new(); + let mut rest = rows; + let mut hint = 0usize; + + // An empty table still writes one page. A zero byte file is + // indistinguishable from a missing one, and a load has to tell "no rows" + // from "nothing landed". + loop { + let (take, archive) = rows_per_page(rest, hint); + hint = take; + let body = archive.as_ref(); + // `rows_per_page` returns at least one so the loop always advances, so + // a body over the limit means that one row does not fit a page. + if body.len() > BODY_SIZE { + return Err(UnloadError::RowTooLarge(RowTooLarge { + row: rows.len() - rest.len(), + bytes: body.len(), + limit: BODY_SIZE, + })); + } + let page = u32::try_from(out.len() / PAGE_SIZE) + .ok() + .and_then(|offset| first_page.checked_add(offset)) + .ok_or(UnloadError::PageIndexOverflow)?; + let last = rest.len() == take; + let next = if last { + page + } else { + page.checked_add(1).ok_or(UnloadError::PageIndexOverflow)? + }; + Header { + version: PAGE_VERSION, + space: 0, + page, + previous: page.saturating_sub(1), + // A last page points at itself, so a chain walker stops rather + // than running off the end. + next, + page_type: PAGE_TYPE_ARCHIVED_ROWS, + body: u32::try_from(body.len()).expect("a body inside u32"), + } + .write(&mut out); + out.extend_from_slice(body); + out.resize(out.len().next_multiple_of(PAGE_SIZE), 0); + + // The directory goes in last, into the tail of the page just written. + let start = out.len() - PAGE_SIZE; + Directory { + rows: u32::try_from(take).expect("a row count inside u32"), + schema, + crc: 0, + } + .write(&mut out[start..]); + let crc = crc32(&out[start..out.len() - 4]); + let end = out.len(); + out[end - 4..].copy_from_slice(&crc.to_le_bytes()); + + rest = &rest[take..]; + if rest.is_empty() { + break; + } + } + Ok(out) +} + +/// One page back into rows, with every header field checked. +fn page_rows( + raw: &[u8], + index: usize, + schema: &mut Option, + previous: Option

, + last: bool, +) -> Result<(Vec, Header), LoadError> +where + Vec: Codec, +{ + let header = Header::read(&raw[..HEADER_SIZE]); + if header.version != PAGE_VERSION { + return Err(LoadError::ForeignPages { + page: index, + version: header.version, + }); + } + if header.page_type != PAGE_TYPE_ARCHIVED_ROWS { + return Err(LoadError::ForeignPageType { + page: index, + page_type: header.page_type, + }); + } + let directory = Directory::read(raw); + let found = crc32(&raw[..PAGE_SIZE - 4]); + if found != directory.crc { + return Err(LoadError::Corrupt { + page: index, + expected: directory.crc, + found, + }); + } + let valid_predecessor = previous.is_none_or(|before| { + if before.next == before.page { + // Independent unloads restart at zero. Explicit append segments + // continue numbering without rewriting the previous terminal page. + header.page == 0 || before.page.checked_add(1) == Some(header.page) + } else { + header.page == before.next + } + }); + if header.space != 0 + || header.previous != header.page.saturating_sub(1) + || !(header.next == header.page || header.page.checked_add(1) == Some(header.next)) + || !valid_predecessor + || (last && header.next != header.page) + { + return Err(LoadError::PageIdentity { page: index }); + } + match schema { + None => *schema = Some(directory.schema), + // Every page names the row type, so a file spliced onto another is + // caught where they stop agreeing rather than concatenated. + Some(first) if *first != directory.schema => { + return Err(LoadError::Inconsistent { page: index }); + } + Some(_) => {} + } + let expected = fingerprint::>(); + if directory.schema != expected { + return Err(LoadError::ForeignRows { + found: directory.schema, + expected, + }); + } + + let take = header.body as usize; + if take > BODY_SIZE { + return Err(LoadError::Overlong { + page: index, + claimed: take, + }); + } + let body = &raw[HEADER_SIZE..HEADER_SIZE + take]; + + // Copied into an AlignedVec because rkyv reads an archive in place and + // needs it aligned. A page body sits at a header's offset into a Vec, + // which is aligned to nothing in particular. + let mut aligned = AlignedVec::<16>::with_capacity(take); + aligned.extend_from_slice(body); + let rows = Vec::::decode(&aligned).map_err(|NotAnArchive| LoadError::Rows { page: index })?; + if rows.len() != directory.rows as usize { + return Err(LoadError::RowCount { + page: index, + expected: directory.rows as usize, + found: rows.len(), + }); + } + Ok((rows, header)) +} + +/// Every page back into one row vector. +/// +/// # Errors +/// +/// [`LoadError`], naming the page that went wrong. +pub fn from_pages(bytes: &[u8]) -> Result, LoadError> +where + Vec: Codec, +{ + if bytes.is_empty() || !bytes.len().is_multiple_of(PAGE_SIZE) { + return Err(LoadError::NotWholePages { found: bytes.len() }); + } + + let mut schema = None; + let mut rows = Vec::new(); + let mut previous = None; + for (index, raw) in bytes.as_chunks::().0.iter().enumerate() { + let last = index + 1 == bytes.len() / PAGE_SIZE; + let (mut page, header) = page_rows(raw, index, &mut schema, previous, last)?; + rows.append(&mut page); + previous = Some(header); + } + Ok(rows) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set_word(page: &mut [u8], offset: usize, value: u32) { + page[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + + fn checksum(page: &mut [u8]) { + let crc = crc32(&page[..PAGE_SIZE - 4]); + set_word(page, PAGE_SIZE - 4, crc); + } + + #[test] + fn corruption_in_header_body_padding_and_directory_is_refused() { + let bytes = to_pages(&[42u64, 97]).unwrap(); + for offset in (0..HEADER_SIZE).chain([ + HEADER_SIZE, + HEADER_SIZE + 8, + PAGE_SIZE / 2, + PAGE_SIZE - 12, + PAGE_SIZE - 8, + PAGE_SIZE - 4, + PAGE_SIZE - 1, + ]) { + let mut damaged = bytes.clone(); + damaged[offset] ^= 1; + assert!(from_pages::(&damaged).is_err(), "accepted damage at {offset}"); + } + assert_eq!(from_pages::(&bytes).unwrap(), [42, 97]); + } + + #[test] + fn ordinary_data_pages_and_invalid_links_are_refused_even_with_valid_crc() { + let bytes = to_pages(&[42u64]).unwrap(); + let mut foreign = bytes.clone(); + set_word(&mut foreign, 20, 2); + checksum(&mut foreign); + assert!(matches!( + from_pages::(&foreign), + Err(LoadError::ForeignPageType { page_type: 2, .. }) + )); + for (offset, value) in [(4, 1), (8, 1), (12, 1), (16, 1)] { + let mut damaged = bytes.clone(); + set_word(&mut damaged, offset, value); + checksum(&mut damaged); + assert!(matches!( + from_pages::(&damaged), + Err(LoadError::PageIdentity { .. }) + )); + } + } + + #[test] + fn page_omission_reordering_and_truncation_are_refused() { + let rows: Vec = (0..8_000).collect(); + let bytes = to_pages(&rows).unwrap(); + assert!(bytes.len() >= PAGE_SIZE * 3); + assert_eq!(from_pages::(&bytes).unwrap(), rows); + assert!(from_pages::(&bytes[..bytes.len() - PAGE_SIZE]).is_err()); + let mut omitted = bytes[..PAGE_SIZE].to_vec(); + omitted.extend_from_slice(&bytes[PAGE_SIZE * 2..]); + assert!(from_pages::(&omitted).is_err()); + let mut swapped = bytes.clone(); + swapped[..PAGE_SIZE].copy_from_slice(&bytes[PAGE_SIZE..2 * PAGE_SIZE]); + swapped[PAGE_SIZE..2 * PAGE_SIZE].copy_from_slice(&bytes[..PAGE_SIZE]); + assert!(from_pages::(&swapped).is_err()); + } + + #[test] + fn append_segments_keep_existing_pages_and_allow_independent_snapshots() { + let first: Vec = (0..4_000).collect(); + let next: Vec = (4_000..8_000).collect(); + let mut bytes = to_pages(&first).unwrap(); + let original = bytes.clone(); + let segment = to_pages_at(&next, (bytes.len() / PAGE_SIZE) as u32).unwrap(); + assert_eq!(from_pages::(&segment).unwrap(), next); + bytes.extend_from_slice(&segment); + assert_eq!(&bytes[..original.len()], &original); + assert_eq!(from_pages::(&bytes).unwrap(), (0..8_000).collect::>()); + bytes.extend_from_slice(&to_pages(&[8_000u64]).unwrap()); + assert_eq!(from_pages::(&bytes).unwrap(), (0..8_001).collect::>()); + } + + #[test] + fn append_page_overflow_is_a_reported_error() { + let rows: Vec = (0..4_000).collect(); + assert_eq!(to_pages_at(&rows, u32::MAX), Err(UnloadError::PageIndexOverflow)); + let last = to_pages_at(&[1u64], u32::MAX).unwrap(); + assert_eq!(from_pages::(&last).unwrap(), [1]); + } + + #[test] + fn trailer_schema_and_count_are_validated_before_accepting_rows() { + let bytes = to_pages(&[42u64]).unwrap(); + assert!(matches!(from_pages::(&bytes), Err(LoadError::ForeignRows { .. }))); + let mut wrong_count = bytes.clone(); + set_word(&mut wrong_count, PAGE_SIZE - DIRECTORY_SIZE, 2); + checksum(&mut wrong_count); + assert!(matches!( + from_pages::(&wrong_count), + Err(LoadError::RowCount { + expected: 2, + found: 1, + .. + }) + )); + let mut overlong = bytes; + set_word(&mut overlong, 24, PAGE_SIZE as u32); + checksum(&mut overlong); + assert!(matches!(from_pages::(&overlong), Err(LoadError::Overlong { .. }))); + } +} diff --git a/tests/data/expected/persist_index_table_of_contents.wt.idx b/tests/data/expected/persist_index_table_of_contents.wt.idx index 9235e590..cae515b4 100644 Binary files a/tests/data/expected/persist_index_table_of_contents.wt.idx and b/tests/data/expected/persist_index_table_of_contents.wt.idx differ diff --git a/tests/data/expected/space_index/indexset/process_create_node.wt.idx b/tests/data/expected/space_index/indexset/process_create_node.wt.idx index 4fc48140..94fdfabd 100644 Binary files a/tests/data/expected/space_index/indexset/process_create_node.wt.idx and b/tests/data/expected/space_index/indexset/process_create_node.wt.idx differ diff --git a/tests/data/expected/space_index/indexset/process_insert_at.wt.idx b/tests/data/expected/space_index/indexset/process_insert_at.wt.idx index 998027e3..5cb7aaa0 100644 Binary files a/tests/data/expected/space_index/indexset/process_insert_at.wt.idx and b/tests/data/expected/space_index/indexset/process_insert_at.wt.idx differ diff --git a/tests/data/expected/space_index/indexset/process_insert_at_big_amount.wt.idx b/tests/data/expected/space_index/indexset/process_insert_at_big_amount.wt.idx index bad6e692..80f6c5b3 100644 Binary files a/tests/data/expected/space_index/indexset/process_insert_at_big_amount.wt.idx and b/tests/data/expected/space_index/indexset/process_insert_at_big_amount.wt.idx differ diff --git a/tests/data/expected/space_index/process_create_node.wt.idx b/tests/data/expected/space_index/process_create_node.wt.idx index 2c81cac6..9023a807 100644 Binary files a/tests/data/expected/space_index/process_create_node.wt.idx and b/tests/data/expected/space_index/process_create_node.wt.idx differ diff --git a/tests/data/expected/space_index/process_create_node_after_remove.wt.idx b/tests/data/expected/space_index/process_create_node_after_remove.wt.idx index 8972871f..ce5fe1a2 100644 Binary files a/tests/data/expected/space_index/process_create_node_after_remove.wt.idx and b/tests/data/expected/space_index/process_create_node_after_remove.wt.idx differ diff --git a/tests/data/expected/space_index/process_create_second_node.wt.idx b/tests/data/expected/space_index/process_create_second_node.wt.idx index e5adb74d..f90ec5b2 100644 Binary files a/tests/data/expected/space_index/process_create_second_node.wt.idx and b/tests/data/expected/space_index/process_create_second_node.wt.idx differ diff --git a/tests/data/expected/space_index/process_insert_at.wt.idx b/tests/data/expected/space_index/process_insert_at.wt.idx index 998027e3..5cb7aaa0 100644 Binary files a/tests/data/expected/space_index/process_insert_at.wt.idx and b/tests/data/expected/space_index/process_insert_at.wt.idx differ diff --git a/tests/data/expected/space_index/process_insert_at_big_amount.wt.idx b/tests/data/expected/space_index/process_insert_at_big_amount.wt.idx index b7d05af1..77404ea5 100644 Binary files a/tests/data/expected/space_index/process_insert_at_big_amount.wt.idx and b/tests/data/expected/space_index/process_insert_at_big_amount.wt.idx differ diff --git a/tests/data/expected/space_index/process_insert_at_removed_place.wt.idx b/tests/data/expected/space_index/process_insert_at_removed_place.wt.idx index c90e8f91..f0dbe7f7 100644 Binary files a/tests/data/expected/space_index/process_insert_at_removed_place.wt.idx and b/tests/data/expected/space_index/process_insert_at_removed_place.wt.idx differ diff --git a/tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx b/tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx index 6f3dad2d..97284ebe 100644 Binary files a/tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx and b/tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx differ diff --git a/tests/data/expected/space_index/process_remove_at.wt.idx b/tests/data/expected/space_index/process_remove_at.wt.idx index 2c81cac6..9023a807 100644 Binary files a/tests/data/expected/space_index/process_remove_at.wt.idx and b/tests/data/expected/space_index/process_remove_at.wt.idx differ diff --git a/tests/data/expected/space_index/process_remove_at_node_id.wt.idx b/tests/data/expected/space_index/process_remove_at_node_id.wt.idx index 8ca971de..a5f933df 100644 Binary files a/tests/data/expected/space_index/process_remove_at_node_id.wt.idx and b/tests/data/expected/space_index/process_remove_at_node_id.wt.idx differ diff --git a/tests/data/expected/space_index/process_remove_node.wt.idx b/tests/data/expected/space_index/process_remove_node.wt.idx index 4a178159..b8be470e 100644 Binary files a/tests/data/expected/space_index/process_remove_node.wt.idx and b/tests/data/expected/space_index/process_remove_node.wt.idx differ diff --git a/tests/data/expected/space_index/process_split_node.wt.idx b/tests/data/expected/space_index/process_split_node.wt.idx index cfc83485..99227d9e 100644 Binary files a/tests/data/expected/space_index/process_split_node.wt.idx and b/tests/data/expected/space_index/process_split_node.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/indexset/process_create_node.wt.idx b/tests/data/expected/space_index_unsized/indexset/process_create_node.wt.idx index 10e357ee..20901656 100644 Binary files a/tests/data/expected/space_index_unsized/indexset/process_create_node.wt.idx and b/tests/data/expected/space_index_unsized/indexset/process_create_node.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/indexset/process_insert_at.wt.idx b/tests/data/expected/space_index_unsized/indexset/process_insert_at.wt.idx index 44388dae..597f9109 100644 Binary files a/tests/data/expected/space_index_unsized/indexset/process_insert_at.wt.idx and b/tests/data/expected/space_index_unsized/indexset/process_insert_at.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx b/tests/data/expected/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx index bd2fc259..a0a3aa6b 100644 Binary files a/tests/data/expected/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx and b/tests/data/expected/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_create_node.wt.idx b/tests/data/expected/space_index_unsized/process_create_node.wt.idx index 36457de6..d5ae06b5 100644 Binary files a/tests/data/expected/space_index_unsized/process_create_node.wt.idx and b/tests/data/expected/space_index_unsized/process_create_node.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx b/tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx index e512c19e..c6469161 100644 Binary files a/tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx and b/tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_create_second_node.wt.idx b/tests/data/expected/space_index_unsized/process_create_second_node.wt.idx index 4e7c27fc..fff7dc0e 100644 Binary files a/tests/data/expected/space_index_unsized/process_create_second_node.wt.idx and b/tests/data/expected/space_index_unsized/process_create_second_node.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_insert_at.wt.idx b/tests/data/expected/space_index_unsized/process_insert_at.wt.idx index 93ae2c58..b56aeee3 100644 Binary files a/tests/data/expected/space_index_unsized/process_insert_at.wt.idx and b/tests/data/expected/space_index_unsized/process_insert_at.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_insert_at_big_amount.wt.idx b/tests/data/expected/space_index_unsized/process_insert_at_big_amount.wt.idx index 810c917c..0801fcdc 100644 Binary files a/tests/data/expected/space_index_unsized/process_insert_at_big_amount.wt.idx and b/tests/data/expected/space_index_unsized/process_insert_at_big_amount.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx b/tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx index 5849ef32..bb92d777 100644 Binary files a/tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx and b/tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx b/tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx index 6582c5a7..b11bdae3 100644 Binary files a/tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx and b/tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_remove_at.wt.idx b/tests/data/expected/space_index_unsized/process_remove_at.wt.idx index cd5b8dc9..86e1f4ec 100644 Binary files a/tests/data/expected/space_index_unsized/process_remove_at.wt.idx and b/tests/data/expected/space_index_unsized/process_remove_at.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx b/tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx index 271afb51..716e3fbe 100644 Binary files a/tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx and b/tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_remove_node.wt.idx b/tests/data/expected/space_index_unsized/process_remove_node.wt.idx index eb83b76d..8e2e6228 100644 Binary files a/tests/data/expected/space_index_unsized/process_remove_node.wt.idx and b/tests/data/expected/space_index_unsized/process_remove_node.wt.idx differ diff --git a/tests/data/expected/space_index_unsized/process_split_node.wt.idx b/tests/data/expected/space_index_unsized/process_split_node.wt.idx index b01ebf72..097eaccc 100644 Binary files a/tests/data/expected/space_index_unsized/process_split_node.wt.idx and b/tests/data/expected/space_index_unsized/process_split_node.wt.idx differ diff --git a/tests/data/expected/test_persist/.wt.data b/tests/data/expected/test_persist/.wt.data index cf48e267..68a4cf72 100644 Binary files a/tests/data/expected/test_persist/.wt.data and b/tests/data/expected/test_persist/.wt.data differ diff --git a/tests/data/expected/test_persist/another_idx.wt.idx b/tests/data/expected/test_persist/another_idx.wt.idx index b8849267..d402e056 100644 Binary files a/tests/data/expected/test_persist/another_idx.wt.idx and b/tests/data/expected/test_persist/another_idx.wt.idx differ diff --git a/tests/data/expected/test_persist/primary.wt.idx b/tests/data/expected/test_persist/primary.wt.idx index 6924ed7a..72cca5b2 100644 Binary files a/tests/data/expected/test_persist/primary.wt.idx and b/tests/data/expected/test_persist/primary.wt.idx differ diff --git a/tests/data/expected/test_without_secondary_indexes/.wt.data b/tests/data/expected/test_without_secondary_indexes/.wt.data index 555d8cb0..18946fcd 100644 Binary files a/tests/data/expected/test_without_secondary_indexes/.wt.data and b/tests/data/expected/test_without_secondary_indexes/.wt.data differ diff --git a/tests/data/expected/test_without_secondary_indexes/primary.wt.idx b/tests/data/expected/test_without_secondary_indexes/primary.wt.idx index c2cce055..7a9fe185 100644 Binary files a/tests/data/expected/test_without_secondary_indexes/primary.wt.idx and b/tests/data/expected/test_without_secondary_indexes/primary.wt.idx differ diff --git a/tests/data/space_index/indexset/process_create_node.wt.idx b/tests/data/space_index/indexset/process_create_node.wt.idx deleted file mode 100644 index 4fc48140..00000000 Binary files a/tests/data/space_index/indexset/process_create_node.wt.idx and /dev/null differ diff --git a/tests/data/space_index/indexset/process_insert_at.wt.idx b/tests/data/space_index/indexset/process_insert_at.wt.idx deleted file mode 100644 index 998027e3..00000000 Binary files a/tests/data/space_index/indexset/process_insert_at.wt.idx and /dev/null differ diff --git a/tests/data/space_index/indexset/process_insert_at_big_amount.wt.idx b/tests/data/space_index/indexset/process_insert_at_big_amount.wt.idx deleted file mode 100644 index bad6e692..00000000 Binary files a/tests/data/space_index/indexset/process_insert_at_big_amount.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_create_node.wt.idx b/tests/data/space_index/process_create_node.wt.idx deleted file mode 100644 index 2c81cac6..00000000 Binary files a/tests/data/space_index/process_create_node.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_create_node_after_remove.wt.idx b/tests/data/space_index/process_create_node_after_remove.wt.idx deleted file mode 100644 index 8972871f..00000000 Binary files a/tests/data/space_index/process_create_node_after_remove.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_create_second_node.wt.idx b/tests/data/space_index/process_create_second_node.wt.idx deleted file mode 100644 index e5adb74d..00000000 Binary files a/tests/data/space_index/process_create_second_node.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_insert_at.wt.idx b/tests/data/space_index/process_insert_at.wt.idx deleted file mode 100644 index 998027e3..00000000 Binary files a/tests/data/space_index/process_insert_at.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_insert_at_big_amount.wt.idx b/tests/data/space_index/process_insert_at_big_amount.wt.idx deleted file mode 100644 index b7d05af1..00000000 Binary files a/tests/data/space_index/process_insert_at_big_amount.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_insert_at_removed_place.wt.idx b/tests/data/space_index/process_insert_at_removed_place.wt.idx deleted file mode 100644 index c90e8f91..00000000 Binary files a/tests/data/space_index/process_insert_at_removed_place.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_insert_at_with_node_id_update.wt.idx b/tests/data/space_index/process_insert_at_with_node_id_update.wt.idx deleted file mode 100644 index 6f3dad2d..00000000 Binary files a/tests/data/space_index/process_insert_at_with_node_id_update.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_remove_at.wt.idx b/tests/data/space_index/process_remove_at.wt.idx deleted file mode 100644 index 2c81cac6..00000000 Binary files a/tests/data/space_index/process_remove_at.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_remove_at_node_id.wt.idx b/tests/data/space_index/process_remove_at_node_id.wt.idx deleted file mode 100644 index 8ca971de..00000000 Binary files a/tests/data/space_index/process_remove_at_node_id.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_remove_node.wt.idx b/tests/data/space_index/process_remove_node.wt.idx deleted file mode 100644 index 4a178159..00000000 Binary files a/tests/data/space_index/process_remove_node.wt.idx and /dev/null differ diff --git a/tests/data/space_index/process_split_node.wt.idx b/tests/data/space_index/process_split_node.wt.idx deleted file mode 100644 index cfc83485..00000000 Binary files a/tests/data/space_index/process_split_node.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/indexset/process_create_node.wt.idx b/tests/data/space_index_unsized/indexset/process_create_node.wt.idx deleted file mode 100644 index 10e357ee..00000000 Binary files a/tests/data/space_index_unsized/indexset/process_create_node.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/indexset/process_insert_at.wt.idx b/tests/data/space_index_unsized/indexset/process_insert_at.wt.idx deleted file mode 100644 index 44388dae..00000000 Binary files a/tests/data/space_index_unsized/indexset/process_insert_at.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx b/tests/data/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx deleted file mode 100644 index bd2fc259..00000000 Binary files a/tests/data/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_create_node.wt.idx b/tests/data/space_index_unsized/process_create_node.wt.idx deleted file mode 100644 index 36457de6..00000000 Binary files a/tests/data/space_index_unsized/process_create_node.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_create_node_after_remove.wt.idx b/tests/data/space_index_unsized/process_create_node_after_remove.wt.idx deleted file mode 100644 index e512c19e..00000000 Binary files a/tests/data/space_index_unsized/process_create_node_after_remove.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_create_second_node.wt.idx b/tests/data/space_index_unsized/process_create_second_node.wt.idx deleted file mode 100644 index 4e7c27fc..00000000 Binary files a/tests/data/space_index_unsized/process_create_second_node.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_insert_at.wt.idx b/tests/data/space_index_unsized/process_insert_at.wt.idx deleted file mode 100644 index 93ae2c58..00000000 Binary files a/tests/data/space_index_unsized/process_insert_at.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_insert_at_big_amount.wt.idx b/tests/data/space_index_unsized/process_insert_at_big_amount.wt.idx deleted file mode 100644 index 810c917c..00000000 Binary files a/tests/data/space_index_unsized/process_insert_at_big_amount.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_insert_at_removed_place.wt.idx b/tests/data/space_index_unsized/process_insert_at_removed_place.wt.idx deleted file mode 100644 index 5849ef32..00000000 Binary files a/tests/data/space_index_unsized/process_insert_at_removed_place.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_insert_at_with_node_id_update.wt.idx b/tests/data/space_index_unsized/process_insert_at_with_node_id_update.wt.idx deleted file mode 100644 index 6582c5a7..00000000 Binary files a/tests/data/space_index_unsized/process_insert_at_with_node_id_update.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_remove_at.wt.idx b/tests/data/space_index_unsized/process_remove_at.wt.idx deleted file mode 100644 index cd5b8dc9..00000000 Binary files a/tests/data/space_index_unsized/process_remove_at.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_remove_at_node_id.wt.idx b/tests/data/space_index_unsized/process_remove_at_node_id.wt.idx deleted file mode 100644 index 271afb51..00000000 Binary files a/tests/data/space_index_unsized/process_remove_at_node_id.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_remove_node.wt.idx b/tests/data/space_index_unsized/process_remove_node.wt.idx deleted file mode 100644 index eb83b76d..00000000 Binary files a/tests/data/space_index_unsized/process_remove_node.wt.idx and /dev/null differ diff --git a/tests/data/space_index_unsized/process_split_node.wt.idx b/tests/data/space_index_unsized/process_split_node.wt.idx deleted file mode 100644 index b01ebf72..00000000 Binary files a/tests/data/space_index_unsized/process_split_node.wt.idx and /dev/null differ diff --git a/tests/dense_partition_memory.rs b/tests/dense_partition_memory.rs new file mode 100644 index 00000000..054846f9 --- /dev/null +++ b/tests/dense_partition_memory.rs @@ -0,0 +1,361 @@ +//! What `partition_max_size` is worth, in bytes the allocator was asked for. +//! +//! # Why this is a separate binary +//! +//! The claim behind the key is about the *fixed apparatus* a partition +//! allocates at creation: an empty partition of the 88-byte-row shape +//! web3.trading runs measures about 28 KB before it holds a single row. +//! +//! `memory_by_key` and `memory_total` cannot see that. They report `used_bytes` +//! by definition, which is row bytes plus index bytes and explicitly excludes +//! the fixed floor, reserved-but-unused page capacity, the router spine and +//! `Arc` overhead. Measured through them the two shapes look identical, which +//! is true of what they measure and useless for this question. See +//! `memory_total_reports_rows_and_cannot_see_the_apparatus` in +//! `tests/worktable/partitioned.rs`, which pins that so the mistake is not made +//! twice. +//! +//! So this counts what the process actually asked the allocator for, which +//! needs a `#[global_allocator]`, which is per binary. Hence a file of its own. +//! +//! # What is measured +//! +//! One declaration, two widths, everything else identical: the same columns, +//! the same routing key, the same number of partitions and rows. The only +//! difference between the arms is `partition_max_size`, so the difference in +//! the result is what the key buys. +//! +//! This counts requested allocation bytes plus positive reallocation growth. +//! Freed bytes are not subtracted, and allocator bookkeeping is invisible. +//! It measures allocation demand, not resident or retained memory; it does not +//! establish a lower bound on either shape's resident-memory saving. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; + +use worktable::prelude::*; +use worktable::worktable; + +/// Counts bytes handed out while it is switched on. +/// +/// Each thread has its own region, so concurrent tests and harness allocations +/// on other threads cannot reset or add to the current test's count. +struct Counting; + +thread_local! { + // Constant initialization does not allocate inside the global allocator. + static ALLOCATED: Cell> = const { Cell::new(None) }; +} + +fn charge(bytes: usize) { + // Allocation during TLS teardown is outside a measurement region. + let _ = ALLOCATED.try_with(|count| { + if let Some(total) = count.get() { + count.set(Some(total + bytes)); + } + }); +} + +struct AllocationRegion; + +impl AllocationRegion { + fn start() -> Self { + ALLOCATED.with(|count| { + assert!(count.get().is_none(), "allocation regions must not overlap"); + count.set(Some(0)); + }); + Self + } + + fn finish(self) -> usize { + ALLOCATED.with(|count| count.take().expect("active allocation region")) + } +} + +impl Drop for AllocationRegion { + fn drop(&mut self) { + // Restore disabled accounting on both normal return and panic. + ALLOCATED.with(|count| count.set(None)); + } +} + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + charge(layout.size()); + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + if new_size > layout.size() { + charge(new_size - layout.size()); + } + unsafe { System.realloc(ptr, layout, new_size) } + } +} + +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +/// Bytes the allocator was asked for while `work` ran. +/// +/// Single-threaded by construction: every caller below builds its partitions on +/// this thread, so the counter is not picking up a background task's +/// allocations. A `worktable!` with `persist: false` starts no tasks. +fn allocated_by(work: impl FnOnce() -> T) -> (T, usize) { + let region = AllocationRegion::start(); + let out = work(); + (out, region.finish()) +} + +// The shape web3.trading runs: an exchange id inside a symbol. +// +// Wide on purpose. The whole finding is that the apparatus dominates a small +// partition, and a narrow row makes the apparatus look even larger relative to +// the data, so a wide row is the conservative choice for the claim. +// +// Written out twice rather than shared through a `macro_rules!`: `worktable!` +// reads tokens and does not expand a nested macro, so a shared block would not +// reach it. The two must stay identical, which is what +// `the_two_arms_declare_the_same_row` checks. +worktable!( + name: Dense, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { + exchange_id: u8 primary_key, + bid: f64, + ask: f64, + bid_size: f64, + ask_size: f64, + last: f64, + volume: f64, + open_interest: f64, + funding: f64, + updated_at: u64, + sequence: u64, + } +); + +worktable!( + name: Full, + partition_by: symbol_id: u16, + partition_max_size: u64, + columns: { + exchange_id: u8 primary_key, + bid: f64, + ask: f64, + bid_size: f64, + ask_size: f64, + last: f64, + volume: f64, + open_interest: f64, + funding: f64, + updated_at: u64, + sequence: u64, + } +); + +/// Exchanges per symbol. `Exchange::TOTAL` is 22 and the loop that fills an +/// order book is inclusive, so the real count is 23, not the 3 an earlier +/// measurement assumed. +const ROWS: u8 = 23; +/// Symbols. The low end of the 40-to-2,000 range the real system runs. +const PARTITIONS: u16 = 200; + +fn dense_row(exchange_id: u8) -> DenseRow { + DenseRow { + exchange_id, + bid: 1.0, + ask: 2.0, + bid_size: 3.0, + ask_size: 4.0, + last: 5.0, + volume: 6.0, + open_interest: 7.0, + funding: 8.0, + updated_at: 9, + sequence: 10, + } +} + +fn full_row(exchange_id: u8) -> FullRow { + FullRow { + exchange_id, + bid: 1.0, + ask: 2.0, + bid_size: 3.0, + ask_size: 4.0, + last: 5.0, + volume: 6.0, + open_interest: 7.0, + funding: 8.0, + updated_at: 9, + sequence: 10, + } +} + +#[tokio::test(flavor = "current_thread")] +async fn a_dense_partition_costs_a_fraction_of_a_full_one() { + // Warm both shapes first. The first partition of either kind pulls in + // one-off allocations that belong to neither arm, and charging them to + // whichever ran first is how a benchmark gets an answer it likes. + { + let warm = DensePartitions::new(); + let table = warm.partition_or_create(0).expect("fresh"); + table.insert(dense_row(0)).expect("fresh"); + + let warm = FullPartitions::new(); + let table = warm.partition_or_create(0).expect("fresh"); + table.insert(full_row(0)).await.expect("fresh"); + } + + let (dense, dense_bytes) = allocated_by(|| { + let books = DensePartitions::new(); + for symbol in 0..PARTITIONS { + let book = books.partition_or_create(symbol).expect("fresh"); + for exchange_id in 0..ROWS { + book.insert(dense_row(exchange_id)).expect("fresh"); + } + } + books + }); + + // The full table's `insert` is async, so the counter is started and stopped + // around the awaits by hand rather than through `allocated_by`. This arm + // therefore carries whatever the futures cost, which is a real cost of the + // shape and not a measurement artefact: a caller of the full table pays it. + // + // The runtime is `current_thread`, so this future stays on the thread whose + // region is active. Other test threads have independent counters. + let region = AllocationRegion::start(); + let books = FullPartitions::new(); + for symbol in 0..PARTITIONS { + let book = books.partition_or_create(symbol).expect("fresh"); + for exchange_id in 0..ROWS { + book.insert(full_row(exchange_id)).await.expect("fresh"); + } + } + let full_bytes = region.finish(); + + let rows = usize::from(PARTITIONS) * usize::from(ROWS); + let payload = rows * core::mem::size_of::(); + + eprintln!( + "DENSE-PARTITION-MEMORY partitions={PARTITIONS} rows_each={ROWS} row={}B payload={payload}B\n\ + \x20 dense={dense_bytes}B ({:.0} B/partition)\n\ + \x20 full={full_bytes}B ({:.0} B/partition)\n\ + \x20 saving={:.1}x", + core::mem::size_of::(), + dense_bytes as f64 / f64::from(PARTITIONS), + full_bytes as f64 / f64::from(PARTITIONS), + full_bytes as f64 / dense_bytes as f64, + ); + + assert_eq!(dense.len(), usize::from(PARTITIONS)); + assert_eq!(books.len(), usize::from(PARTITIONS)); + + // The claim, as a test rather than a printout. A factor of two is well + // inside what was measured and leaves room for an allocator that rounds + // differently, so this fails on a regression and not on a machine. + assert!( + full_bytes > dense_bytes * 2, + "a dense partition should cost a fraction of a full one: {dense_bytes} against {full_bytes}" + ); + + // And the dense arm should be close to its rows, because there is nothing + // else in it. Four times the payload allows for the slot vector doubling as + // it grows and the router's own spine. + assert!( + dense_bytes < payload * 4, + "a dense partition should be mostly rows: {dense_bytes} against {payload} of payload" + ); +} + +#[test] +fn an_empty_dense_partition_allocates_almost_nothing() { + // The sharpest form of the finding: the full shape's cost is paid at + // creation, before any row exists, so an empty partition is where the gap + // is widest. + { + let warm = DensePartitions::new(); + warm.partition_or_create(0).expect("fresh"); + let warm = FullPartitions::new(); + warm.partition_or_create(0).expect("fresh"); + } + + let (_, dense_bytes) = allocated_by(|| { + let books = DensePartitions::new(); + for symbol in 0..PARTITIONS { + books.partition_or_create(symbol).expect("fresh"); + } + books + }); + + let (_, full_bytes) = allocated_by(|| { + let books = FullPartitions::new(); + for symbol in 0..PARTITIONS { + books.partition_or_create(symbol).expect("fresh"); + } + books + }); + + eprintln!( + "EMPTY-PARTITION-MEMORY partitions={PARTITIONS}\n\ + \x20 dense={dense_bytes}B ({:.0} B/partition)\n\ + \x20 full={full_bytes}B ({:.0} B/partition)\n\ + \x20 saving={:.1}x", + dense_bytes as f64 / f64::from(PARTITIONS), + full_bytes as f64 / f64::from(PARTITIONS), + full_bytes as f64 / dense_bytes.max(1) as f64, + ); + + assert!( + full_bytes > dense_bytes * 4, + "an empty full partition carries apparatus an empty dense one does not: \ + {dense_bytes} against {full_bytes}" + ); +} + +#[test] +fn concurrent_allocation_regions_are_independent() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let ready = AtomicUsize::new(0); + let done = AtomicUsize::new(0); + std::thread::scope(|scope| { + let measure = |bytes| { + let (allocation, counted) = allocated_by(|| { + ready.fetch_add(1, Ordering::SeqCst); + while ready.load(Ordering::SeqCst) != 2 { + std::hint::spin_loop(); + } + let allocation = vec![0u8; bytes]; + std::hint::black_box(&allocation); + done.fetch_add(1, Ordering::SeqCst); + while done.load(Ordering::SeqCst) != 2 { + std::hint::spin_loop(); + } + allocation + }); + assert_eq!(counted, bytes); + assert_eq!(allocation.len(), bytes); + }; + let first = scope.spawn(move || measure(1024)); + let second = scope.spawn(move || measure(4096)); + first.join().unwrap(); + second.join().unwrap(); + }); +} + +#[test] +fn unwinding_disables_allocation_accounting() { + let _ = std::panic::catch_unwind(|| allocated_by(|| panic!("end region"))); + ALLOCATED.with(|count| assert_eq!(count.get(), None)); + let (allocation, counted) = allocated_by(|| vec![0u8; 1024]); + std::hint::black_box(&allocation); + assert_eq!(counted, 1024); +} diff --git a/tests/fixtures/page-format/README.md b/tests/fixtures/page-format/README.md new file mode 100644 index 00000000..4cab346f --- /dev/null +++ b/tests/fixtures/page-format/README.md @@ -0,0 +1,10 @@ +# Old page-format fixture + +`v2.wt.data` is an actual v2 store retained from the local persistence tests +before the v3 implementation on 2026-09-11. Its source was +`tests/data/unsized_primary_and_other_sync/update_query_pk/test_sync/.wt.data`. +It contains synthetic test rows. Its first header identifies format 2, and +the file includes metadata and row data (16,596 bytes). + +This fixture verifies refusal at the format boundary. It is not evidence of +v2-to-v3 conversion support, which is outside this release's runtime contract. diff --git a/tests/data/persist_index_table_of_contents.wt.idx b/tests/fixtures/page-format/v2.wt.data similarity index 97% rename from tests/data/persist_index_table_of_contents.wt.idx rename to tests/fixtures/page-format/v2.wt.data index 9235e590..03c78870 100644 Binary files a/tests/data/persist_index_table_of_contents.wt.idx and b/tests/fixtures/page-format/v2.wt.data differ diff --git a/tests/flavor_registry.rs b/tests/flavor_registry.rs new file mode 100644 index 00000000..a6f28e46 --- /dev/null +++ b/tests/flavor_registry.rs @@ -0,0 +1,104 @@ +//! The registry and its mirror, held together. +//! +//! `worktable::runtime::Flavor` is the registry: stable discriminants, the +//! tuning each name selects, and the spelling `WT_DEFAULT_RUNTIME` parses. +//! `worktable_dsl::model::Flavor` is a mirror of it, and exists so the parser +//! can read a flavor without the runtime crate's dependencies. +//! +//! Two lists of the same thing drift. Nothing about adding a flavor to one and +//! not the other is a compile error: the DSL would simply reject a spelling +//! the runtime accepts, or the runtime would refuse a spelling a schema is +//! allowed to write. Both failures land at run time, on a benchmark arm, as a +//! panic or a silent fallback. + +use worktable::prelude::Flavor; +use worktable_dsl::model::Flavor as Mirror; + +#[test] +fn the_mirror_has_the_same_flavors_in_the_same_order() { + let registry: Vec<&str> = Flavor::ALL.iter().map(|flavor| flavor.name()).collect(); + let mirror: Vec<&str> = Mirror::ALL.iter().map(|flavor| flavor.name()).collect(); + assert_eq!( + registry, mirror, + "the DSL's flavor mirror and the runtime registry disagree; \ + a flavor was added to one and not the other" + ); +} + +#[test] +fn every_spelling_the_mirror_accepts_the_registry_accepts() { + for mirrored in Mirror::ALL { + let flavor = Flavor::from_name(mirrored.name()) + .unwrap_or_else(|error| panic!("the registry rejects `{}`: {error}", mirrored.name())); + assert_eq!(flavor.name(), mirrored.name()); + } +} + +#[test] +fn every_spelling_the_registry_accepts_the_mirror_accepts() { + for flavor in Flavor::ALL { + assert_eq!( + Mirror::from_name(flavor.name()).map(|m| m.name()), + Some(flavor.name()), + "the DSL rejects `{}`, which the runtime accepts", + flavor.name() + ); + } +} + +/// The marker type the macro emits has to be a real export, or a table that +/// names the flavor fails to compile in the consumer's crate with an error +/// pointing at generated code. +#[test] +fn every_mirrored_marker_type_is_spelled_the_way_the_prelude_exports_it() { + // Named rather than derived, because the point is to check the string the + // macro emits against the identifier that actually exists. + let exported: &[(&str, &str)] = &[ + ("locality", "Locality"), + ("spread", "Spread"), + ("throughput", "Throughput"), + ("low_latency", "LowLatency"), + ("wide_injector", "WideInjector"), + ("shared_slot", "SharedSlot"), + ]; + assert_eq!(exported.len(), Flavor::ALL.len(), "a flavor has no marker listed here"); + for (name, type_name) in exported { + let mirrored = Mirror::from_name(name).expect("the mirror knows every listed flavor"); + assert_eq!(mirrored.type_name(), *type_name); + } + // And that each of those identifiers resolves. A name that does not exist + // is a compile error in this file, which is the point. + let _: fn() -> worktable::prelude::Tuning = + ::tuning; + let _: fn() -> worktable::prelude::Tuning = + ::tuning; + let _: fn() -> worktable::prelude::Tuning = + ::tuning; + let _: fn() -> worktable::prelude::Tuning = + ::tuning; + let _: fn() -> worktable::prelude::Tuning = + ::tuning; + let _: fn() -> worktable::prelude::Tuning = + ::tuning; +} + +/// A marker's `FLAVOR` byte and its `tuning()` have to agree, or the pool a +/// table dispatches to is not the pool its declared flavor names. +#[test] +fn each_marker_resolves_to_its_own_registry_row() { + use worktable::prelude::{FlavorMarker, Locality, LowLatency, SharedSlot, Spread, Throughput, WideInjector}; + + assert_eq!(Locality::FLAVOR, Flavor::Locality); + assert_eq!(Spread::FLAVOR, Flavor::Spread); + assert_eq!(Throughput::FLAVOR, Flavor::Throughput); + assert_eq!(LowLatency::FLAVOR, Flavor::LowLatency); + assert_eq!(WideInjector::FLAVOR, Flavor::WideInjector); + assert_eq!(SharedSlot::FLAVOR, Flavor::SharedSlot); + + assert_eq!(Locality::tuning(), Flavor::Locality.tuning()); + assert_eq!(Spread::tuning(), Flavor::Spread.tuning()); + assert_eq!(Throughput::tuning(), Flavor::Throughput.tuning()); + assert_eq!(LowLatency::tuning(), Flavor::LowLatency.tuning()); + assert_eq!(WideInjector::tuning(), Flavor::WideInjector.tuning()); + assert_eq!(SharedSlot::tuning(), Flavor::SharedSlot.tuning()); +} diff --git a/tests/generation_swap_requirement.rs b/tests/generation_swap_requirement.rs index fc6968ab..41f154c2 100644 --- a/tests/generation_swap_requirement.rs +++ b/tests/generation_swap_requirement.rs @@ -55,7 +55,17 @@ worktable!( }, ); -const DIR: &str = "tests/data/generation_swap/persisted"; +/// One directory per test, and never a shared one. +/// +/// `a_retired_generation_releases_its_memory` and +/// `a_generation_can_report_what_it_holds` used to share a single `DIR` const, +/// each removing and recreating it on entry. The harness runs tests on +/// parallel threads, so both attached a table to the same files and filled +/// them at once, and the loser reported a corrupt index. Sharing a fixture +/// directory between tests that write it is never safe here, however quiet it +/// stays. +const RETIRED_DIR: &str = "tests/data/generation_swap/retired"; +const REPORT_DIR: &str = "tests/data/generation_swap/report"; /// A generation big enough that releasing it is worth reporting. const ROWS: u64 = 2_000; @@ -87,12 +97,12 @@ async fn fill(table: &GenerationSwapWorkTable) { table.wait_for_ops().await.expect("the queue drains"); } -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_retired_generation_releases_its_memory() { - let _ = std::fs::remove_dir_all(DIR); - std::fs::create_dir_all(DIR).expect("a directory"); + let _ = std::fs::remove_dir_all(RETIRED_DIR); + std::fs::create_dir_all(RETIRED_DIR).expect("a directory"); - let generation = Arc::new(attach(DIR).await); + let generation = Arc::new(attach(RETIRED_DIR).await); fill(&generation).await; // A reader in flight, exactly as during a swap. @@ -124,21 +134,21 @@ async fn a_retired_generation_releases_its_memory() { "the generation had memory to release" ); - let _ = std::fs::remove_dir_all(DIR); + let _ = std::fs::remove_dir_all(RETIRED_DIR); } #[tokio::test] async fn a_generation_can_report_what_it_holds() { - let _ = std::fs::remove_dir_all(DIR); - std::fs::create_dir_all(DIR).expect("a directory"); + let _ = std::fs::remove_dir_all(REPORT_DIR); + std::fs::create_dir_all(REPORT_DIR).expect("a directory"); - let generation = attach(DIR).await; + let generation = attach(REPORT_DIR).await; fill(&generation).await; let held = generation.heap_size(); assert!(held > 0, "a filled generation holds memory: {held}"); generation.close().await.expect("generation closes"); - let _ = std::fs::remove_dir_all(DIR); + let _ = std::fs::remove_dir_all(REPORT_DIR); } #[tokio::test] diff --git a/tests/mod.rs b/tests/mod.rs index 86ec0ed7..2faff6c3 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -45,13 +45,21 @@ pub fn check_if_dirs_are_same(got: String, expected: String) -> bool { } pub async fn remove_file_if_exists(path: String) { - if Path::new(path.as_str()).exists() { - tokio::fs::remove_file(path.as_str()).await.unwrap(); + let path = Path::new(path.as_str()); + if path.exists() { + ::worktable::prelude::fsx::remove_file(path).await.unwrap(); + } + + // Output directories are ignored and therefore absent in a clean checkout. + // Recreate the parent so direct SpaceIndex tests do not depend on residue + // from an earlier local run. + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); } } pub async fn remove_dir_if_exists(path: String) { if Path::new(path.as_str()).exists() { - tokio::fs::remove_dir_all(path).await.unwrap() + ::worktable::prelude::fsx::remove_dir_all(path).await.unwrap() } } diff --git a/tests/narrow_key_lint.rs b/tests/narrow_key_lint.rs new file mode 100644 index 00000000..7d80a708 --- /dev/null +++ b/tests/narrow_key_lint.rs @@ -0,0 +1,58 @@ +//! The narrow-primary-key lint, as a compiler warning rather than as tokens. +//! +//! A codegen test can only say the tokens are emitted. Whether rustc turns them +//! into a warning, and whether `#[allow(deprecated)]` silences it, is a +//! property of the expansion in a real crate, so it is checked in one. +//! +//! Both tables here are deliberate, so both are silenced. What this file +//! asserts is that the silencing works: if `#[allow(deprecated)]` stopped +//! covering the expansion, this file would warn, and `-D warnings` in CI would +//! fail it. That is the regression worth catching, because a lint a consumer +//! cannot turn off is worse than no lint. + +use worktable::worktable; + +/// 256 rows is genuinely what this means: one row per exchange, no partitions. +#[allow(deprecated)] +mod deliberate { + use worktable::worktable; + + worktable!( + name: Exchange, + vec: true, + columns: { + id: u8 primary_key, + name_len: u32, + } + ); + + pub fn build() -> ExchangeWorkTable { + let mut table = ExchangeWorkTable::new(); + table.insert(ExchangeRow { id: 3, name_len: 7 }).expect("fresh"); + table + } +} + +// A wide key is not linted, so it needs no `allow`. If the lint ever started +// firing on `u64`, this file would warn and `-D warnings` in CI would catch it. +worktable!( + name: Wide, + vec: true, + columns: { + id: u64 primary_key, + v: u64, + } +); + +#[test] +fn a_silenced_narrow_key_still_works() { + let table = deliberate::build(); + assert_eq!(table.select(&3).expect("present").name_len, 7); +} + +#[test] +fn a_wide_key_needs_no_allow() { + let mut table = WideWorkTable::new(); + table.insert(WideRow { id: 9, v: 1 }).expect("fresh"); + assert_eq!(table.select(&9).expect("present").v, 1); +} diff --git a/tests/non-existent/test_persist/.wt.data b/tests/non-existent/test_persist/.wt.data deleted file mode 100644 index 6db83571..00000000 Binary files a/tests/non-existent/test_persist/.wt.data and /dev/null differ diff --git a/tests/non-existent/test_persist/another_idx.wt.idx b/tests/non-existent/test_persist/another_idx.wt.idx deleted file mode 100644 index b6b43e41..00000000 Binary files a/tests/non-existent/test_persist/another_idx.wt.idx and /dev/null differ diff --git a/tests/non-existent/test_persist/primary.wt.idx b/tests/non-existent/test_persist/primary.wt.idx deleted file mode 100644 index b6b43e41..00000000 Binary files a/tests/non-existent/test_persist/primary.wt.idx and /dev/null differ diff --git a/tests/nostd-consumer/Cargo.toml b/tests/nostd-consumer/Cargo.toml new file mode 100644 index 00000000..e5cc1ae1 --- /dev/null +++ b/tests/nostd-consumer/Cargo.toml @@ -0,0 +1,21 @@ +# A consumer that takes worktable without `std` and invokes the macro. +# +# Deliberately outside the workspace: it has to resolve `worktable` with +# `default-features = false`, and a workspace member shares the feature +# unification of everything built alongside it, which would quietly turn `std` +# back on and make this pass for the wrong reason. +[package] +name = "nostd-consumer" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +worktable = { path = "../..", default-features = false } +rkyv = { version = "0.8", default-features = false, features = ["alloc", "bytecheck", "uuid-1"] } +derive_more = { version = "2", default-features = false, features = ["from", "error", "display", "debug", "into"] } + +[dev-dependencies] +nagoya = { version = "^0.1", default-features = false } + +[workspace] diff --git a/tests/nostd-consumer/src/lib.rs b/tests/nostd-consumer/src/lib.rs new file mode 100644 index 00000000..854bae79 --- /dev/null +++ b/tests/nostd-consumer/src/lib.rs @@ -0,0 +1,95 @@ +//! Proof that `worktable!` emits nothing a `no_std` consumer cannot resolve. +//! +//! **The crate under test cannot check this itself.** `worktable` builds with +//! `--no-default-features` whether or not the macro is sound, because the +//! expansion only happens where the macro is invoked. So the verifier has to be +//! a separate crate that invokes it, which is what this is. +//! +//! Three names have gone through here: `ArtPersistenceKey`, `WorkTableVacuum` +//! and `EmptyDataVacuum`. All three are std-only for real reasons, so the fix +//! was to stop emitting them rather than to export them, and the mechanism is +//! `worktable::__wt_if_std!`. +#![no_std] + +extern crate alloc; +#[cfg(test)] +extern crate std; + +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: NoStdTable, + columns: { + id: u64 primary_key autoincrement, + value: u64, + } +); + +/// Proof that the table's *operations* compile without `std`, not merely its +/// declaration. +/// +/// The `worktable!` invocation above only proves the macro expands. That is a +/// weaker claim than it looks: a type can name itself fine and still be +/// unusable. This calls the three operations any consumer actually needs, so a +/// std-only path inside one of them fails the build. +/// +/// Not run, because running needs an allocator and an executor that a +/// `no_std` target brings itself. Compiling is the claim being made. +pub async fn smoke(table: &NoStdTableWorkTable) -> Option { + table.insert(NoStdTableRow { id: 1, value: 42 }).await.ok()?; + let selected = table.select(NoStdTablePrimaryKey::from(1u64))?; + let all = table.select_all().execute().ok()?; + core::mem::drop(all); + Some(selected.value) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_calls_run_with_the_no_std_dependency_graph() { + let table = NoStdTableWorkTable::default(); + assert_eq!(nagoya::block_on(smoke(&table)), Some(42)); + nagoya::block_on(table.delete(1u64)).unwrap(); + assert!(table.select(1u64).is_none()); + } + + #[test] + fn snapshot_growth_and_reads_remain_safe_across_threads() { + let table = NoStdTableWorkTable::default(); + std::thread::scope(|scope| { + for worker in 0..4u64 { + let table = &table; + scope.spawn(move || { + for i in 0..16_384u64 { + let id = worker * 16_384 + i; + nagoya::block_on(table.insert(NoStdTableRow { id, value: id + 1 })).unwrap(); + assert_eq!(table.select(id).unwrap().value, id + 1); + } + }); + } + }); + let rows = table.select_all().execute().unwrap(); + assert_eq!(rows.len(), 65_536); + } + + #[test] + fn operation_identifiers_use_the_os_clock_and_remain_ordered() { + let first = OperationId::default(); + for _ in 0..1024 { + let next = OperationId::default(); + assert!(next > first); + } + let OperationId::Single(id) = first else { + panic!("expected single operation"); + }; + let (seconds, _) = id.get_timestamp().unwrap().to_unix(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + assert!(now.abs_diff(seconds) < 10); + } +} diff --git a/tests/persistence/bulk_load_stall.rs b/tests/persistence/bulk_load_stall.rs index 949447d1..2af274fa 100644 --- a/tests/persistence/bulk_load_stall.rs +++ b/tests/persistence/bulk_load_stall.rs @@ -86,7 +86,7 @@ fn test_bulk_insert_delete_persistence() { table.delete(*id).await.unwrap(); } - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled on bulk insert+delete") .expect("persistence engine failed"); diff --git a/tests/persistence/concurrent_upsert_batch.rs b/tests/persistence/concurrent_upsert_batch.rs new file mode 100644 index 00000000..bf2c02d7 --- /dev/null +++ b/tests/persistence/concurrent_upsert_batch.rs @@ -0,0 +1,129 @@ +//! Concurrent upserts against a persisted table. +//! +//! Found by the persisted benchmark grid, which panicked on worker threads in +//! `persistence::space::data::save_batch_data`: +//! +//! ```text +//! should be available as pages parsed from these ids +//! ``` +//! +//! The lookup that fails is `batch_data.get(&id)` over the union of the pages +//! the batch created and the pages it parsed back. Every id in both sets comes +//! from `batch_data.keys()`, so the only way the lookup misses is for a parsed +//! page to carry a header id that was never requested. + +use worktable::prelude::*; +use worktable::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: ConcurrentUpsert, + persist: true, + columns: { + id: u64 primary_key, + payload: u64, + }, +); + +/// Several tasks upserting overlapping keys, which is what any table behind a +/// service does. +/// +/// Deliberately `multi_thread`: on the default current-thread runtime the +/// tasks never overlap and the batch path only ever sees one writer, which is +/// how a bug in it stays hidden. See `tests/worktable/multi_thread_discipline`. +/// +/// It used to panic twice, reliably: +/// +/// ```text +/// src/persistence/space/data.rs should be available as pages parsed from these ids +/// async-task/src/task.rs:452 Task polled after completion +/// ``` +/// +/// The cause was a gap in the page sequence, reduced to two calls in +/// `SpaceData::create_pages_up_to`'s test. Writers alone do not reproduce it: +/// a four-writer version of this test passes. It needs readers overlapping the +/// writers, because that is what makes two writers allocate pages at once. +/// +/// The size matters and is not arbitrary. At 5,000 rows this passed even with +/// the bug, and the wall time was the tell rather than the panic: 0.43s +/// passing at 5,000, 158s failing at 10,000, 0.89s passing at 10,000 once +/// fixed. The minutes were the panic's aftermath, not the cost of persisting. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_upserts_do_not_lose_a_page() { + let dir = "tests/data/concurrent_upsert_batch/persisted"; + remove_dir_if_exists(dir.to_string()).await; + + let config = DiskConfig::new_with_table_name( + dir, + ConcurrentUpsertWorkTable::name_snake_case(), + ConcurrentUpsertWorkTable::version(), + ); + let engine = ConcurrentUpsertPersistenceEngine::new(config).await.expect("an engine"); + let table = std::sync::Arc::new(ConcurrentUpsertWorkTable::load(engine).await.expect("a table")); + + const ROWS: u64 = 20_000; + /// Operations per task. Bounded independently of `ROWS`, because scaling + /// both together made this an 80,000,000 upsert test that ran for ten + /// minutes and told us nothing the first thousand had not. + const OPS: u64 = 4_000; + for id in 0..ROWS { + table + .insert(ConcurrentUpsertRow { id, payload: id }) + .await + .expect("fresh key"); + } + + // Enough rows to span many pages, and overlapping key ranges so two + // writers can land in one batch for the same page. + // Readers alongside the writers. The benchmark that found this ran six + // selects against two upserts, and a select takes the same page latch the + // flush needs, so leaving them out changes which paths overlap. + let mut handles = Vec::new(); + for reader in 0..6u64 { + let table = std::sync::Arc::clone(&table); + handles.push(tokio::spawn(async move { + for step in 0..OPS { + let id = (step * 11 + reader * 17) % ROWS; + let _ = table.select(id); + } + })); + } + for writer in 0..4u64 { + let table = std::sync::Arc::clone(&table); + handles.push(tokio::spawn(async move { + for step in 0..OPS { + let id = (step * 7 + writer * 13) % ROWS; + table + .upsert(ConcurrentUpsertRow { + id, + payload: writer * 1_000_000 + step, + }) + .await + .expect("an upsert"); + } + })); + } + for handle in handles { + handle.await.expect("a writer"); + } + + // Every key must still be readable, and the table must close cleanly: + // `close` returning `Ok` is the only proof the queue drained to disk. + for id in 0..ROWS { + assert!(table.select(id).is_some(), "row {id} went missing"); + } + let table = std::sync::Arc::try_unwrap(table).unwrap_or_else(|_| panic!("the writers are joined")); + // Bounded, because the failure mode this test guards against is a drain + // that takes minutes rather than one that returns an error. An unbounded + // `close` turns that regression into a hung suite instead of a red test. + // The isolated test takes about two seconds, but the all-features suite + // runs many persistence and CPU-heavy tests concurrently. Five seconds + // repeatedly expires under that contention despite a successful isolated + // run. This is a deadlock watchdog; the persistence benchmark measures + // latency. Keep it below the historical multi-minute failure mode. + tokio::time::timeout(std::time::Duration::from_secs(30), table.close()) + .await + .expect("close must drain in seconds, not minutes") + .expect("a clean close"); +} diff --git a/tests/persistence/custom_page_size.rs b/tests/persistence/custom_page_size.rs new file mode 100644 index 00000000..b7563cae --- /dev/null +++ b/tests/persistence/custom_page_size.rs @@ -0,0 +1,70 @@ +use worktable::prelude::*; +use worktable::worktable; + +use crate::remove_dir_if_exists; + +/// Half the default stride. Chosen so a page written at this size lands where +/// the default would put the middle of a page: a table that quietly fell back +/// to the default could not produce the file lengths asserted below. +const HALF: u32 = 8192; + +worktable! ( + name: HalfPage, + persist: true, + columns: { + id: u64 primary_key, + payload: u64, + }, + config: { + page_size: 8192 + } +); + +/// `page_size` beside `persist: true` used to be refused outright: the seeks +/// computed every offset from a hardcoded stride while the generated table +/// threaded the configured one, and the two disagreeing corrupted the file. +/// +/// A round trip alone would not prove the setting took effect, because a table +/// that fell back to the default would still reload its own writes. So the +/// file length is checked too, and it can only be a multiple of the configured +/// stride. +#[tokio::test] +async fn a_persisted_table_with_a_custom_page_size_reloads_what_it_wrote() { + let dir = "tests/data/custom_page_size/persisted"; + remove_dir_if_exists(dir.to_string()).await; + let config = + DiskConfig::new_with_table_name(dir, HalfPageWorkTable::name_snake_case(), HalfPageWorkTable::version()); + + let mut expected = Vec::new(); + { + let engine = HalfPagePersistenceEngine::new(config.clone()).await.unwrap(); + let table = HalfPageWorkTable::load(engine).await.unwrap(); + // Enough rows to need several pages at this stride, so the page + // transitions are exercised and not just the first page. + for i in 0..2_000u64 { + let row = HalfPageRow { id: i, payload: i }; + table.insert(row.clone()).await.unwrap(); + expected.push(row); + } + table.wait_for_ops().await.unwrap(); + } + + let data_file_path = format!("{dir}/{}/.wt.data", HalfPageWorkTable::name_snake_case()); + let length = std::fs::metadata(&data_file_path).unwrap().len(); + let pages = length.div_ceil(u64::from(HALF)); + assert!( + pages >= 2, + "expected several {HALF}-byte pages, got {pages} from {length} bytes" + ); + + let engine = HalfPagePersistenceEngine::new(config).await.unwrap(); + let table = HalfPageWorkTable::load(engine) + .await + .expect("a table with a custom page size must reload"); + assert_eq!(table.select_all().execute().unwrap().len(), expected.len()); + for row in &expected { + assert_eq!(table.select(row.id).as_ref(), Some(row)); + } + + remove_dir_if_exists(dir.to_string()).await; +} diff --git a/tests/persistence/duplicate_key_index_reload.rs b/tests/persistence/duplicate_key_index_reload.rs index d3b41405..3a73b531 100644 --- a/tests/persistence/duplicate_key_index_reload.rs +++ b/tests/persistence/duplicate_key_index_reload.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; @@ -111,12 +112,11 @@ async fn assert_straddling_topology(dir: &str) { use data_bucket::INNER_PAGE_SIZE; use std::sync::Arc; use std::sync::atomic::AtomicU32; - use tokio::fs::OpenOptions; let path = format!("{dir}/duplicate_key_reload/score_idx.wt.idx"); - let mut file = OpenOptions::new().read(true).write(true).open(&path).await.unwrap(); + let mut file = worktable::prelude::fsx::open(&path).await.unwrap(); let next_id_gen = Arc::new(AtomicU32::new(1)); - let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -133,9 +133,13 @@ async fn assert_straddling_topology(dir: &str) { let mut pages_per_key: BTreeMap = BTreeMap::new(); for page_id in mappings { - let page = parse_page::, { DUPLICATE_KEY_RELOAD_PAGE_SIZE as u32 }>(&mut file, page_id.into()) - .await - .unwrap(); + let page = parse_page::< + IndexPage, + { (DUPLICATE_KEY_RELOAD_PAGE_SIZE - data_bucket::GENERAL_HEADER_SIZE) as u32 }, + DEFAULT_PAGE_STRIDE, + >(&mut file, page_id.into()) + .await + .unwrap(); let keys: BTreeSet = page.inner.index_values[..page.inner.current_length as usize] .iter() .map(|v| v.key) @@ -215,7 +219,7 @@ fn test_duplicate_key_secondary_index_survives_reload() { model.assert_matches(&table, "in-memory before first persist"); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled on bulk insert") .expect("persistence engine failed"); @@ -278,7 +282,7 @@ fn test_duplicate_key_secondary_index_survives_reload() { model.assert_matches(&table, "in-memory after post-reload mutations"); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled on post-reload mutations") .expect("persistence engine failed"); @@ -331,7 +335,7 @@ fn test_single_key_all_duplicates_survives_reload() { } assert_eq!(table.select_by_score(42).execute().unwrap().len() as u64, ROWS); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled on bulk insert") .expect("persistence engine failed"); @@ -365,7 +369,7 @@ fn test_single_key_all_duplicates_survives_reload() { }) .await .unwrap(); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled on post-reload insert") .expect("persistence engine failed"); @@ -417,7 +421,7 @@ fn test_duplicate_key_mutations_without_reload() { .unwrap(); model.insert(i, i % KEYS, bucket); } - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled on bulk insert") .expect("persistence engine failed"); @@ -466,7 +470,7 @@ fn test_duplicate_key_mutations_without_reload() { model.assert_matches(&table, "in-memory after mutations (no reload)"); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled on mutations without any reload") .expect("persistence engine failed"); diff --git a/tests/persistence/exact_boundary_load.rs b/tests/persistence/exact_boundary_load.rs index d72a9db8..06edcd38 100644 --- a/tests/persistence/exact_boundary_load.rs +++ b/tests/persistence/exact_boundary_load.rs @@ -36,18 +36,12 @@ async fn table_whose_data_file_ends_on_an_exact_page_boundary_loads() { table.wait_for_ops().await.unwrap(); } - // Pad the data file to the next exact page-size multiple. + // V3 persists the directory and checksum at the page tail, so every + // completed data page ends exactly on its stride boundary. let data_file_path = format!("{dir}/{}/.wt.data", TestPersistWorkTable::name_snake_case()); let stride = TEST_PERSIST_PAGE_SIZE as u64; let len = std::fs::metadata(&data_file_path).unwrap().len(); - assert!( - len % stride != 0, - "fixture must start off the boundary for the padding below to construct it" - ); - let padded = len.div_ceil(stride) * stride; - let file = std::fs::OpenOptions::new().write(true).open(&data_file_path).unwrap(); - file.set_len(padded).unwrap(); - drop(file); + assert_eq!(len % stride, 0, "v3 data pages must fill their disk slots"); let engine = TestPersistPersistenceEngine::new(config).await.unwrap(); let table = TestPersistWorkTable::load(engine) diff --git a/tests/persistence/index_page/read.rs b/tests/persistence/index_page/read.rs index 5cc90005..ebc19d28 100644 --- a/tests/persistence/index_page/read.rs +++ b/tests/persistence/index_page/read.rs @@ -1,16 +1,13 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::{INNER_PAGE_SIZE, IndexPage, parse_page}; -use tokio::fs::OpenOptions; #[tokio::test] async fn test_index_page_read_in_space() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/test_persist/primary.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/primary.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 99); @@ -20,14 +17,11 @@ async fn test_index_page_read_in_space() { #[tokio::test] async fn test_index_page_read_after_create_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -40,14 +34,11 @@ async fn test_index_page_read_after_create_node_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -64,14 +55,12 @@ async fn test_index_page_read_after_insert_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") + .await + .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 7); @@ -88,14 +77,11 @@ async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index #[tokio::test] async fn test_index_page_read_after_remove_at_node_id_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 3); @@ -109,14 +95,11 @@ async fn test_index_page_read_after_remove_at_node_id_in_space_index() { #[tokio::test] async fn test_index_page_read_after_remove_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_at.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -130,14 +113,12 @@ async fn test_index_page_read_after_remove_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_removed_place.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_removed_place.wt.idx") + .await + .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 7); @@ -159,14 +140,11 @@ async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_second_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_second_node.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -176,7 +154,7 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { assert_eq!(page.inner.index_values.first().unwrap().key, 5); assert_eq!(page.inner.index_values.first().unwrap().link.length, 24); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 3) .await .unwrap(); assert_eq!(page.inner.node_id.key, 15); @@ -189,14 +167,12 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_creation_of_node_after_remove_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") + .await + .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 10); @@ -206,7 +182,7 @@ async fn test_index_pages_read_after_creation_of_node_after_remove_node_in_space assert_eq!(page.inner.index_values.first().unwrap().key, 10); assert_eq!(page.inner.index_values.first().unwrap().link.length, 24); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 3) .await .unwrap(); assert_eq!(page.inner.node_id.key, 15); @@ -219,14 +195,11 @@ async fn test_index_pages_read_after_creation_of_node_after_remove_node_in_space #[tokio::test] async fn test_index_pages_read_full_page() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_big_amount.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_big_amount.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 1000); @@ -236,21 +209,18 @@ async fn test_index_pages_read_full_page() { #[tokio::test] async fn test_index_pages_read_after_node_split() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_split_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_split_node.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 457); assert_eq!(page.inner.current_index, 0); assert_eq!(page.inner.current_length, 453); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 3) .await .unwrap(); assert_eq!(page.inner.node_id.key, 1000); diff --git a/tests/persistence/index_page/unsized_read.rs b/tests/persistence/index_page/unsized_read.rs index e4673d55..c2e2fec4 100644 --- a/tests/persistence/index_page/unsized_read.rs +++ b/tests/persistence/index_page/unsized_read.rs @@ -1,19 +1,18 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::{INNER_PAGE_SIZE, Link, UnsizedIndexPage, parse_page}; -use tokio::fs::OpenOptions; #[tokio::test] async fn test_index_page_read_after_create_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -31,18 +30,18 @@ async fn test_index_page_read_after_create_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") - .await - .unwrap(); - - let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") .await .unwrap(); + let page = + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); + assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 1); let value = page.inner.index_values.first().unwrap(); @@ -57,9 +56,11 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { ); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 3, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Someone from somewhere".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -77,17 +78,16 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_remove_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 3, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Someone from somewhere".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -105,17 +105,16 @@ async fn test_index_pages_read_after_remove_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_insert_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 2); @@ -143,17 +142,16 @@ async fn test_index_pages_read_after_insert_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_remove_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -171,18 +169,18 @@ async fn test_index_page_read_after_remove_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_remove_at_node_id_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") - .await - .unwrap(); - - let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") .await .unwrap(); + let page = + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); + assert_eq!(page.inner.node_id.key, "Something else".to_string()); assert_eq!(page.inner.index_values.len(), 1); let value = page.inner.index_values.first().unwrap(); @@ -199,17 +197,18 @@ async fn test_index_page_read_after_remove_at_node_id_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx") - .await - .unwrap(); + let mut file = worktable::prelude::fsx::open( + "tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx", + ) + .await + .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone 1".to_string()); assert_eq!(page.inner.index_values.len(), 2); @@ -237,18 +236,18 @@ async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index #[tokio::test] async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx") - .await - .unwrap(); - - let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx") .await .unwrap(); + let page = + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); + assert_eq!(page.inner.node_id.key, "Something from someone 1".to_string()); assert_eq!(page.inner.index_values.len(), 3); let first_value = &page.inner.index_values[0]; @@ -285,24 +284,25 @@ async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_node_split() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_split_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_split_node.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone 52"); assert_eq!(page.inner.slots_size, 53); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 3, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone _100"); assert_eq!(page.inner.slots_size, 48); } diff --git a/tests/persistence/insert_cost_shape.rs b/tests/persistence/insert_cost_shape.rs new file mode 100644 index 00000000..90b09cec --- /dev/null +++ b/tests/persistence/insert_cost_shape.rs @@ -0,0 +1,216 @@ +//! Where the time in an insert goes, since it is not the disk. +//! +//! WorkTable's bulk load runs at about 200 MB/s while the write path under it +//! does gigabytes, and removing persistence entirely changes nothing. So the +//! cost is in the insert. This asks the first question that splits the +//! candidates: does it scale with the size of the row, or is it a fixed price +//! per row? + +use worktable::prelude::*; +use worktable_codegen::worktable; + +worktable!( + name: InsertShape, + columns: { + id: u64 primary_key, + payload: String, + } +); + +#[test] +#[ignore = "a measurement, not an assertion"] +fn does_insert_cost_scale_with_row_size() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + const ROWS: u64 = 25_000; + println!(" payload rows/s us/row MB/s"); + for size in [8usize, 64, 256, 1024, 2048, 4096, 8192] { + let payload = "x".repeat(size); + // Warm: allocator and the table's first growth are not the subject. + { + let warm = InsertShapeWorkTable::default(); + for id in 0..1_000u64 { + warm.insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + } + let table = InsertShapeWorkTable::default(); + let at = std::time::Instant::now(); + for id in 0..ROWS { + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + let elapsed = at.elapsed().as_secs_f64(); + println!( + " {size:>7} {:>9.0} {:>8.2} {:>7.0}", + ROWS as f64 / elapsed, + elapsed * 1e6 / ROWS as f64, + (ROWS as usize * size) as f64 / 1e6 / elapsed, + ); + } + }); +} + +/// The same rows, with the table taken out of it: what the loop costs when all +/// it does is build the row and drop it. Anything the insert arm spends beyond +/// this is the table. +#[test] +#[ignore = "a measurement, not an assertion"] +fn what_the_loop_costs_without_the_table() { + const ROWS: u64 = 25_000; + println!(" payload rows/s us/row"); + for size in [8usize, 4096] { + let payload = "x".repeat(size); + let at = std::time::Instant::now(); + let mut sink = 0usize; + for id in 0..ROWS { + let row = InsertShapeRow { + id, + payload: payload.clone(), + }; + sink = sink.wrapping_add(row.payload.len()); + std::hint::black_box(&row); + } + let elapsed = at.elapsed().as_secs_f64(); + std::hint::black_box(sink); + println!( + " {size:>7} {:>9.0} {:>8.3}", + ROWS as f64 / elapsed, + elapsed * 1e6 / ROWS as f64, + ); + } +} + +/// A long run of the expensive case, so a sampling profiler has something to +/// look at. Not a measurement in itself. +#[test] +#[ignore = "for profiling only"] +fn keep_inserting_four_kilobyte_rows() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let payload = "x".repeat(4096); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while std::time::Instant::now() < deadline { + let table = InsertShapeWorkTable::default(); + for id in 0..20_000u64 { + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + } + }); +} + +/// Does an insert get slower as the table fills? +/// +/// A cost that scales with rows already present is O(n) per insert and O(n^2) +/// overall, which is what a super-linear response to row size would look like +/// if bigger rows simply reach any given page count sooner. +#[test] +#[ignore = "a measurement, not an assertion"] +fn does_insert_slow_down_as_the_table_fills() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + for size in [256usize, 4096] { + let payload = "x".repeat(size); + let table = InsertShapeWorkTable::default(); + const BLOCK: u64 = 2_000; + const BLOCKS: u64 = 10; + println!(" payload {size}: us/row by block of {BLOCK}"); + let mut line = String::new(); + for block in 0..BLOCKS { + let at = std::time::Instant::now(); + for n in 0..BLOCK { + let id = block * BLOCK + n; + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + let per_row = at.elapsed().as_secs_f64() * 1e6 / BLOCK as f64; + line.push_str(&format!("{per_row:>8.2}")); + } + println!(" {line}"); + } + }); +} + +/// The per-row cost with the page-list clone nearly absent, which is the floor +/// a fix for it would approach. +/// +/// The clone is O(pages), so a table that has barely any pages barely pays it. +/// Timing small tables gives the cost of everything else: the row clone, the +/// rkyv serialize, and the copy into the page. +#[test] +#[ignore = "a measurement, not an assertion"] +fn the_floor_with_almost_no_pages() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + println!(" payload rows pages us/row MB/s"); + for size in [256usize, 1024, 4096] { + let payload = "x".repeat(size); + let per_page = (16356 / size).max(1); + for rows in [30u64, 120, 480] { + // Median of several fresh tables: a single small run is noise. + let mut samples = Vec::new(); + for _ in 0..25 { + let table = InsertShapeWorkTable::default(); + let at = std::time::Instant::now(); + for id in 0..rows { + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + samples.push(at.elapsed().as_secs_f64() / rows as f64); + } + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let per_row = samples[samples.len() / 2]; + println!( + " {size:>7} {rows:>5} {:>5} {:>7.3} {:>7.0}", + rows as usize / per_page, + per_row * 1e6, + size as f64 / 1e6 / per_row, + ); + } + } + }); +} diff --git a/tests/persistence/insert_latency.rs b/tests/persistence/insert_latency.rs new file mode 100644 index 00000000..00afe6a0 --- /dev/null +++ b/tests/persistence/insert_latency.rs @@ -0,0 +1,124 @@ +//! Per-insert latency, split by whether the insert had to allocate a page. +//! +//! These are two populations, not one distribution. An insert that fits in the +//! page already open is cheap; one that has to add a page pays for the page +//! list as well. Blending them hides the second behind the first, and how much +//! it hides depends on row size: at 4 KiB a page holds three rows, so a third +//! of all inserts allocate and the expensive population is not a tail at all. + +use worktable::prelude::PersistedWorkTable; +use worktable::prelude::*; +use worktable_codegen::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: InsertLatency, + persist: true, + columns: { + id: u64 primary_key, + payload: String, + } +); + +worktable!( + name: InsertLatencyMemory, + columns: { + id: u64 primary_key, + payload: String, + } +); + +fn report(label: &str, mut us: Vec, of: usize) { + if us.is_empty() { + println!(" {label:<34} (none)"); + return; + } + us.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let at = |q: f64| us[((us.len() as f64 * q) as usize).min(us.len() - 1)]; + println!( + " {label:<34} {:>5.1}% of inserts p50 {:>7.2} p99 {:>8.2} max {:>9.2} (us)", + 100.0 * us.len() as f64 / of as f64, + at(0.50), + at(0.99), + us[us.len() - 1], + ); +} + +#[test] +#[ignore = "a measurement, not an assertion"] +fn insert_latency_split_by_page_allocation() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + const ROWS: u64 = 20_000; + for size in [256usize, 4096] { + let payload = "x".repeat(size); + println!("payload {size} B, {ROWS} rows"); + + // ---- no persistence + let table = InsertLatencyMemoryWorkTable::default(); + let (mut same, mut fresh) = (Vec::new(), Vec::new()); + let mut pages = table.0.data.get_page_count(); + for id in 0..ROWS { + let at = std::time::Instant::now(); + table + .insert(InsertLatencyMemoryRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + let took = at.elapsed().as_secs_f64() * 1e6; + let now = table.0.data.get_page_count(); + if now == pages { + same.push(took) + } else { + fresh.push(took) + } + pages = now; + } + report("in memory, existing page", same, ROWS as usize); + report("in memory, allocated a page", fresh, ROWS as usize); + + // ---- persisted + let dir = "tests/data/insert_latency"; + remove_dir_if_exists(dir.to_string()).await; + let config = DiskConfig::new_with_table_name( + dir, + InsertLatencyWorkTable::name_snake_case(), + InsertLatencyWorkTable::version(), + ); + let engine = InsertLatencyPersistenceEngine::new(config).await.unwrap(); + let persisted = InsertLatencyWorkTable::load(engine).await.unwrap(); + let (mut same, mut fresh) = (Vec::new(), Vec::new()); + let mut pages = persisted.0.data.get_page_count(); + for id in 0..ROWS { + let at = std::time::Instant::now(); + persisted + .insert(InsertLatencyRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + let took = at.elapsed().as_secs_f64() * 1e6; + let now = persisted.0.data.get_page_count(); + if now == pages { + same.push(took) + } else { + fresh.push(took) + } + pages = now; + } + report("persisted, existing page", same, ROWS as usize); + report("persisted, allocated a page", fresh, ROWS as usize); + persisted.wait_for_ops().await.expect("the queue drains"); + remove_dir_if_exists(dir.to_string()).await; + } + }); +} diff --git a/tests/persistence/loaded_index_growth.rs b/tests/persistence/loaded_index_growth.rs index 12ff9b90..6c60fd08 100644 --- a/tests/persistence/loaded_index_growth.rs +++ b/tests/persistence/loaded_index_growth.rs @@ -95,7 +95,7 @@ fn test_primary_index_grows_on_a_loaded_table() { for i in 0..ROWS_BEFORE_RELOAD { table.insert(row(i)).await.unwrap(); } - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled building the initial store") .expect("persistence engine failed"); @@ -115,7 +115,7 @@ fn test_primary_index_grows_on_a_loaded_table() { .await .unwrap_or_else(|error| panic!("insert {i} into the loaded table was refused: {error:?}")); } - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled appending to the loaded store") .expect("persistence engine failed"); @@ -166,7 +166,7 @@ fn test_primary_index_grows_on_a_loaded_table() { // And the grown, reloaded table must still be writable: the // production stores died on exactly this insert. table.insert(row(ROWS_BEFORE_RELOAD + ROWS_AFTER_RELOAD)).await.unwrap(); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled on the post-reload insert") .expect("persistence engine failed"); diff --git a/tests/persistence/local_write_bandwidth.rs b/tests/persistence/local_write_bandwidth.rs new file mode 100644 index 00000000..5d74f201 --- /dev/null +++ b/tests/persistence/local_write_bandwidth.rs @@ -0,0 +1,201 @@ +//! What WorkTable's local persistence path sustains, in bytes per second. +//! +//! Measured through `insert` and `wait_for_ops` rather than against +//! `persist_page` directly, so it counts everything the engine does to make a +//! write durable and not just the call at the bottom of it. + +use worktable::prelude::PersistedWorkTable; +use worktable::prelude::*; +use worktable_codegen::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: WriteBandwidth, + persist: true, + columns: { + id: u64 primary_key, + payload: String, + } +); + +// The same table without persistence, so the cost of being a WorkTable can be +// told apart from the cost of writing to disk. Anything this arm spends is +// spent by the persisted one too, before any page is written. +worktable!( + name: WriteBandwidthMemory, + columns: { + id: u64 primary_key, + payload: String, + } +); + +/// A page of the on-disk format, which is the unit a write actually lands in. +const PAGE: usize = 4096 * 4; + +/// Per-page checksums of every file, so two snapshots say how many pages a +/// stretch of work really wrote. +fn page_checksums(dir: &str) -> Vec<(String, Vec)> { + fn walk(dir: &std::path::Path, into: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.is_dir() { + walk(&path, into); + } else if path.is_file() { + into.push(path); + } + } + } + let mut paths = Vec::new(); + walk(std::path::Path::new(dir), &mut paths); + paths.sort(); + paths + .into_iter() + .map(|path| { + let bytes = std::fs::read(&path).expect("a table file"); + ( + path.to_string_lossy().into_owned(), + bytes.chunks(PAGE).map(crc32fast::hash).collect(), + ) + }) + .collect() +} + +/// Bytes that differ between two snapshots, counted a page at a time. +fn written_bytes(before: &[(String, Vec)], after: &[(String, Vec)]) -> u64 { + let mut pages = 0u64; + for (name, now) in after { + let then = before + .iter() + .find(|(n, _)| n == name) + .map(|(_, c)| c.as_slice()) + .unwrap_or(&[]); + pages += now + .iter() + .enumerate() + .filter(|(index, checksum)| then.get(*index) != Some(*checksum)) + .count() as u64; + } + pages * PAGE as u64 +} + +fn table_bytes(dir: &str) -> u64 { + fn walk(dir: &std::path::Path, total: &mut u64) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.is_dir() { + walk(&path, total); + } else if let Ok(meta) = entry.metadata() { + *total += meta.len(); + } + } + } + let mut total = 0; + walk(std::path::Path::new(dir), &mut total); + total +} + +#[test] +#[ignore = "a measurement, not an assertion"] +fn local_write_bandwidth() { + let dir = "tests/data/local_write_bandwidth"; + let config = DiskConfig::new_with_table_name( + dir, + WriteBandwidthWorkTable::name_snake_case(), + WriteBandwidthWorkTable::version(), + ); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_io() + .enable_time() + .build() + .unwrap(); + + runtime.block_on(async { + remove_dir_if_exists(dir.to_string()).await; + let engine = WriteBandwidthPersistenceEngine::new(config.clone()).await.unwrap(); + let table = WriteBandwidthWorkTable::load(engine).await.unwrap(); + let payload = "x".repeat(4096); + + // ---- bulk load: consecutive pages, which is the batch path's shape + const ROWS: u64 = 25_000; + let at = std::time::Instant::now(); + for id in 0..ROWS { + table + .insert(WriteBandwidthRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + table.wait_for_ops().await.expect("the queue drains"); + let bulk = at.elapsed().as_secs_f64(); + let bytes = table_bytes(dir); + + // ---- scattered updates into what already exists + const UPDATES: u64 = 2_000; + let replacement = "y".repeat(4096); + let pages_before = page_checksums(dir); + let at = std::time::Instant::now(); + for n in 0..UPDATES { + // Spread across the whole table rather than a contiguous run. + let id = (n * (ROWS / UPDATES)) % ROWS; + table + .update(WriteBandwidthRow { + id, + payload: replacement.clone(), + }) + .await + .unwrap(); + } + table.wait_for_ops().await.expect("the queue drains"); + let scattered = at.elapsed().as_secs_f64(); + let scattered_bytes = written_bytes(&pages_before, &page_checksums(dir)); + + println!("table {:.1} MB on disk", bytes as f64 / 1e6); + println!( + " bulk insert, {ROWS} rows : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s", + bulk * 1e3, + bytes as f64 / 1e6 / bulk, + ROWS as f64 / bulk, + ); + println!( + " scattered update, {UPDATES} rows : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s {:.1} MB written", + scattered * 1e3, + scattered_bytes as f64 / 1e6 / scattered, + UPDATES as f64 / scattered, + scattered_bytes as f64 / 1e6, + ); + + // ---- the same inserts with nothing underneath them + let memory = WriteBandwidthMemoryWorkTable::default(); + let at = std::time::Instant::now(); + for id in 0..ROWS { + memory + .insert(WriteBandwidthMemoryRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + let in_memory = at.elapsed().as_secs_f64(); + println!( + " in memory, no persistence : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s", + in_memory * 1e3, + bytes as f64 / 1e6 / in_memory, + ROWS as f64 / in_memory, + ); + println!( + " ^ persistence adds {:.1} ms on top of {:.1} ms of table work", + (bulk - in_memory) * 1e3, + in_memory * 1e3, + ); + + remove_dir_if_exists(dir.to_string()).await; + }); +} diff --git a/tests/persistence/mod.rs b/tests/persistence/mod.rs index e282cc9f..157b644f 100644 --- a/tests/persistence/mod.rs +++ b/tests/persistence/mod.rs @@ -5,15 +5,21 @@ use worktable::worktable; mod bulk_delete_durability; mod bulk_load_stall; mod concurrent; +mod concurrent_upsert_batch; +mod custom_page_size; mod duplicate_key_index_reload; mod exact_boundary_load; mod failure; mod in_place_durability; mod index_page; +mod insert_cost_shape; +mod insert_latency; mod insert_many; mod insert_many_bench; mod loaded_index_growth; +mod local_write_bandwidth; mod multi_row_backend_order; +mod persistence_is_what; mod read; mod recovery_load; mod same_size_in_place; diff --git a/tests/persistence/persistence_is_what.rs b/tests/persistence/persistence_is_what.rs new file mode 100644 index 00000000..ca17484a --- /dev/null +++ b/tests/persistence/persistence_is_what.rs @@ -0,0 +1,60 @@ +//! Is the persistence path waiting, or working? +//! +//! Bulk load persists at about 310 MB/s while the disk under it does gigabytes +//! and DataBucket's own write path does 500+ MB/s single threaded. So something +//! between them is the limit. CPU time against wall time says which kind of +//! limit it is: near or above wall means it is computing, well under means it +//! is waiting. + +use worktable::prelude::PersistedWorkTable; +use worktable::prelude::*; +use worktable_codegen::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: PersistShape, + persist: true, + columns: { id: u64 primary_key, payload: String } +); + +#[test] +#[ignore = "a measurement, not an assertion"] +fn is_persistence_waiting_or_working() { + let dir = "tests/data/persistence_is_what"; + let config = DiskConfig::new_with_table_name( + dir, + PersistShapeWorkTable::name_snake_case(), + PersistShapeWorkTable::version(), + ); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + remove_dir_if_exists(dir.to_string()).await; + let engine = PersistShapePersistenceEngine::new(config).await.unwrap(); + let table = PersistShapeWorkTable::load(engine).await.unwrap(); + let payload = "x".repeat(4096); + + // Marked so the times either side can be attributed to this and not to + // building the table or tearing it down. + println!("MARK begin"); + let at = std::time::Instant::now(); + for id in 0..25_000u64 { + table + .insert(PersistShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + table.wait_for_ops().await.expect("the queue drains"); + println!("MARK end {:.1} ms", at.elapsed().as_secs_f64() * 1e3); + + remove_dir_if_exists(dir.to_string()).await; + }); +} diff --git a/tests/persistence/read.rs b/tests/persistence/read.rs index d56ec83c..e6a11c3e 100644 --- a/tests/persistence/read.rs +++ b/tests/persistence/read.rs @@ -1,4 +1,4 @@ -use tokio::fs::File; +use data_bucket::DEFAULT_PAGE_STRIDE; use worktable::prelude::PersistedWorkTable; use worktable::prelude::*; @@ -10,8 +10,10 @@ use crate::remove_dir_if_exists; #[tokio::test] async fn test_info_parse() { - let mut file = File::open("tests/data/expected/test_persist/.wt.data").await.unwrap(); - let info = parse_page::, { TEST_PERSIST_INNER_SIZE as u32 }>(&mut file, 0) + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/.wt.data") + .await + .unwrap(); + let info = parse_page::, { TEST_PERSIST_INNER_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 0) .await .unwrap(); @@ -20,7 +22,7 @@ async fn test_info_parse() { assert_eq!(info.header.previous_id, 0.into()); assert_eq!(info.header.next_id, 0.into()); assert_eq!(info.header.page_type, PageType::SpaceInfo); - assert_eq!(info.header.data_length, 72); + assert_eq!(info.header.data_length, 80); assert_eq!(info.inner.id, 0.into()); assert_eq!(info.inner.page_count, 1); @@ -31,12 +33,13 @@ async fn test_info_parse() { #[tokio::test] async fn test_primary_index_parse() { - let mut file = File::open("tests/data/expected/test_persist/primary.wt.idx") - .await - .unwrap(); - let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/primary.wt.idx") .await .unwrap(); + let index = + parse_page::, { data_bucket::INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) + .await + .unwrap(); assert_eq!(index.header.space_id, 0.into()); assert_eq!(index.header.page_id, 2.into()); @@ -66,12 +69,13 @@ async fn test_primary_index_parse() { #[tokio::test] async fn test_another_idx_index_parse() { - let mut file = File::open("tests/data/expected/test_persist/another_idx.wt.idx") - .await - .unwrap(); - let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/another_idx.wt.idx") .await .unwrap(); + let index = + parse_page::, { data_bucket::INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) + .await + .unwrap(); assert_eq!(index.header.space_id, 0.into()); assert_eq!(index.header.page_id, 2.into()); @@ -101,10 +105,16 @@ async fn test_another_idx_index_parse() { #[tokio::test] async fn test_data_parse() { - let mut file = File::open("tests/data/expected/test_persist/.wt.data").await.unwrap(); - let data = parse_data_page::<{ TEST_PERSIST_PAGE_SIZE as u32 }, { TEST_PERSIST_INNER_SIZE }>(&mut file, 1) + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/.wt.data") .await .unwrap(); + let data = parse_data_page::< + { TEST_PERSIST_PAGE_SIZE as u32 }, + { TEST_PERSIST_INNER_SIZE }, + { TEST_PERSIST_PAGE_SIZE as u32 }, + >(&mut file, 1) + .await + .unwrap(); assert_eq!(data.header.space_id, 0.into()); assert_eq!(data.header.page_id, 1.into()); diff --git a/tests/persistence/recovery_load.rs b/tests/persistence/recovery_load.rs index 3cf8feaf..611f1e4c 100644 --- a/tests/persistence/recovery_load.rs +++ b/tests/persistence/recovery_load.rs @@ -52,7 +52,7 @@ async fn recovery_mode_reads_valid_rows_through_a_surviving_secondary_index() { let table_dir = format!("{DIR}/{}", RecoveryLoadWorkTable::name_snake_case()); let primary_path = format!("{table_dir}/primary{WT_INDEX_EXTENSION}"); - tokio::fs::rename(&primary_path, format!("{primary_path}.damaged")) + worktable::prelude::fsx::rename(&primary_path, format!("{primary_path}.damaged")) .await .unwrap(); @@ -104,7 +104,7 @@ async fn recovery_mode_rejects_corrupt_rows_reached_through_a_secondary_index() let table_dir = format!("{CORRUPT_DIR}/{}", RecoveryLoadWorkTable::name_snake_case()); let primary_path = format!("{table_dir}/primary{WT_INDEX_EXTENSION}"); - tokio::fs::rename(&primary_path, format!("{primary_path}.damaged")) + worktable::prelude::fsx::rename(&primary_path, format!("{primary_path}.damaged")) .await .unwrap(); @@ -127,7 +127,7 @@ async fn recovery_mode_rejects_corrupt_rows_reached_through_a_secondary_index() .downcast_ref::() .expect("recovery must return a typed corruption error"); assert!( - typed.reason().contains("project_idx") && typed.reason().contains("key does not match"), + typed.reason().contains("v3 data page checksum"), "unexpected recovery-load reason: {}", typed.reason() ); diff --git a/tests/persistence/s3/mod.rs b/tests/persistence/s3/mod.rs index 04957c96..d8c9844a 100644 --- a/tests/persistence/s3/mod.rs +++ b/tests/persistence/s3/mod.rs @@ -1,7 +1,14 @@ use crate::remove_dir_if_exists; +// A tokio `TcpStream`, so tokio's extension traits: this is the mock S3 +// server the test talks to, not the storage path the crate took off tokio. +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tokio::task::JoinHandle; +use worktable::database_s3_persistence; use worktable::prelude::*; use worktable::s3_sync_persistence; use worktable::worktable; @@ -12,19 +19,58 @@ worktable!( columns: { id: u64 primary_key autoincrement, value: u64, + payload: String, }, ); s3_sync_persistence!(TestS3WorkTable); +database_s3_persistence!(TestS3WorkTable); + +fn data_page_image(address: worktable::data_bucket::storage::PageAddress, rows: u32, fill: u8) -> Vec { + use worktable::data_bucket::{DataPage, GeneralHeader, INNER_PAGE_SIZE, Link, PAGE_SIZE, PageType}; -async fn fake_s3() -> (String, JoinHandle<()>) { + let mut page = DataPage::::new(); + for index in 0..rows { + let offset = index * 64; + page.update_at( + Link { + page_id: address.page_id, + offset, + length: 64, + }, + &[fill; 64], + ) + .unwrap(); + } + let mut header = GeneralHeader::new(address.page_id, PageType::Data, address.space_id); + header.data_length = page.length; + let mut image = worktable::prelude::rkyv::to_bytes::(&header) + .unwrap() + .to_vec(); + image.extend_from_slice(&page.encode(INNER_PAGE_SIZE).unwrap()); + assert_eq!(image.len(), PAGE_SIZE); + image +} + +#[derive(Clone, Default)] +struct FakeS3State { + objects: Arc>>>, + puts: Arc>>, + gets: Arc>>, + reject_manifest_puts: Arc, +} + +async fn fake_s3() -> (String, FakeS3State, JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); + let state = FakeS3State::default(); + let server_state = state.clone(); let task = tokio::spawn(async move { loop { let Ok((mut socket, _)) = listener.accept().await else { break; }; + let state = server_state.clone(); tokio::spawn(async move { let mut request = Vec::new(); let mut chunk = [0_u8; 8192]; @@ -57,22 +103,266 @@ async fn fake_s3() -> (String, JoinHandle<()>) { request.extend_from_slice(&chunk[..read]); } - let is_list = request.starts_with(b"GET "); - let body = if is_list { - "test01000false" + let request_line = std::str::from_utf8(&request[..header_end]) + .unwrap() + .lines() + .next() + .unwrap(); + let mut request_parts = request_line.split_whitespace(); + let method = request_parts.next().unwrap(); + let target = request_parts.next().unwrap(); + let path = target.split('?').next().unwrap(); + let key = path.strip_prefix("/test/").unwrap_or(path.trim_start_matches('/')); + + let headers = std::str::from_utf8(&request[..header_end]) + .unwrap() + .lines() + .skip(1) + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_string())) + .collect::>(); + + let (status, content_type, body, etag) = if method == "PUT" { + let body = request[header_end..header_end + content_length].to_vec(); + if key.ends_with("/manifest.v1") && state.reject_manifest_puts.load(Ordering::Acquire) { + ( + "500 Internal Server Error", + "text/plain", + b"injected failure".to_vec(), + None, + ) + } else { + let mut objects = state.objects.lock().unwrap(); + let current_etag = objects + .get(key) + .map(|body| format!("\"{}\"", blake3::hash(body).to_hex())); + let conflict = headers.get("if-none-match").is_some_and(|value| value == "*") + && current_etag.is_some() + || headers + .get("if-match") + .is_some_and(|value| Some(value) != current_etag.as_ref()); + if conflict { + ( + "412 Precondition Failed", + "text/plain", + b"conflict".to_vec(), + current_etag, + ) + } else { + let etag = format!("\"{}\"", blake3::hash(&body).to_hex()); + objects.insert(key.to_string(), body); + drop(objects); + state.puts.lock().unwrap().push((key.to_string(), content_length)); + ("200 OK", "application/octet-stream", Vec::new(), Some(etag)) + } + } + } else if target.contains("list-type=2") { + ( + "200 OK", + "application/xml", + b"test01000false".to_vec(), + None, + ) + } else if let Some(stored) = state.objects.lock().unwrap().get(key).cloned() { + state.gets.lock().unwrap().push(key.to_string()); + let etag = format!("\"{}\"", blake3::hash(&stored).to_hex()); + if method == "HEAD" { + ("200 OK", "application/octet-stream", Vec::new(), Some(etag)) + } else if let Some(range) = headers.get("range") { + let range = range.strip_prefix("bytes=").unwrap(); + let (start, end) = range.split_once('-').unwrap(); + let start = start.parse::().unwrap(); + let end = end.parse::().unwrap(); + ( + "206 Partial Content", + "application/octet-stream", + stored[start..=end].to_vec(), + Some(etag), + ) + } else { + ("200 OK", "application/octet-stream", stored, Some(etag)) + } } else { - "" + ("404 Not Found", "text/plain", b"not found".to_vec(), None) }; + let etag = etag.map_or_else(String::new, |value| format!("ETag: {value}\r\n")); let response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\n{etag}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() ); socket.write_all(response.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); }); } }); - (format!("http://{address}"), task) + (format!("http://{address}"), state, task) +} + +#[test] +fn data_bucket_domain_uses_one_generated_catalog_and_page_range_reads() { + use worktable::S3Database; + use worktable::data_bucket::storage::s3::S3Config as DataBucketS3Config; + use worktable::data_bucket::storage::{PageAddress, PageKind, StorageDomainId}; + use worktable::data_bucket::{PAGE_SIZE, PageId, SpaceId}; + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_io() + .enable_time() + .build() + .unwrap(); + runtime.block_on(async { + let (endpoint, state, server) = fake_s3().await; + let domain = StorageDomainId([9; 16]); + let config = DataBucketS3Config { + bucket_name: "test".to_string(), + endpoint, + access_key: "test".to_string(), + secret_key: "test".to_string(), + session_token: None, + region: "auto".to_string(), + prefix: Some("db-domain".to_string()), + virtual_host_style: false, + }; + let database = S3Database::open_s3(domain, 1, config.clone()).unwrap(); + let table = database.register_table("orders", 3).unwrap(); + let address = PageAddress { + domain, + table_id: table, + space_id: SpaceId(2), + page_id: PageId::from(4), + page_kind: PageKind::Data, + }; + + let before = state.puts.lock().unwrap().iter().map(|(_, bytes)| bytes).sum::(); + let image = data_page_image(address, 9, 0x5a); + let mut generation = database.begin_generation().unwrap(); + generation.put_page(address, image.clone(), 9, 777); + database.commit_generation(generation.finish()).unwrap(); + let after = state.puts.lock().unwrap().iter().map(|(_, bytes)| bytes).sum::(); + println!("DB_S3_INCREMENTAL_BYTES={}", after - before); + assert!( + after - before < PAGE_SIZE * 3, + "one page and its catalog update uploaded {} bytes", + after - before + ); + assert_eq!(database.read_page(address).unwrap(), Some(image.clone())); + assert_eq!(database.catalog().system_tables()[0].row_count, 9); + + let mut generation = database.begin_generation().unwrap(); + for page in 1_u32..=200 { + if page == 4 { + continue; + } + generation.put_page( + PageAddress { + page_id: PageId::from(page), + ..address + }, + data_page_image( + PageAddress { + page_id: PageId::from(page), + ..address + }, + 1, + page as u8, + ), + 1, + 64, + ); + } + database.commit_generation(generation.finish()).unwrap(); + + drop(database); + let reopened = S3Database::open_s3(domain, 2, config).unwrap(); + assert_eq!(reopened.read_page(address).unwrap(), Some(image)); + assert_eq!(reopened.catalog().system_pages().len(), 200); + + let address = PageAddress { + page_id: PageId::from(150), + ..address + }; + let before = state.puts.lock().unwrap().iter().map(|(_, bytes)| bytes).sum::(); + let image = data_page_image(address, 2, 0xa5); + let mut generation = reopened.begin_generation().unwrap(); + generation.put_page(address, image.clone(), 2, 128); + reopened.commit_generation(generation.finish()).unwrap(); + let after = state.puts.lock().unwrap().iter().map(|(_, bytes)| bytes).sum::(); + println!("DB_S3_LARGE_CATALOG_INCREMENTAL_BYTES={}", after - before); + assert!( + after - before < PAGE_SIZE * 4, + "one page update with a multi-page catalog uploaded {} bytes", + after - before + ); + assert_eq!(reopened.read_page(address).unwrap(), Some(image)); + server.abort(); + }); +} + +#[test] +fn generated_table_uses_the_shared_database_domain_and_restores() { + use worktable::data_bucket::storage::StorageDomainId; + use worktable::data_bucket::storage::s3::S3Config as DataBucketS3Config; + use worktable::{DatabaseS3DiskConfig, S3Database}; + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_io() + .enable_time() + .build() + .unwrap(); + runtime.block_on(async { + let path = "tests/data/s3/database_domain"; + remove_dir_if_exists(path.to_string()).await; + let (endpoint, _, server) = fake_s3().await; + let domain = StorageDomainId([0x44; 16]); + let s3 = DataBucketS3Config { + bucket_name: "test".to_string(), + endpoint, + access_key: "test".to_string(), + secret_key: "test".to_string(), + session_token: None, + region: "auto".to_string(), + prefix: Some("generated-database".to_string()), + virtual_host_style: false, + }; + let disk = DiskConfig::new_with_table_name(path, "orders", TestS3WorkTable::version()); + let database = S3Database::open_s3(domain, 10, s3.clone()).unwrap(); + { + let engine = TestS3DatabaseS3PersistenceEngine::new(DatabaseS3DiskConfig { + disk: disk.clone(), + database: database.clone(), + }) + .await + .unwrap(); + let table = TestS3WorkTable::load(engine).await.unwrap(); + table + .insert(TestS3Row { + id: table.get_next_pk().into(), + value: 7, + payload: "remote".repeat(200), + }) + .await + .unwrap(); + table.wait_for_ops().await.unwrap(); + assert!(!database.catalog().system_pages().is_empty()); + } + + remove_dir_if_exists(disk.table_path().to_string()).await; + let reopened_database = S3Database::open_s3(domain, 11, s3).unwrap(); + let engine = TestS3DatabaseS3PersistenceEngine::new(DatabaseS3DiskConfig { + disk: disk.clone(), + database: reopened_database.clone(), + }) + .await + .unwrap(); + let reopened = TestS3WorkTable::load(engine).await.unwrap(); + let row = reopened.select(0).expect("row restored through the database catalog"); + assert_eq!(row.value, 7); + assert_eq!(reopened_database.catalog().system_tables().len(), 1); + remove_dir_if_exists(path.to_string()).await; + server.abort(); + }); } #[test] @@ -87,7 +377,7 @@ fn s3_engine_reuses_logical_persistence_for_a_loaded_default_arctic_table() { runtime.block_on(async { remove_dir_if_exists("tests/data/s3/compile_test".to_string()).await; - let (endpoint, server) = fake_s3().await; + let (endpoint, s3, server) = fake_s3().await; let config = S3DiskConfig { disk: DiskConfig::new_with_table_name( @@ -112,16 +402,17 @@ fn s3_engine_reuses_logical_persistence_for_a_loaded_default_arctic_table() { { let engine = TestS3PersistenceEngine::new(config.disk.clone()).await.unwrap(); let table = TestS3WorkTable::load(engine).await.unwrap(); - for value in 0..512 { + for value in 0..2600 { table .insert(TestS3Row { id: table.get_next_pk().into(), value, + payload: format!("{value:0>4096}"), }) .await .unwrap(); } - assert_eq!(table.select_all().execute().unwrap().len(), 512); + assert_eq!(table.select_all().execute().unwrap().len(), 2600); table.wait_for_ops().await.unwrap(); } @@ -135,24 +426,109 @@ fn s3_engine_reuses_logical_persistence_for_a_loaded_default_arctic_table() { let mut row = table.select(257).expect("persisted row"); row.value = 10_000; table.update(row).await.unwrap(); - table.insert(TestS3Row { id: 512, value: 512 }).await.unwrap(); + table.wait_for_ops().await.unwrap(); + + let uploaded_before = s3.puts.lock().unwrap().iter().map(|(_, length)| length).sum::(); + let table_bytes = std::fs::read_dir(config.disk.table_path()) + .unwrap() + .map(|entry| entry.unwrap().metadata().unwrap().len() as usize) + .sum::(); + let mut row = table.select(1300).expect("persisted row"); + row.value = 20_000; + table.update(row).await.unwrap(); + table.wait_for_ops().await.unwrap(); + let uploaded_after = s3.puts.lock().unwrap().iter().map(|(_, length)| length).sum::(); + let incremental_bytes = uploaded_after - uploaded_before; + println!( + "S3_TRANSFER table_bytes={table_bytes} incremental_bytes={incremental_bytes} ratio={:.3}", + incremental_bytes as f64 / table_bytes as f64 + ); + assert!( + incremental_bytes < data_bucket::PAGE_SIZE * 2, + "one row update uploaded {incremental_bytes} bytes for a {table_bytes}-byte table" + ); + + table + .insert(TestS3Row { + id: 2600, + value: 2600, + payload: "x".repeat(4096), + }) + .await + .unwrap(); + table.wait_for_ops().await.unwrap(); table.delete(100).await.unwrap(); table.wait_for_ops().await.unwrap(); + + // New immutable segments may arrive before the commit point. If the + // manifest PUT fails, a fresh reader must still see the preceding + // complete table generation. + s3.reject_manifest_puts.store(true, Ordering::Release); + table + .insert(TestS3Row { + id: 2601, + value: 2601, + payload: "y".repeat(4096), + }) + .await + .unwrap(); + assert!(table.wait_for_ops().await.is_err()); } + s3.reject_manifest_puts.store(false, Ordering::Release); + + let puts = s3.puts.lock().unwrap().clone(); + assert!(puts.iter().any(|(key, _)| key.ends_with("/manifest.v1"))); + assert!(puts.iter().any(|(key, _)| key.contains("/chunks/"))); + assert!( + puts.iter() + .all(|(key, _)| key.ends_with("/manifest.v1") || key.contains("/chunks/")), + "new S3 writes must use immutable segments and the table manifest: {puts:?}" + ); + // Removing the complete local table forces a strict remote restore. + // The one manifest must reconstruct data and every index before the + // directory is atomically installed for DiskPersistenceEngine. + remove_dir_if_exists(config.disk.table_path().to_string()).await; { - let engine = TestS3PersistenceEngine::new(config.disk.clone()).await.unwrap(); + let engine = TestS3S3SyncPersistenceEngine::new(config.clone()).await.unwrap(); let table = TestS3WorkTable::load(engine).await.unwrap(); let rows = table.select_all().execute().unwrap(); - assert_eq!(rows.len(), 512); + assert_eq!(rows.len(), 2600); assert!(table.select(100).is_none(), "deleted primary key returned"); - for id in (0..=512).filter(|id| *id != 100) { + assert!(table.select(2601).is_none(), "uncommitted S3 generation became visible"); + for id in (0..=2600).filter(|id| *id != 100) { let row = table.select(id).expect("every primary key survives"); - let expected = if id == 257 { 10_000 } else { id }; + let expected = if id == 257 { + 10_000 + } else if id == 1300 { + 20_000 + } else { + id + }; assert_eq!(row.value, expected, "wrong value for primary key {id}"); } } + // A committed manifest is authoritative, but a failed restore must + // leave a usable local table untouched until the remote damage is + // repaired. + let missing_segment = s3 + .gets + .lock() + .unwrap() + .iter() + .find(|key| key.contains("/chunks/")) + .cloned() + .unwrap(); + s3.objects.lock().unwrap().remove(&missing_segment); + assert!(TestS3S3SyncPersistenceEngine::new(config.clone()).await.is_err()); + { + let engine = TestS3PersistenceEngine::new(config.disk.clone()).await.unwrap(); + let table = TestS3WorkTable::load(engine).await.unwrap(); + assert_eq!(table.select_all().execute().unwrap().len(), 2600); + assert_eq!(table.select(257).unwrap().value, 10_000); + } + server.abort(); }); } diff --git a/tests/persistence/schema.rs b/tests/persistence/schema.rs index c815a18a..728d690f 100644 --- a/tests/persistence/schema.rs +++ b/tests/persistence/schema.rs @@ -1,7 +1,6 @@ -use tokio::fs::File; - use super::*; use crate::remove_dir_if_exists; +use data_bucket::DEFAULT_PAGE_STRIDE; worktable!( name: SchemaMetadata, @@ -37,10 +36,13 @@ async fn generated_schema_is_persisted_and_mismatches_are_rejected() { let table = SchemaMetadataWorkTable::load(engine).await.unwrap(); table.close().await.unwrap(); - let mut file = File::open(format!("{table_path}/{}", WT_DATA_EXTENSION)).await.unwrap(); - let info = parse_page::, { PAGE_SIZE as u32 }>(&mut file, 0) + let mut file = worktable::prelude::fsx::open(format!("{table_path}/{}", WT_DATA_EXTENSION)) .await .unwrap(); + let info = + parse_page::, { data_bucket::INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 0) + .await + .unwrap(); assert_eq!( info.inner.row_schema, vec![ @@ -80,10 +82,13 @@ async fn loading_a_legacy_empty_schema_does_not_rewrite_the_file() { let table = SchemaMetadataWorkTable::load(engine).await.unwrap(); table.close().await.unwrap(); - let mut file = File::open(format!("{table_path}/{}", WT_DATA_EXTENSION)).await.unwrap(); - let info = parse_page::, { PAGE_SIZE as u32 }>(&mut file, 0) + let mut file = worktable::prelude::fsx::open(format!("{table_path}/{}", WT_DATA_EXTENSION)) .await .unwrap(); + let info = + parse_page::, { data_bucket::INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 0) + .await + .unwrap(); assert!(info.inner.row_schema.is_empty()); assert!(info.inner.primary_key_fields.is_empty()); assert!(info.inner.secondary_index_types.is_empty()); diff --git a/tests/persistence/space_data.rs b/tests/persistence/space_data.rs index 191c0322..70376369 100644 --- a/tests/persistence/space_data.rs +++ b/tests/persistence/space_data.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use data_bucket::DEFAULT_PAGE_STRIDE; +use worktable::prelude::HashMap; use data_bucket::{INNER_PAGE_SIZE, Link, PAGE_SIZE, parse_general_header_by_index}; use worktable::prelude::{SpaceData, SpaceDataOps}; @@ -46,7 +47,9 @@ async fn rewriting_one_link_does_not_inflate_the_persisted_data_length() { space.save_batch_data(batch).await.unwrap(); assert_eq!(space.current_data_length, 48); - let header = parse_general_header_by_index(&mut space.data_file, 1).await.unwrap(); + let header = parse_general_header_by_index::(&mut space.data_file, 1) + .await + .unwrap(); assert_eq!(header.data_length, 48); drop(space); @@ -86,7 +89,9 @@ async fn reclaiming_thousands_of_pages_bounds_the_info_page_instead_of_corruptin assert!(kept < 2_000, "the overflow condition was not constructed"); // Page 1 must be untouched by the info persist. - let header = parse_general_header_by_index(&mut space.data_file, 1).await.unwrap(); + let header = parse_general_header_by_index::(&mut space.data_file, 1) + .await + .unwrap(); assert_eq!(header.page_type, PageType::Data); drop(space); @@ -132,8 +137,9 @@ async fn a_data_file_ending_on_an_exact_page_boundary_reopens() { // Fill page 1 completely: the file then ends exactly on a page boundary // (2 * PAGE_SIZE), the case where the old floor division computed a last // page id one past EOF and reopening failed on the header read. - let full_page = vec![3u8; INNER_PAGE_SIZE]; - let batch = HashMap::from([(1.into(), vec![(link(1, 0, INNER_PAGE_SIZE as u32), full_page)])]); + let row_capacity = INNER_PAGE_SIZE - data_bucket::DATA_TRAILER_SIZE - data_bucket::ROW_SLOT_SIZE; + let full_page = vec![3u8; row_capacity]; + let batch = HashMap::from([(1.into(), vec![(link(1, 0, row_capacity as u32), full_page)])]); space.save_batch_data(batch).await.unwrap(); drop(space); @@ -146,7 +152,7 @@ async fn a_data_file_ending_on_an_exact_page_boundary_reopens() { let space = TestSpaceData::from_table_files_path(&dir, 1).await.unwrap(); assert_eq!(space.last_page_id, 1); - assert_eq!(space.current_data_length, INNER_PAGE_SIZE as u32); + assert_eq!(space.current_data_length, row_capacity as u32); drop(space); std::fs::remove_dir_all(&dir).unwrap(); diff --git a/tests/persistence/space_index/indexset_compatibility.rs b/tests/persistence/space_index/indexset_compatibility.rs index c9c9ba9a..6f702a5a 100644 --- a/tests/persistence/space_index/indexset_compatibility.rs +++ b/tests/persistence/space_index/indexset_compatibility.rs @@ -1,4 +1,5 @@ mod sized { + use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use data_bucket::{INNER_PAGE_SIZE, Link}; @@ -11,7 +12,7 @@ mod sized { async fn test_indexset_node_creation() { remove_file_if_exists("tests/data/space_index/indexset/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/indexset/process_create_node.wt.idx", 0.into(), 1, @@ -46,7 +47,7 @@ mod sized { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/indexset/process_insert_at.wt.idx", 0.into(), 1, @@ -81,7 +82,7 @@ mod sized { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/indexset/process_insert_at_big_amount.wt.idx", 0.into(), 1, @@ -124,6 +125,7 @@ mod sized { } mod unsized_ { + use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use crate::{check_if_files_are_same, remove_file_if_exists}; @@ -137,7 +139,7 @@ mod unsized_ { async fn test_indexset_node_creation() { remove_file_if_exists("tests/data/space_index_unsized/indexset/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/indexset/process_create_node.wt.idx", 0.into(), 1, @@ -172,7 +174,7 @@ mod unsized_ { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/indexset/process_insert_at.wt.idx", 0.into(), 1, @@ -210,7 +212,7 @@ mod unsized_ { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx", 0.into(), 1, diff --git a/tests/persistence/space_index/unsized_write.rs b/tests/persistence/space_index/unsized_write.rs index 61175c97..da84b980 100644 --- a/tests/persistence/space_index/unsized_write.rs +++ b/tests/persistence/space_index/unsized_write.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use data_bucket::{INNER_PAGE_SIZE, Link}; @@ -14,7 +15,7 @@ mod run_first { async fn test_space_index_process_create_node() { remove_file_if_exists("tests/data/space_index_unsized/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_create_node.wt.idx", 0.into(), 1, @@ -52,7 +53,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_create_second_node.wt.idx", 0.into(), 1, @@ -90,7 +91,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_remove_node.wt.idx", 0.into(), 1, @@ -128,7 +129,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at.wt.idx", 0.into(), 1, @@ -175,7 +176,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at_big_amount.wt.idx", 0.into(), 1, @@ -249,7 +250,7 @@ async fn test_space_index_process_remove_at() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_remove_at.wt.idx", 0.into(), 1, @@ -296,7 +297,7 @@ async fn test_space_index_process_remove_at_node_id() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_remove_at_node_id.wt.idx", 0.into(), 1, @@ -344,7 +345,7 @@ async fn test_space_index_process_insert_at_with_node_id_update() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at_with_node_id_update.wt.idx", 0.into(), 1, @@ -391,7 +392,7 @@ async fn test_space_index_process_insert_at_removed_place() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at_removed_place.wt.idx", 0.into(), 1, @@ -484,7 +485,7 @@ async fn test_space_index_process_create_node_after_remove() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_create_node_after_remove.wt.idx", 0.into(), 1, @@ -522,7 +523,7 @@ async fn test_space_index_process_split_node() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_split_node.wt.idx", 0.into(), 1, diff --git a/tests/persistence/space_index/write.rs b/tests/persistence/space_index/write.rs index d6f8d2c6..81a0befa 100644 --- a/tests/persistence/space_index/write.rs +++ b/tests/persistence/space_index/write.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use data_bucket::{INNER_PAGE_SIZE, Link}; @@ -14,7 +15,7 @@ mod run_first { async fn test_space_index_process_create_node() { remove_file_if_exists("tests/data/space_index/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_create_node.wt.idx", 0.into(), 1, @@ -52,7 +53,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_create_second_node.wt.idx", 0.into(), 1, @@ -90,7 +91,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at.wt.idx", 0.into(), 1, @@ -137,7 +138,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at_big_amount.wt.idx", 0.into(), 1, @@ -210,7 +211,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_remove_node.wt.idx", 0.into(), 1, @@ -249,7 +250,7 @@ async fn test_space_index_process_insert_at_with_node_id_update() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at_with_node_id_update.wt.idx", 0.into(), 1, @@ -296,7 +297,7 @@ async fn test_space_index_process_remove_at() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_remove_at.wt.idx", 0.into(), 1, @@ -343,7 +344,7 @@ async fn test_space_index_process_remove_at_node_id() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_remove_at_node_id.wt.idx", 0.into(), 1, @@ -390,7 +391,7 @@ async fn test_space_index_process_insert_at_removed_place() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at_removed_place.wt.idx", 0.into(), 1, @@ -483,7 +484,7 @@ async fn test_space_index_process_create_node_after_remove() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_create_node_after_remove.wt.idx", 0.into(), 1, @@ -521,7 +522,7 @@ async fn test_space_index_process_split_node() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_split_node.wt.idx", 0.into(), 1, @@ -560,10 +561,13 @@ async fn test_space_index_process_split_node() { async fn batch_split_then_max_remove_then_historical_identity_insert_applies() { remove_file_if_exists("tests/data/space_index/batch_alias.wt.idx".to_string()).await; - let mut space_index = - SpaceIndex::::new("tests/data/space_index/batch_alias.wt.idx", 0.into(), 1) - .await - .unwrap(); + let mut space_index = SpaceIndex::::new( + "tests/data/space_index/batch_alias.wt.idx", + 0.into(), + 1, + ) + .await + .unwrap(); fn link(offset: u32) -> Link { Link { @@ -636,7 +640,7 @@ async fn batch_split_then_max_remove_then_historical_identity_insert_applies() { async fn batch_replay_of_real_cdc_stream_with_splits_matches_the_source() { remove_file_if_exists("tests/data/space_index/batch_cdc_replay.wt.idx".to_string()).await; - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/batch_cdc_replay.wt.idx", 0.into(), 1, diff --git a/tests/persistence/sync/repeated_string_upsert.rs b/tests/persistence/sync/repeated_string_upsert.rs index 02ae114d..6ad12a2f 100644 --- a/tests/persistence/sync/repeated_string_upsert.rs +++ b/tests/persistence/sync/repeated_string_upsert.rs @@ -76,7 +76,7 @@ fn repeated_varying_string_upserts_keep_the_worker_healthy() { .unwrap(); } - timeout(Duration::from_secs(15), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled after repeated string upserts") .expect("persistence worker failed after repeated string upserts"); diff --git a/tests/persistence/sync/string_secondary_index.rs b/tests/persistence/sync/string_secondary_index.rs index 93a6fafc..e4b65ffc 100644 --- a/tests/persistence/sync/string_secondary_index.rs +++ b/tests/persistence/sync/string_secondary_index.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use worktable::prelude::PersistedWorkTable; use worktable::prelude::*; use worktable_codegen::worktable; @@ -104,11 +105,12 @@ fn fragmented_string_index_compacts_after_restart_before_appending() { } let index_path = format!("{path}/fragmented_string_secondary/project_idx.wt.idx"); - let mut index_file = tokio::fs::File::open(index_path).await.unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>( - &mut index_file, - 2, - ) + let mut index_file = worktable::prelude::fsx::open(index_path).await.unwrap(); + let page = parse_page::< + UnsizedIndexPage, + { INNER_PAGE_SIZE as u32 }, + DEFAULT_PAGE_STRIDE, + >(&mut index_file, 2) .await .unwrap(); let utility_size = worktable::data_bucket::UnsizedIndexPageUtility::::persisted_size( diff --git a/tests/persistence/toc/read.rs b/tests/persistence/toc/read.rs index d36d59b0..0f7f14ef 100644 --- a/tests/persistence/toc/read.rs +++ b/tests/persistence/toc/read.rs @@ -1,37 +1,34 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::sync::Arc; use std::sync::atomic::AtomicU32; use data_bucket::{INNER_PAGE_SIZE, Link}; -use tokio::fs::OpenOptions; use worktable::prelude::IndexTableOfContents; #[tokio::test] async fn test_index_table_of_contents_read() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/persist_index_table_of_contents.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/persist_index_table_of_contents.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(1)); - let toc = - IndexTableOfContents::::parse_from_file(&mut file, 0.into(), next_id_gen) - .await - .unwrap(); + let toc = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + next_id_gen, + ) + .await + .unwrap(); assert_eq!(toc.get(&13), Some(1.into())) } #[tokio::test] async fn test_index_table_of_contents_read_from_space() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/test_persist/primary.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/primary.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(1)); - let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -53,14 +50,11 @@ async fn test_index_table_of_contents_read_from_space() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -83,14 +77,11 @@ async fn test_index_table_of_contents_read_from_space_index() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_after_insert() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -113,14 +104,12 @@ async fn test_index_table_of_contents_read_from_space_index_after_insert() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_updated_node_id() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -143,14 +132,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_updated_node_id #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_remove_at_node_id() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -173,14 +159,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_remove_at_node_ #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -214,14 +197,12 @@ async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_create_node_after_remove_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -255,14 +236,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_create_node_aft #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_after_split_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_split_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_split_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -297,36 +275,39 @@ async fn test_index_table_of_contents_read_from_space_index_after_split_node() { #[tokio::test] async fn test_truncated_table_of_contents_is_an_error_not_an_empty_index() { let path = std::env::temp_dir().join(format!("worktable-toc-truncated-{}.wt.idx", uuid::Uuid::new_v4())); - let mut file = OpenOptions::new() - .write(true) - .read(true) - .create(true) - .truncate(true) - .open(&path) - .await - .unwrap(); + let mut file = worktable::prelude::fsx::create(&path).await.unwrap(); - // A DATA_LENGTH of 20 forces the table of contents to span several pages. - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + // A DATA_LENGTH of 32 forces the table of contents to span several pages. + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); for key in 0..10 { toc.insert(key, u32::from(key).into()); } assert!(toc.pages.len() > 1, "fixture must span TOC pages"); toc.persist(&mut file).await.unwrap(); - file.sync_all().await.unwrap(); + worktable::prelude::fsx::sync_all(&mut file).await.unwrap(); // The intact file must round-trip. - let reloaded = IndexTableOfContents::::parse_from_file(&mut file, 0.into(), Arc::new(AtomicU32::new(1))) - .await - .unwrap(); + let reloaded = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + Arc::new(AtomicU32::new(1)), + ) + .await + .unwrap(); assert_eq!(reloaded.pages.len(), toc.pages.len()); // Tear the first table-of-contents page: the file still extends into its // slot, so the load must fail loudly instead of yielding an empty index. - file.set_len(data_bucket::PAGE_SIZE as u64 + 10).await.unwrap(); - let error = IndexTableOfContents::::parse_from_file(&mut file, 0.into(), Arc::new(AtomicU32::new(1))) + worktable::prelude::fsx::set_len(&mut file, data_bucket::PAGE_SIZE as u64 + 10) .await - .unwrap_err(); + .unwrap(); + let error = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + Arc::new(AtomicU32::new(1)), + ) + .await + .unwrap_err(); assert!( error.to_string().contains("table of contents page 1 failed to parse"), "unexpected error: {error:#}" @@ -334,13 +315,16 @@ async fn test_truncated_table_of_contents_is_an_error_not_an_empty_index() { // A file that never grew past page 0 is a real bootstrap and still loads // as a fresh empty table of contents. - file.set_len(0).await.unwrap(); - let bootstrapped = - IndexTableOfContents::::parse_from_file(&mut file, 0.into(), Arc::new(AtomicU32::new(1))) - .await - .unwrap(); + worktable::prelude::fsx::set_len(&mut file, 0).await.unwrap(); + let bootstrapped = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + Arc::new(AtomicU32::new(1)), + ) + .await + .unwrap(); assert_eq!(bootstrapped.pages.len(), 1); drop(file); - tokio::fs::remove_file(&path).await.unwrap(); + worktable::prelude::fsx::remove_file(&path).await.unwrap(); } diff --git a/tests/persistence/toc/unsized_read.rs b/tests/persistence/toc/unsized_read.rs index 1dffc0ad..ed460df7 100644 --- a/tests/persistence/toc/unsized_read.rs +++ b/tests/persistence/toc/unsized_read.rs @@ -1,20 +1,17 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::sync::Arc; use std::sync::atomic::AtomicU32; use data_bucket::{INNER_PAGE_SIZE, Link}; -use tokio::fs::OpenOptions; use worktable::prelude::IndexTableOfContents; #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -37,14 +34,12 @@ async fn test_index_table_of_contents_read_from_space_index_unsized() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_with_two_nodes() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(3)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -78,14 +73,11 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_with_two_nod #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -119,14 +111,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_insert_at() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -149,14 +138,11 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_after_insert #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove_at() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -179,14 +165,12 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove_at_node_id() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -209,14 +193,13 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_create_node_after_remove() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx") - .await - .unwrap(); + let mut file = worktable::prelude::fsx::open( + "tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx", + ) + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, diff --git a/tests/persistence/toc/write.rs b/tests/persistence/toc/write.rs index 0bb04709..910ec0e9 100644 --- a/tests/persistence/toc/write.rs +++ b/tests/persistence/toc/write.rs @@ -1,7 +1,7 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::INNER_PAGE_SIZE; use std::sync::Arc; use std::sync::atomic::AtomicU32; -use tokio::fs::File; use worktable::prelude::IndexTableOfContents; use crate::{check_if_files_are_same, remove_file_if_exists}; @@ -10,11 +10,14 @@ use crate::{check_if_files_are_same, remove_file_if_exists}; async fn test_persist_index_table_of_contents() { remove_file_if_exists("tests/data/persist_index_table_of_contents.wt.idx".to_string()).await; - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new( + 0.into(), + Arc::new(AtomicU32::new(1)), + ); // Compile-time compatibility regression: the public API before PR #63 // returned unit, including for callers that bind the expression's type. let _: () = toc.insert(13, 1.into()); - let mut file = File::create("tests/data/persist_index_table_of_contents.wt.idx") + let mut file = worktable::prelude::fsx::create("tests/data/persist_index_table_of_contents.wt.idx") .await .unwrap(); toc.persist(&mut file).await.unwrap(); diff --git a/tests/persistence/torn_shutdown.rs b/tests/persistence/torn_shutdown.rs index 9891e5cc..c141ab9f 100644 --- a/tests/persistence/torn_shutdown.rs +++ b/tests/persistence/torn_shutdown.rs @@ -137,7 +137,7 @@ fn tear_the_store_repeatedly() { for i in 0..200 { table.insert(row(i)).await.unwrap(); } - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled building the base store") .expect("persistence engine failed"); @@ -263,7 +263,7 @@ fn test_store_survives_torn_shutdowns() { } // And the survivor must still accept writes and a drain. table.insert(row(9_000_000)).await.unwrap(); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled appending to the survivor store") .expect("persistence engine failed"); @@ -407,7 +407,7 @@ fn test_many_clean_sessions_stay_readable() { .unwrap_or_else(|error| panic!("session {session}: insert {next_id} refused: {error:?}")); next_id += 1; } - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .unwrap_or_else(|_| panic!("session {session}: drain stalled")) .expect("persistence engine failed"); diff --git a/tests/persistence/vacuum.rs b/tests/persistence/vacuum.rs index 7048e543..e6383497 100644 --- a/tests/persistence/vacuum.rs +++ b/tests/persistence/vacuum.rs @@ -73,7 +73,7 @@ fn test_vacuum_on_persisted_table_survives_reload() { table.delete(*id).await.unwrap(); } - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence should catch up before vacuum") .expect("persistence engine failed"); @@ -82,7 +82,7 @@ fn test_vacuum_on_persisted_table_survives_reload() { let vacuum = table.vacuum(); let stats = vacuum.vacuum().await.unwrap(); assert!(stats.pages_freed > 0, "vacuum should have moved rows off a page"); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence should catch up after vacuum") .expect("persistence engine failed"); @@ -106,7 +106,7 @@ fn test_vacuum_on_persisted_table_survives_reload() { table.insert(row.clone()).await.unwrap(); rows.insert(id, row); if i % 50 == 49 { - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled after vacuum on persisted table") .expect("persistence engine failed"); @@ -116,7 +116,7 @@ fn test_vacuum_on_persisted_table_survives_reload() { // Without CDC-aware vacuum this stalls forever: the moved links // never reach the persistence stream while their event ids are // consumed, leaving a permanent gap the batch validator defers on. - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled after vacuum on persisted table") .expect("persistence engine failed"); @@ -157,7 +157,7 @@ fn test_vacuum_on_persisted_table_survives_reload() { reused_after_reload_id = reused_id; table.insert(reused_row.clone()).await.unwrap(); rows.insert(reused_id, reused_row); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence should catch up after durable page reuse") .expect("persistence engine failed"); @@ -195,7 +195,7 @@ fn test_vacuum_on_persisted_table_survives_reload() { exchange: "second-reuse-after-reload".to_string(), }; table.insert(second_reused_row).await.unwrap(); - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence should catch up after a second durable page reuse") .expect("persistence engine failed"); @@ -291,7 +291,7 @@ fn test_persisted_vacuum_survives_inserts_reusing_space_mid_sweep() { for id in &deleted { table.delete(*id).await.unwrap(); } - timeout(Duration::from_secs(30), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence should catch up before vacuum") .expect("persistence engine failed"); @@ -337,7 +337,7 @@ fn test_persisted_vacuum_survives_inserts_reusing_space_mid_sweep() { // Longer than the engine's own give-up budget, so a stall surfaces // as its diagnostic naming the missing event id rather than as a // bare timeout here, which says nothing. - timeout(Duration::from_secs(90), table.wait_for_ops()) + timeout(Duration::from_secs(5), table.wait_for_ops()) .await .expect("persistence stalled after a sweep interleaved with inserts") .expect("persistence engine failed"); diff --git a/tests/runtime_execution.rs b/tests/runtime_execution.rs new file mode 100644 index 00000000..36f438c5 --- /dev/null +++ b/tests/runtime_execution.rs @@ -0,0 +1,296 @@ +//! Runtime selection must execute work, not merely retain metadata. +use std::sync::{Arc, Mutex}; +use std::thread::ThreadId; +use worktable::{prelude::*, runtimes, worktable}; + +runtimes! { scheduled: nagoya(shared_slot), } +worktable! { + name: Scheduled, + runtime: nagoya(shared_slot), + columns: { id: u64 primary_key, value: u64, group: u64 }, + indexes: { group_idx: group }, + queries: { + update runtime scheduled: { ValueById(value) by id }, + delete runtime scheduled: { ByGroup() by group }, + in_place runtime scheduled: { ValueById(value) by id }, + } +} + +static DISPATCH_THREAD: Mutex> = Mutex::new(None); +struct Observed; +impl Profile for Observed { + type Backend = NagoyaRt; + fn tuning() -> Tuning { + scheduled::tuning() + } + fn dispatcher() -> worktable::runtime::Dispatch { + |work| { + Box::pin(worktable::runtime::run_on::, _>(async move { + *DISPATCH_THREAD.lock().unwrap() = Some(std::thread::current().id()); + work(); + })) + } + } +} + +#[test] +fn generated_profiles_dispatch_mutations_and_owned_selects() { + nagoya::block_on(async { + let table = Arc::new(ScheduledWorkTable::default()); + for id in 0..10 { + table + .insert(ScheduledRow { + id, + value: id, + group: id % 2, + }) + .await + .unwrap(); + } + table + .update_value_by_id(ValueByIdQuery { value: 100 }, 9u64) + .await + .unwrap(); + let caller = std::thread::current().id(); + table + .update_value_by_id_in_place( + move |value| { + assert_ne!(std::thread::current().id(), caller); + *value = 101.into(); + }, + 9u64, + ) + .await + .unwrap(); + assert_eq!(table.select(9u64).unwrap().value, 101); + assert!(matches!( + table.select_all().runtime(scheduled).execute(), + Err(WorkTableError::RuntimeRequiresAsync) + )); + // The borrowed predicate is evaluated before scheduling. It need not be Send or 'static. + let minimum = std::rc::Rc::new(2u64); + let future = table + .select_all() + .where_by(|row| row.id >= *minimum) + .range_on(ScheduledRowFields::Value, 0u64..50) + .order_on(ScheduledRowFields::Id, Order::Desc) + .offset(1) + .limit(3) + .runtime(Observed) + .execute_async(); + drop(minimum); // The returned future no longer borrows the predicate state. + let selected = future.await.unwrap(); + assert_eq!(selected.iter().map(|r| r.id).collect::>(), vec![7, 6, 5]); + assert_ne!(DISPATCH_THREAD.lock().unwrap().unwrap(), caller); + assert_eq!(table.select_all().execute_async().await.unwrap().len(), 10); + table.delete_by_group(1u64).await.unwrap(); + assert_eq!(table.select_all().execute().unwrap().len(), 5); + assert!(table.select(9u64).is_none()); + }); +} + +#[test] +fn nested_dispatch_progresses_on_one_worker() { + nagoya::block_on(async { + let result = worktable::runtime::run_on::, _>(async { + let table = Arc::new(ScheduledWorkTable::default()); + table + .insert(ScheduledRow { + id: 1, + value: 2, + group: 3, + }) + .await + .unwrap(); + table + .update_value_by_id(ValueByIdQuery { value: 4 }, 1u64) + .await + .unwrap(); + table.select_all().runtime(scheduled).execute_async().await.unwrap()[0].value + }) + .await + .unwrap(); + assert_eq!(result, 4); + }); +} + +#[test] +fn dropping_a_pending_dispatch_cancels_its_owned_future() { + use std::sync::atomic::{AtomicBool, Ordering}; + struct Dropped(Arc); + impl Drop for Dropped { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } + } + let dropped = Arc::new(AtomicBool::new(false)); + let started = Arc::new(AtomicBool::new(false)); + let signal = started.clone(); + let guard = Dropped(dropped.clone()); + let mut task = Box::pin(worktable::runtime::run_on::, _>(async move { + let _guard = guard; + signal.store(true, Ordering::Release); + std::future::pending::<()>().await; + })); + let waker = std::task::Waker::noop(); + let mut cx = std::task::Context::from_waker(waker); + assert!(std::future::Future::poll(task.as_mut(), &mut cx).is_pending()); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !started.load(Ordering::Acquire) { + assert!(std::time::Instant::now() < deadline); + std::thread::yield_now(); + } + drop(task); + while !dropped.load(Ordering::Acquire) { + assert!(std::time::Instant::now() < deadline); + std::thread::yield_now(); + } +} + +#[test] +fn a_panicking_owned_task_does_not_hang_the_caller() { + let result = std::panic::catch_unwind(|| { + nagoya::block_on(worktable::runtime::run_on::, _>(async { + panic!("owned task panic") + })) + }); + assert!(result.is_err()); +} + +worktable! { + name: ScheduledDisk, + persist: true, + runtime: nagoya(shared_slot), + columns: { id: u64 primary_key, value: u64 }, + queries: { update runtime scheduled: { DiskValueById(value) by id } } +} + +#[test] +fn scheduled_mutation_is_persisted_and_reopened() { + nagoya::block_on(async { + let dir = std::path::PathBuf::from(format!("tests/data/runtime-execution-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let config = DiskConfig::new_with_table_name( + dir.to_str().unwrap(), + ScheduledDiskWorkTable::name_snake_case(), + ScheduledDiskWorkTable::version(), + ); + { + let engine = ScheduledDiskPersistenceEngine::new(config.clone()).await.unwrap(); + let table = Arc::new(ScheduledDiskWorkTable::load(engine).await.unwrap()); + table.insert(ScheduledDiskRow { id: 1, value: 2 }).await.unwrap(); + table + .update_disk_value_by_id(DiskValueByIdQuery { value: 99 }, 1u64) + .await + .unwrap(); + assert_eq!( + table.select_all().runtime(scheduled).execute_async().await.unwrap()[0].value, + 99 + ); + table.wait_for_ops().await.unwrap(); + } + let engine = ScheduledDiskPersistenceEngine::new(config).await.unwrap(); + let table = ScheduledDiskWorkTable::load(engine).await.unwrap(); + assert_eq!(table.select(1u64).unwrap().value, 99); + drop(table); + std::fs::remove_dir_all(dir).unwrap(); + }); +} + +#[cfg(feature = "tokio-runtime")] +mod tokio_execution { + use super::*; + runtimes! { on_tokio: tokio, } + worktable! { + name: TokioScheduled, + runtime: tokio, + columns: { id: u64 primary_key, value: u64 }, + queries: { in_place runtime on_tokio: { TokioValueById(value) by id } } + } + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn tokio_profiles_dispatch_on_the_entered_runtime() { + let caller = std::thread::current().id(); + let table = Arc::new(TokioScheduledWorkTable::default()); + table.insert(TokioScheduledRow { id: 1, value: 0 }).await.unwrap(); + table + .update_tokio_value_by_id_in_place( + move |value| { + assert_ne!(std::thread::current().id(), caller); + *value = 8.into(); + }, + 1u64, + ) + .await + .unwrap(); + assert_eq!( + table.select_all().runtime(on_tokio).execute_async().await.unwrap()[0].value, + 8 + ); + } +} + +runtimes! { default_profile: nagoya, } +worktable! { name: DefaultScheduled, columns: { id: u64 primary_key } } +#[test] +fn an_omitted_table_runtime_matches_a_bare_nagoya_profile() { + nagoya::block_on(async { + let table = DefaultScheduledWorkTable::default(); + table.insert(DefaultScheduledRow { id: 1 }).await.unwrap(); + assert_eq!( + table + .select_all() + .runtime(default_profile) + .execute_async() + .await + .unwrap() + .len(), + 1 + ); + }); +} + +#[test] +fn an_owned_select_future_outlives_the_table() { + let future = { + let table = ScheduledWorkTable::default(); + nagoya::block_on(table.insert(ScheduledRow { + id: 1, + value: 2, + group: 3, + })) + .unwrap(); + let future = table.select_all().runtime(scheduled).execute_async(); + drop(table); + future + }; + assert_eq!(nagoya::block_on(future).unwrap()[0].id, 1); +} + +runtimes! { on_spread: nagoya(spread), } +worktable! { + name: Tunable, + columns: { id: u64 primary_key, value: u64 }, + queries: { in_place runtime on_spread: { TunedValue(value) by id } } +} +#[test] +fn callsites_can_tune_nagoya_without_changing_the_table_default() { + nagoya::block_on(async { + let table = Arc::new(TunableWorkTable::default()); + table.insert(TunableRow { id: 1, value: 2 }).await.unwrap(); + let caller = std::thread::current().id(); + table + .update_tuned_value_in_place( + move |value| { + assert_ne!(std::thread::current().id(), caller); + *value = 3.into(); + }, + 1u64, + ) + .await + .unwrap(); + assert_eq!( + table.select_all().runtime(on_spread).execute_async().await.unwrap()[0].value, + 3 + ); + }); +} diff --git a/tests/runtimes.rs b/tests/runtimes.rs new file mode 100644 index 00000000..f9e5b87d --- /dev/null +++ b/tests/runtimes.rs @@ -0,0 +1,106 @@ +//! `runtimes!` and the call-site `.runtime()` builder link. +//! +//! The cases that must **fail** to compile are drafted in `tests/ui-drafts/` +//! and belong to the trybuild lane; this file covers only what compiles, since +//! a test that a bound holds is a test that this file builds at all. + +use worktable::prelude::*; +use worktable::runtimes; + +runtimes! { + fast_local: nagoya(locality), + wide: nagoya(spread), + batch: nagoya(throughput), + bare: nagoya, +} + +// A tokio profile only resolves when the backend is in the graph. `TokioRt` +// lives behind `tokio-runtime`, which is off by default so that taking +// WorkTable off tokio stays true for anyone who does not ask for it back. +#[cfg(feature = "tokio-runtime")] +runtimes! { + tokio_max: tokio, +} + +/// Check the concrete type emitted for a profile. Callsite compatibility +/// separately admits different Nagoya flavors from the same backend family. +fn assert_backend() +where + P: Profile, + B: Runtime, +{ +} + +#[test] +fn each_profile_resolves_to_its_backend() { + #[cfg(feature = "tokio-runtime")] + assert_backend::(); + assert_backend::>(); + assert_backend::>(); + assert_backend::>(); +} + +#[test] +fn a_bare_backend_is_its_default_flavor() { + assert_backend::>(); + assert_eq!(::tuning(), ::tuning()); +} + +#[test] +fn each_profile_resolves_to_its_tuning() { + assert_eq!(::tuning(), Tuning::locality()); + assert_eq!(::tuning(), Tuning::spread()); + assert_eq!(::tuning(), Tuning::throughput()); + #[cfg(feature = "tokio-runtime")] + assert_eq!(::tuning(), Tuning::default()); +} + +#[test] +fn a_profile_is_a_value_as_well_as_a_type() { + // What lets `.runtime(wide)` and `runtime wide:` spell the profile the same + // way. A unit struct occupies both namespaces, so there is no case + // convention between the schema and the call site. + let _ = wide; + assert_eq!(wide, ::default()); +} + +/// Stands in for a generated row type. The codegen lane emits both of these for +/// every table; a table without the first has no `.runtime()` at all, and one +/// without the second has it pinned by the schema. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Trade { + id: u64, +} + +impl TableRuntime for Trade { + type Backend = NagoyaRt; +} + +impl RuntimeUnpinned for Trade {} + +fn trades() -> SelectQueryBuilder, (), ()> { + SelectQueryBuilder::new(vec![Trade { id: 1 }, Trade { id: 2 }].into_iter()) +} + +#[test] +fn a_matching_profile_compiles_and_records_its_tuning() { + let builder = trades().limit(2).runtime(wide); + assert_eq!(builder.params.tuning, Some(Tuning::spread())); + assert_eq!(builder.params.limit, Some(2)); +} + +#[test] +fn no_runtime_call_records_no_tuning() { + assert_eq!(trades().limit(2).params.tuning, None); +} + +#[test] +fn runtime_chains_rather_than_widens() { + // One argument, and the link sits among the others rather than replacing + // any of them. A knob added later becomes a further link, never a second + // argument here. + let builder = trades().offset(1).runtime(wide).limit(1); + assert_eq!(builder.params.tuning, Some(Tuning::spread())); + assert_eq!(builder.params.offset, Some(1)); + assert_eq!(builder.params.limit, Some(1)); +} diff --git a/tests/slotted_page_requirement.rs b/tests/slotted_page_requirement.rs new file mode 100644 index 00000000..4b039bff --- /dev/null +++ b/tests/slotted_page_requirement.rs @@ -0,0 +1,196 @@ +//! V3 data pages locate their live rows independently of indexes. +//! +//! The release deliberately cuts over from v2. Most deployments recreate their +//! stores; this runtime refuses old bytes rather than carrying a v2 reader. +//! See docs/on-disk-v3-cutover.md for the release contract. + +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: SlottedRow, + version: 1, + persist: true, + columns: { + id: u64 primary_key autoincrement, + blob: String, + }, +); + +/// Enough rows to fill more than one page, so a directory would be doing real +/// work rather than describing a single row. +const ROWS: u64 = 4_000; + +async fn filled(dir: &str) -> SlottedRowWorkTable { + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir).expect("a directory"); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("an engine"); + let table = SlottedRowWorkTable::load(engine).await.expect("a table"); + + for n in 0..ROWS { + table + .insert(SlottedRowRow { + id: table.get_next_pk().into(), + blob: format!("row {n}, long enough to make the page boundaries interesting"), + }) + .await + .expect("a row"); + } + table.wait_for_ops().await.expect("the queue drains"); + table +} + +fn data_file(dir: &str) -> std::path::PathBuf { + std::path::Path::new(dir) + .join(SlottedRowWorkTable::name_snake_case()) + .join(".wt.data") +} + +fn scan_rows(dir: &str) -> std::collections::BTreeMap { + let bytes = std::fs::read(data_file(dir)).unwrap(); + assert_eq!(bytes.len() % PAGE_SIZE, 0); + let mut rows = std::collections::BTreeMap::new(); + for (id, page) in bytes.as_chunks::().0.iter().enumerate().skip(1) { + assert_eq!(u32::from_le_bytes(page[..4].try_into().unwrap()), 3); + assert_eq!(u32::from_le_bytes(page[8..12].try_into().unwrap()), id as u32); + let length = u32::from_le_bytes(page[24..28].try_into().unwrap()); + let decoded = data_bucket::DataPage::::decode(&page[GENERAL_HEADER_SIZE..], length).unwrap(); + for slot in &decoded.rows { + // An aligned copy lets rkyv validate each independently located + // archive without depending on its file offset's alignment. + let mut archive = rkyv::util::AlignedVec::<16>::new(); + archive.extend_from_slice(&decoded.data[slot.offset as usize..][..slot.length as usize]); + let wrapped = + rkyv::from_bytes::<::WrappedRow, rkyv::rancor::Error>(&archive).unwrap(); + let row = wrapped.get_inner(); + assert!(rows.insert(row.id, row.blob).is_none(), "duplicate live primary key"); + } + } + rows +} + +#[tokio::test] +async fn directory_survives_deletes_moves_reuse_and_reopen_without_index_access() { + let dir = "tests/data/slotted_page/churn"; + let table = filled(dir).await; + let mut expected = std::collections::BTreeMap::new(); + for id in 0..ROWS { + if id % 3 == 0 { + table.delete(id).await.unwrap(); + } else { + let blob = "grown".repeat(10 + (id % 71) as usize); + table.upsert(SlottedRowRow { id, blob: blob.clone() }).await.unwrap(); + expected.insert(id, blob); + } + } + table.close().await.unwrap(); + assert_eq!(scan_rows(dir), expected); + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .unwrap(); + let table = SlottedRowWorkTable::load(engine).await.unwrap(); + for (id, blob) in &expected { + assert_eq!(table.select(*id).unwrap().blob, *blob); + } + table.close().await.unwrap(); + assert_eq!(scan_rows(dir), expected); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[tokio::test] +async fn actual_v2_store_is_refused_without_modifying_it() { + let dir = "tests/data/slotted_page/v2_refused"; + let path = data_file(dir); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let original = include_bytes!("fixtures/page-format/v2.wt.data"); + std::fs::write(&path, original).unwrap(); + let result = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await; + let error = match result { + Ok(_) => panic!("v2 unexpectedly opened"), + Err(error) => error, + }; + assert!(format!("{error:#}").contains("page format v2"), "{error:#}"); + assert_eq!(std::fs::read(&path).unwrap(), original); + std::fs::remove_dir_all(dir).unwrap(); +} + +/// A data page should say where its rows are, without an index. +/// +/// Read the bytes directly so an index cannot hide a missing directory. +#[tokio::test] +async fn a_data_page_says_where_its_rows_are() { + let dir = "tests/data/slotted_page/self_describing"; + let table = filled(dir).await; + table.close().await.expect("the table closes"); + + let bytes = std::fs::read(data_file(dir)).expect("the file"); + assert!( + bytes.len() > PAGE_SIZE, + "the fixture has to span pages: {} bytes", + bytes.len() + ); + + // The count is the final u32; the preceding word is the CRC. + let mut described = 0usize; + let (pages, _) = bytes.as_chunks::(); + for page in pages.iter().skip(1) { + let mut tail = [0u8; 4]; + tail.copy_from_slice(&page[PAGE_SIZE - 4..]); + described += u32::from_le_bytes(tail) as usize; + } + + assert_eq!( + described, ROWS as usize, + "no page says how many rows it holds, so the {ROWS} rows in this file \ + cannot be found without the index. A row directory in the data page is \ + what makes a page readable on its own, and what makes the next format \ + change an upgrade instead of a regeneration." + ); + assert_eq!(scan_rows(dir).len(), ROWS as usize); + let _ = std::fs::remove_dir_all(dir); +} + +/// A store reopens without being deleted first. +/// +/// The new writer and reader must agree after a clean close. +#[tokio::test] +async fn a_store_reopens_without_being_rebuilt() { + let dir = "tests/data/slotted_page/reopen"; + let table = filled(dir).await; + table.close().await.expect("the table closes"); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("an engine"); + let reopened = SlottedRowWorkTable::load(engine) + .await + .expect("a store reopens rather than needing to be rebuilt"); + + assert_eq!( + reopened.select_all().execute().expect("a read").len(), + ROWS as usize, + "no rows are lost reopening a store" + ); + reopened.close().await.expect("the table closes"); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/tests/storage_domain_catalog.rs b/tests/storage_domain_catalog.rs new file mode 100644 index 00000000..76b8c6f1 --- /dev/null +++ b/tests/storage_domain_catalog.rs @@ -0,0 +1,215 @@ +use std::collections::BTreeMap; +use std::fmt::{Display, Formatter}; +use std::sync::{Arc, Mutex}; + +use worktable::Database; +use worktable::data_bucket::storage::{ + CatalogMutation, CatalogName, CatalogRecord, CommittedGeneration, GenerationPlan, Head, MutationKind, ObjectRef, + PageAddress, PageKind, PageRef, PageStore, StagedGeneration, StagedPage, StorageDomainId, SystemTableRecord, + TableId, +}; +use worktable::data_bucket::{PAGE_SIZE, PageId, SpaceId}; + +fn data_page_image(address: PageAddress, rows: u32, live_bytes: u32, fill: u8) -> Vec { + use worktable::data_bucket::{DataPage, GeneralHeader, INNER_PAGE_SIZE, Link, PageType}; + + let mut page = DataPage::::new(); + let base = live_bytes / rows; + let mut offset = 0; + for index in 0..rows { + let length = if index + 1 == rows { live_bytes - offset } else { base }; + page.update_at( + Link { + page_id: address.page_id, + offset, + length, + }, + &vec![fill; length as usize], + ) + .unwrap(); + offset += length; + } + let mut header = GeneralHeader::new(address.page_id, PageType::Data, address.space_id); + header.data_length = page.length; + let mut image = worktable::prelude::rkyv::to_bytes::(&header) + .unwrap() + .to_vec(); + image.extend_from_slice(&page.encode(INNER_PAGE_SIZE).unwrap()); + assert_eq!(image.len(), PAGE_SIZE); + image +} + +#[derive(Clone, Debug)] +struct MemoryError; + +impl Display for MemoryError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str("in-memory page store rejected the generation") + } +} + +impl std::error::Error for MemoryError {} + +#[derive(Default)] +struct State { + next_object: u64, + head: Option, + catalogs: BTreeMap<[u8; 32], Vec>, + pages: BTreeMap<[u8; 32], Vec>, +} + +#[derive(Clone, Default)] +struct MemoryStore(Arc>); + +impl MemoryStore { + fn store_object(state: &mut State, bytes: Vec) -> ObjectRef { + state.next_object += 1; + let mut object = [0; 32]; + object[..8].copy_from_slice(&state.next_object.to_le_bytes()); + let length = bytes.len() as u32; + state.pages.insert(object, bytes); + ObjectRef { + object, + offset: 0, + encoded_length: length, + decoded_length: length, + checksum: object, + } + } +} + +impl PageStore for MemoryStore { + type Error = MemoryError; + + fn load_head(&self, _domain: StorageDomainId) -> Result, Self::Error> { + Ok(self.0.lock().unwrap().head.clone()) + } + + fn load_catalog(&self, head: &Head) -> Result, Self::Error> { + self.0 + .lock() + .unwrap() + .catalogs + .get(&head.catalog.object) + .cloned() + .ok_or(MemoryError) + } + + fn read_page(&self, page: &PageRef) -> Result, Self::Error> { + self.0 + .lock() + .unwrap() + .pages + .get(&page.object.object) + .cloned() + .ok_or(MemoryError) + } + + fn stage(&self, plan: &GenerationPlan) -> Result { + let mut state = self.0.lock().unwrap(); + let mut pages = Vec::new(); + for mutation in &plan.pages { + if let MutationKind::Put { + image, + live_rows, + live_bytes, + } = &mutation.kind + { + pages.push(StagedPage { + address: mutation.address, + object: Self::store_object(&mut state, image.clone()), + live_rows: *live_rows, + live_bytes: *live_bytes, + }); + } + } + Ok(StagedGeneration { + domain: plan.domain, + generation: plan.id, + parent: plan.parent, + writer_epoch: plan.writer_epoch, + pages, + catalog: None, + }) + } + + fn stage_catalog(&self, staged: &mut StagedGeneration, checkpoint: &[u8]) -> Result<(), Self::Error> { + let mut state = self.0.lock().unwrap(); + let object = Self::store_object(&mut state, checkpoint.to_vec()); + state.catalogs.insert(object.object, checkpoint.to_vec()); + staged.catalog = Some(object); + Ok(()) + } + + fn commit(&self, staged: StagedGeneration) -> Result { + let mut state = self.0.lock().unwrap(); + if state.head.as_ref().map_or(0, |head| head.generation) != staged.parent { + return Err(MemoryError); + } + let head = Head { + domain: staged.domain, + generation: staged.generation, + parent: staged.parent, + writer_epoch: staged.writer_epoch, + catalog: staged.catalog.ok_or(MemoryError)?, + }; + state.head = Some(head.clone()); + Ok(CommittedGeneration { head }) + } +} + +#[test] +fn generated_catalog_commits_pages_and_restores_the_database() { + let id = StorageDomainId([7; 16]); + let store = MemoryStore::default(); + let database = Database::new(id, 11, store.clone()); + let table_id = database.register_table("orders", 3).unwrap(); + assert_eq!(table_id, TableId(1)); + + let address = PageAddress { + domain: id, + table_id, + space_id: SpaceId(2), + page_id: PageId::from(4), + page_kind: PageKind::Data, + }; + let image = data_page_image(address, 9, 777, 0x5a); + let mut generation = database.begin_generation().unwrap(); + generation.put_page(address, image.clone(), 9, 777); + database.commit_generation(generation.finish()).unwrap(); + + let catalog = database.catalog(); + let tables = catalog.system_tables(); + assert_eq!(tables.len(), 1); + assert_eq!(tables[0].name.as_str(), "orders"); + assert_eq!(tables[0].row_count, 9); + assert_eq!(tables[0].live_row_bytes, 777); + assert_eq!(tables[0].live_data_pages, 1); + assert_eq!(database.read_page(address).unwrap(), Some(image.clone())); + + drop(database); + let reopened = Database::open(id, 12, store).unwrap(); + assert_eq!(reopened.catalog().system_pages().len(), 1); + assert_eq!(reopened.read_page(address).unwrap(), Some(image)); + + let table = SystemTableRecord { + table_id, + name: CatalogName::new("orders").unwrap(), + schema_version: 4, + data_space_id: SpaceId(2), + page_stride: PAGE_SIZE as u32, + row_count: 9, + live_row_bytes: 777, + allocated_data_pages: 1, + live_data_pages: 1, + primary_index_entries: 0, + secondary_index_entries: 0, + tombstones: 0, + applied_generation: 2, + durable_generation: 2, + }; + let mut generation = reopened.begin_generation().unwrap(); + generation.update_catalog(CatalogMutation::Upsert(CatalogRecord::Table(table))); + reopened.commit_generation(generation.finish()).unwrap(); + assert_eq!(reopened.catalog().system_tables()[0].schema_version, 4); +} diff --git a/tests/ui-drafts/README.md b/tests/ui-drafts/README.md new file mode 100644 index 00000000..15eb6de4 --- /dev/null +++ b/tests/ui-drafts/README.md @@ -0,0 +1,27 @@ +# Compile-fail drafts for `runtimes!` and `.runtime()` + +Case bodies only. Wiring them into `trybuild` (and capturing the `.stderr` +files) belongs to the trybuild lane; nothing here is compiled by `cargo test` +today, because `tests/ui-drafts` is a directory rather than a test target. + +Each file names, in a header comment, the message content that must appear. The +messages were produced by rustc 1.97.1 against this branch, so the `.stderr` +files can be generated with `TRYBUILD=overwrite` and then read rather than +guessed at. + +| file | what it proves | +|---|---| +| `backend-mismatch.rs` | a `tokio` profile on a `nagoya` table, named at a call site | +| `pinned-by-schema.rs` | a section annotation and a `.runtime()` both present | +| `no-runtime-on-point-select.rs` | `select(pk)` has no `.runtime()` at all | +| `not-implemented-backend.rs` | `forte`, `blocking` and `bwos` are rejected, not accepted inert | +| `unknown-backend.rs` | an unknown backend names what does exist | +| `unknown-flavor.rs` | an unknown flavor lists the three | +| `tokio-has-no-flavor.rs` | `tokio(spread)` | +| `duplicate-profile.rs` | two profiles with one name | +| `runtime-takes-one-argument.rs` | `.runtime()` does not widen | + +Eight of the nine were checked against this branch and produce the message +their header claims. `no-runtime-on-point-select.rs` is the exception: it uses +`worktable!`, so it needs the generated table's `TableRuntime` impl, which is +the codegen lane's. Run it once that lands. diff --git a/tests/ui-drafts/backend-mismatch.rs b/tests/ui-drafts/backend-mismatch.rs new file mode 100644 index 00000000..d48422c1 --- /dev/null +++ b/tests/ui-drafts/backend-mismatch.rs @@ -0,0 +1,31 @@ +// Must fail with E0271, naming both backends: +// +// type mismatch resolving `::Backend == NagoyaRt` +// expected struct `worktable::runtime::NagoyaRt` +// found struct `TokioRt` +// +// This one is unconditional and can never be waived: the table's `runtime:` +// selects the RwLock, Notify and JoinHandle it is built from, so nothing at a +// call site can change it. + +use worktable::prelude::*; +use worktable::runtimes; + +runtimes! { + tokio_max: tokio, +} + +struct Trade { + id: u64, +} + +impl TableRuntime for Trade { + type Backend = NagoyaRt; +} + +impl RuntimeUnpinned for Trade {} + +fn main() { + let rows = vec![Trade { id: 1 }]; + let _ = SelectQueryBuilder::::new(rows.into_iter()).runtime(tokio_max); +} diff --git a/tests/ui-drafts/duplicate-profile.rs b/tests/ui-drafts/duplicate-profile.rs new file mode 100644 index 00000000..a84f2a2a --- /dev/null +++ b/tests/ui-drafts/duplicate-profile.rs @@ -0,0 +1,16 @@ +// Must fail at macro expansion with two spans, the second declaration first: +// +// duplicate runtime profile `wide` +// `wide` was already declared here +// +// A profile name is the whole of the call-site surface, so two of them is an +// ambiguity rather than a last-one-wins. + +use worktable::runtimes; + +runtimes! { + wide: nagoya(spread), + wide: tokio, +} + +fn main() {} diff --git a/tests/ui-drafts/no-runtime-on-point-select.rs b/tests/ui-drafts/no-runtime-on-point-select.rs new file mode 100644 index 00000000..3b057e26 --- /dev/null +++ b/tests/ui-drafts/no-runtime-on-point-select.rs @@ -0,0 +1,40 @@ +// Must fail with "no method named `runtime` found for enum `Option`", on the +// `select(pk)` line only. The `select_all()` line above it must compile. +// +// `.runtime()` is on the builder-returning selects, `select_all` and +// `select_by_pk_range`. `select(pk)` returns `Option` rather than a +// builder, so it has no `.runtime()`, and this is the one place where a missing +// method is the right message: there is no builder to pin. +// +// The reason is measured. A spawn is 21 ns and the wake that follows it about +// 2,250 ns at the median, against roughly 400 ns for a point read, so the hop +// costs several times the operation. `.runtime()` is for work already measured +// in microseconds. +// +// Needs a real generated table and a `TableRuntime` impl for its row type, so +// this case waits on the codegen lane; the other drafts hand-build the builder. + +use worktable::prelude::*; +use worktable::{runtimes, worktable}; + +runtimes! { + wide: nagoya(spread), +} + +worktable! ( + name: Trade, + columns: { + id: u64 primary_key, + qty: u64, + } +); + +fn main() { + let table = TradeWorkTable::default(); + + // Fine: a builder. + let _ = table.select_all().limit(10).runtime(wide); + + // Not fine: a row. + let _ = table.select(1u64).runtime(wide); +} diff --git a/tests/ui-drafts/not-implemented-backend.rs b/tests/ui-drafts/not-implemented-backend.rs new file mode 100644 index 00000000..e6a9d632 --- /dev/null +++ b/tests/ui-drafts/not-implemented-backend.rs @@ -0,0 +1,21 @@ +// Must fail at macro expansion with a message that names the +// backend, says it is not implemented and lists what is. The parser stops at +// the first bad entry, so only `forte` is reported; `blocking` and `bwos` need +// their own cases if each message is to be asserted. +// +// runtime backend `forte` is not implemented; the backends that are: nagoya, tokio +// +// Rejected rather than accepted inert: a declaration that reads as if it +// selected something either did or failed to build. `forte`, `blocking` and +// `bwos` are a string list in the parser, not enum variants, and exist only so +// this message can be written. + +use worktable::runtimes; + +runtimes! { + a: forte, + b: blocking, + c: bwos, +} + +fn main() {} diff --git a/tests/ui-drafts/pinned-by-schema.rs b/tests/ui-drafts/pinned-by-schema.rs new file mode 100644 index 00000000..d6ffa311 --- /dev/null +++ b/tests/ui-drafts/pinned-by-schema.rs @@ -0,0 +1,35 @@ +// Must fail with E0277 and the `#[diagnostic::on_unimplemented]` text: +// +// error[E0277]: `Ledger` already has a runtime pinned by the schema +// | +// | let _ = builder.runtime(wide); +// | ^^^^^^^ remove this `.runtime()`, or remove `runtime` from the section +// +// It must NOT be "no method named `runtime` found for struct +// `SelectQueryBuilder`", which is what omitting the method would give and which +// points at the builder rather than at the two declarations that disagree. +// +// `Ledger` stands in for a table whose `select` section is annotated +// `runtime wide:`: generated code emits its `TableRuntime` impl and withholds +// the `RuntimeUnpinned` one. + +use worktable::prelude::*; +use worktable::runtimes; + +runtimes! { + wide: nagoya(spread), +} + +struct Ledger { + id: u64, +} + +impl TableRuntime for Ledger { + type Backend = NagoyaRt; +} + +fn main() { + let rows = vec![Ledger { id: 1 }]; + let builder = SelectQueryBuilder::::new(rows.into_iter()); + let _ = builder.runtime(wide); +} diff --git a/tests/ui-drafts/runtime-takes-one-argument.rs b/tests/ui-drafts/runtime-takes-one-argument.rs new file mode 100644 index 00000000..8d4fefce --- /dev/null +++ b/tests/ui-drafts/runtime-takes-one-argument.rs @@ -0,0 +1,32 @@ +// Must fail with an arity error on `.runtime()`: +// +// this method takes 1 argument but 2 arguments were supplied +// +// `.runtime()` takes a profile and nothing else. Every distinct +// parameterisation is a distinct thread pool, so free-form numbers here would +// mean a pool set nobody can enumerate; with names only, every pool the process +// will ever create is visible by reading one `runtimes!` block. A knob added +// later arrives as a further builder link, `.runtime(wide).workers(12)`, never +// as a second argument, because an arity change breaks every existing call. + +use worktable::prelude::*; +use worktable::runtimes; + +runtimes! { + wide: nagoya(spread), +} + +struct Trade { + id: u64, +} + +impl TableRuntime for Trade { + type Backend = NagoyaRt; +} + +impl RuntimeUnpinned for Trade {} + +fn main() { + let rows = vec![Trade { id: 1 }]; + let _ = SelectQueryBuilder::::new(rows.into_iter()).runtime(wide, 12); +} diff --git a/tests/ui-drafts/tokio-has-no-flavor.rs b/tests/ui-drafts/tokio-has-no-flavor.rs new file mode 100644 index 00000000..7c4063d5 --- /dev/null +++ b/tests/ui-drafts/tokio-has-no-flavor.rs @@ -0,0 +1,14 @@ +// Must fail at macro expansion with: +// +// tokio has no flavors; write `tokio`. Flavors belong to nagoya, whose pool they tune +// +// The span must be on `spread`, not on `tokio`: the backend is fine and the +// flavor is the part to delete. + +use worktable::runtimes; + +runtimes! { + p: tokio(spread), +} + +fn main() {} diff --git a/tests/ui-drafts/unknown-backend.rs b/tests/ui-drafts/unknown-backend.rs new file mode 100644 index 00000000..fb4bef8f --- /dev/null +++ b/tests/ui-drafts/unknown-backend.rs @@ -0,0 +1,11 @@ +// Must fail at macro expansion with: +// +// unknown runtime backend `smol`; expected one of: nagoya, tokio + +use worktable::runtimes; + +runtimes! { + p: smol, +} + +fn main() {} diff --git a/tests/ui-drafts/unknown-flavor.rs b/tests/ui-drafts/unknown-flavor.rs new file mode 100644 index 00000000..efcbe5d4 --- /dev/null +++ b/tests/ui-drafts/unknown-flavor.rs @@ -0,0 +1,11 @@ +// Must fail at macro expansion, listing the three flavors: +// +// unknown nagoya flavor `banana`; expected one of: locality, spread, throughput + +use worktable::runtimes; + +runtimes! { + p: nagoya(banana), +} + +fn main() {} diff --git a/tests/ui.rs b/tests/ui.rs new file mode 100644 index 00000000..240194fa --- /dev/null +++ b/tests/ui.rs @@ -0,0 +1,38 @@ +//! Compile-fail tests for `worktable!`. +//! +//! Roughly half of what the macro promises is a refusal: a declaration that is +//! grammatical but wrong has to be rejected, with a message that says which +//! rule it broke. Nothing in `tests/` could express that, because a test only +//! runs once its crate has compiled, so every one of those rules was +//! unverified. `trybuild` compiles each case in `tests/ui/` on its own and +//! diffs the compiler's output against a committed `.stderr`. +//! +//! The assertion is the message, not the failure. A case that only checked +//! "this did not compile" would stay green while the diagnostic decayed into +//! something that points at the wrong line, which is the failure mode these +//! rules exist to prevent. +//! +//! Cases are listed one by one rather than globbed. `tests/ui/runtime_*.rs` +//! are drafts for the runtime-backend work and describe a feature the parser +//! does not have yet, so a glob would fail them for the wrong reason. The lane +//! that lands `runtime:` adds them here; see `tests/ui/README.md`. + +#[test] +fn compile_fail() { + let t = trybuild::TestCases::new(); + + // Grammar and shape. + t.compile_fail("tests/ui/no_primary_key.rs"); + t.compile_fail("tests/ui/unknown_index_backend.rs"); + t.compile_fail("tests/ui/query_over_unknown_column.rs"); + + // Index backend rules. + t.compile_fail("tests/ui/indexset_unsized_key.rs"); + t.compile_fail("tests/ui/nonunique_congee_index.rs"); + t.compile_fail("tests/ui/congee_unsupported_key_type.rs"); + t.compile_fail("tests/ui/congee_without_persist.rs"); + + // Query rules. + t.compile_fail("tests/ui/autoincrement_unsupported_key.rs"); + t.compile_fail("tests/ui/in_place_over_indexed_column.rs"); +} diff --git a/tests/ui/README.md b/tests/ui/README.md new file mode 100644 index 00000000..dcfa09f9 --- /dev/null +++ b/tests/ui/README.md @@ -0,0 +1,84 @@ +# Compile-fail tests + +Every file here is a program that must **not** compile, paired with a +`.stderr` holding the diagnostic it must produce. `tests/ui.rs` runs them +through `trybuild`. + +Roughly half of what `worktable!` promises is a refusal. A test in `tests/` +cannot express one, because a test only runs once its crate has compiled, so +every rule of that kind was unverified until this directory existed. + +The assertion is the **message**, not the failure. A case that only checked +"this did not compile" stays green while its diagnostic decays into one that +points at the wrong line, which is the failure these rules exist to prevent. + +```sh +cargo test --test ui +``` + +## Adding a case + +1. Write `tests/ui/.rs`. Keep it minimal: one `worktable!` invocation + breaking one rule, plus `fn main() {}`. Open with a comment saying which + rule it pins and why that rule exists, not what the code does. +2. Import only `use worktable::worktable;`. Do **not** add + `use worktable::prelude::*;` unless the case actually needs it: the macro + errors before the import is used, so the prelude lands an + `unused_imports` warning in the `.stderr` and that warning's wording moves + between compiler releases. +3. Add a `t.compile_fail("tests/ui/.rs");` line to `tests/ui.rs`. Cases + are listed one by one on purpose, not globbed. See "Drafts" below. +4. Generate the expectation, read it, commit it: + + ```sh + TRYBUILD=overwrite cargo test --test ui + ``` + +Read the generated `.stderr` before committing it. `TRYBUILD=overwrite` +records whatever the compiler said, including a message that is wrong, so +blessing without reading turns the harness into a transcript of current +behaviour rather than a check on it. + +## Regenerating expectations + +```sh +TRYBUILD=overwrite cargo test --test ui +``` + +That rewrites every `.stderr` in place. Diff them afterwards. A change you did +not intend is the finding. + +## The `.stderr` files are compiler-version sensitive + +They are the compiler's output verbatim: message text, line and column +numbers, the underline, the trailing notes. Anything rustc changes about how +it renders a diagnostic changes these files, on code nobody touched. + +The cases here are all `syn::Error` text emitted by `worktable!` through +`compile_error!`, which is the least fragile shape available: the message is +ours, and rustc contributes only the span rendering. Cases that lean on +rustc's own diagnostics, such as the trait-bound and +`#[diagnostic::on_unimplemented]` cases the runtime lane will add, are more +exposed. + +Generated with **rustc 1.97.1 (8bab26f4f 2026-07-14)**. If CI runs a newer +stable than your toolchain, expect the first mismatch to come from CI, not +from your terminal. `scripts/ci-local.sh` prints both versions. + +## Drafts + +`runtime_*.rs` are written but **not** listed in `tests/ui.rs`. They pin the +rules in section 7 of the runtime-backend contract, and the parser has no +`runtime:` arm yet, so today they fail with + +``` +Unexpected token `runtime`; expected one of `columns`, `indexes`, `queries`, `config` +``` + +which is the right verdict for the wrong reason. Wiring them up now would +commit a `.stderr` asserting that the feature is missing, and that file would +pass right up until the feature landed and then have to be rewritten. + +Each draft carries a comment saying which lane enables it. That lane adds its +`t.compile_fail(...)` line and blesses its `.stderr` in the same commit that +lands the rule. diff --git a/tests/ui/autoincrement_unsupported_key.rs b/tests/ui/autoincrement_unsupported_key.rs new file mode 100644 index 00000000..06b3995b --- /dev/null +++ b/tests/ui/autoincrement_unsupported_key.rs @@ -0,0 +1,15 @@ +// Rule: `autoincrement` maps the key type to an atomic counter. `usize` reads +// like one of the accepted set and is not in the mapping, so it is the case +// worth pinning. +use worktable::worktable; + +worktable! { + name: AutoincrementUsize, + persist: false, + columns: { + id: usize primary_key autoincrement, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/autoincrement_unsupported_key.stderr b/tests/ui/autoincrement_unsupported_key.stderr new file mode 100644 index 00000000..7c35dfba --- /dev/null +++ b/tests/ui/autoincrement_unsupported_key.stderr @@ -0,0 +1,5 @@ +error: primary key `id` is `autoincrement` over key type `usize`, which cannot be generated; supported types: u8, u16, u32, u64, i8, i16, i32, i64 + --> tests/ui/autoincrement_unsupported_key.rs:10:9 + | +10 | id: usize primary_key autoincrement, + | ^^ diff --git a/tests/ui/congee_unsupported_key_type.rs b/tests/ui/congee_unsupported_key_type.rs new file mode 100644 index 00000000..588a0552 --- /dev/null +++ b/tests/ui/congee_unsupported_key_type.rs @@ -0,0 +1,15 @@ +// Rule: congee keys are the unsigned integers its public API accepts. A +// `String` primary key is refused here rather than at the point where the +// generated codec would fail to build. +use worktable::worktable; + +worktable! { + name: CongeeStringKey, + persist: false, + columns: { + id: String primary_key using congee, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/congee_unsupported_key_type.stderr b/tests/ui/congee_unsupported_key_type.stderr new file mode 100644 index 00000000..6c9ad5d3 --- /dev/null +++ b/tests/ui/congee_unsupported_key_type.stderr @@ -0,0 +1,5 @@ +error: `using congee` requires a directly named primitive primary-key type; found `String`; supported types: u8, u16, u32, u64, usize (type aliases cannot be resolved by the macro) + --> tests/ui/congee_unsupported_key_type.rs:10:9 + | +10 | id: String primary_key using congee, + | ^^ diff --git a/tests/ui/congee_without_persist.rs b/tests/ui/congee_without_persist.rs new file mode 100644 index 00000000..5c64dfb5 --- /dev/null +++ b/tests/ui/congee_without_persist.rs @@ -0,0 +1,14 @@ +// Rule: the backends that persist differently from the default require the +// author to say which they meant. Omitting `persist` leaves the choice to the +// macro, and for these backends that choice is not one it should make. +use worktable::worktable; + +worktable! { + name: CongeeNoPersist, + columns: { + id: u64 primary_key autoincrement using congee, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/congee_without_persist.stderr b/tests/ui/congee_without_persist.stderr new file mode 100644 index 00000000..dfab648e --- /dev/null +++ b/tests/ui/congee_without_persist.stderr @@ -0,0 +1,5 @@ +error: primary index `id` uses `congee`, which requires an explicit `persist: true` or `persist: false` + --> tests/ui/congee_without_persist.rs:9:9 + | +9 | id: u64 primary_key autoincrement using congee, + | ^^ diff --git a/tests/ui/in_place_over_indexed_column.rs b/tests/ui/in_place_over_indexed_column.rs new file mode 100644 index 00000000..b982ac51 --- /dev/null +++ b/tests/ui/in_place_over_indexed_column.rs @@ -0,0 +1,23 @@ +// Rule: an `in_place` query writes the archived column bytes and maintains no +// index, so a column any index is built over cannot be mutated on that path. +// The index would keep resolving the old value. +use worktable::worktable; + +worktable! { + name: InPlaceIndexed, + persist: false, + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, + indexes: { + value_idx: value unique, + }, + queries: { + in_place: { + ValueById(value) by id, + } + }, +} + +fn main() {} diff --git a/tests/ui/in_place_over_indexed_column.stderr b/tests/ui/in_place_over_indexed_column.stderr new file mode 100644 index 00000000..9b615ad9 --- /dev/null +++ b/tests/ui/in_place_over_indexed_column.stderr @@ -0,0 +1,5 @@ +error: in_place query `ValueById` mutates column `value`, which is covered by an index; indexed columns cannot be updated in place because secondary indexes are not maintained on this path. Use an `update` query instead + --> tests/ui/in_place_over_indexed_column.rs:18:23 + | +18 | ValueById(value) by id, + | ^^^^^ diff --git a/tests/ui/indexset_unsized_key.rs b/tests/ui/indexset_unsized_key.rs new file mode 100644 index 00000000..10e89293 --- /dev/null +++ b/tests/ui/indexset_unsized_key.rs @@ -0,0 +1,19 @@ +// Rule: `using indexset` cannot hold a variable-sized key. The upstream crate +// has no node type for one, so the declaration is refused with the backend +// that can, rather than being accepted and failing deep inside the emitted +// generic types. +use worktable::worktable; + +worktable! { + name: IndexsetUnsized, + persist: false, + columns: { + id: u64 primary_key autoincrement, + name: String, + }, + indexes: { + name_idx: name unique using indexset, + }, +} + +fn main() {} diff --git a/tests/ui/indexset_unsized_key.stderr b/tests/ui/indexset_unsized_key.stderr new file mode 100644 index 00000000..941cef43 --- /dev/null +++ b/tests/ui/indexset_unsized_key.stderr @@ -0,0 +1,5 @@ +error: `using indexset` does not yet support variable-sized keys; use `worktables_index` for this index + --> tests/ui/indexset_unsized_key.rs:12:15 + | +12 | name: String, + | ^^^^^^ diff --git a/tests/ui/no_primary_key.rs b/tests/ui/no_primary_key.rs new file mode 100644 index 00000000..a304e7e0 --- /dev/null +++ b/tests/ui/no_primary_key.rs @@ -0,0 +1,15 @@ +// Rule: every table needs a primary key. Without one there is nothing to +// resolve a row by, so the parser refuses the `columns` block outright rather +// than generating a table that can only be scanned. +use worktable::worktable; + +worktable! { + name: NoPrimaryKey, + persist: false, + columns: { + id: u64, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/no_primary_key.stderr b/tests/ui/no_primary_key.stderr new file mode 100644 index 00000000..30697f50 --- /dev/null +++ b/tests/ui/no_primary_key.stderr @@ -0,0 +1,5 @@ +error: Primary key must be set + --> tests/ui/no_primary_key.rs:7:5 + | +7 | name: NoPrimaryKey, + | ^^^^ diff --git a/tests/ui/nonunique_congee_index.rs b/tests/ui/nonunique_congee_index.rs new file mode 100644 index 00000000..42c33dc2 --- /dev/null +++ b/tests/ui/nonunique_congee_index.rs @@ -0,0 +1,18 @@ +// Rule: a non-unique index needs a backend that can store several rows under +// one key. Congee is an ART over unique keys, so pairing the two is refused +// and the message names the backends that do work. +use worktable::worktable; + +worktable! { + name: NonUniqueCongee, + persist: false, + columns: { + id: u64 primary_key autoincrement, + group_id: u64, + }, + indexes: { + group_idx: group_id using congee, + }, +} + +fn main() {} diff --git a/tests/ui/nonunique_congee_index.stderr b/tests/ui/nonunique_congee_index.stderr new file mode 100644 index 00000000..bff9e38c --- /dev/null +++ b/tests/ui/nonunique_congee_index.stderr @@ -0,0 +1,5 @@ +error: non-unique index `group_idx` cannot use `congee`; non-unique indexes currently require `worktables_index` or `arctic` + --> tests/ui/nonunique_congee_index.rs:14:9 + | +14 | group_idx: group_id using congee, + | ^^^^^^^^^ diff --git a/tests/ui/query_over_unknown_column.rs b/tests/ui/query_over_unknown_column.rs new file mode 100644 index 00000000..4e754589 --- /dev/null +++ b/tests/ui/query_over_unknown_column.rs @@ -0,0 +1,20 @@ +// Rule: a query names columns of its own table. `missing` is not one, and the +// error has to say so at the column rather than somewhere inside the generated +// row type. +use worktable::worktable; + +worktable! { + name: UnknownQueryColumn, + persist: false, + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, + queries: { + update: { + MissingById(missing) by id, + } + }, +} + +fn main() {} diff --git a/tests/ui/query_over_unknown_column.stderr b/tests/ui/query_over_unknown_column.stderr new file mode 100644 index 00000000..a39135ce --- /dev/null +++ b/tests/ui/query_over_unknown_column.stderr @@ -0,0 +1,5 @@ +error: Unexpected column name + --> tests/ui/query_over_unknown_column.rs:15:25 + | +15 | MissingById(missing) by id, + | ^^^^^^^ diff --git a/tests/ui/runtime_before_name.rs b/tests/ui/runtime_before_name.rs new file mode 100644 index 00000000..cfc48ba0 --- /dev/null +++ b/tests/ui/runtime_before_name.rs @@ -0,0 +1,31 @@ +// DRAFT. Not wired into tests/ui.rs. The parser lane that lands `runtime:` +// adds it there and blesses the .stderr. +// +// Rule (contract section 2): `name`, `version`, `persist` and `partition_by` +// are a fixed ordered prefix, and the free-order section loop starts after +// them. So `runtime` cannot precede `name`. +// +// NOTE for whoever wires this up. This is the one draft whose current message +// is already the right verdict: the ordered prefix reads the first identifier, +// finds it is not `name`, and says +// +// Expected `name` field. `WorkTable` name must be specified +// +// which is correct and confusing at once. It names the field that is missing +// rather than the one that is in the wrong place, so a reader who put +// `runtime` first has to work out that `runtime` is legal but not here. +// Whether to special-case it is a judgement call for the parser lane, not +// something this test decides. Bless whichever message that lane settles on. +use worktable::worktable; + +worktable! { + runtime: nagoya, + name: RuntimeBeforeName, + persist: false, + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/runtime_declared_twice.rs b/tests/ui/runtime_declared_twice.rs new file mode 100644 index 00000000..46007c24 --- /dev/null +++ b/tests/ui/runtime_declared_twice.rs @@ -0,0 +1,21 @@ +// DRAFT. Not wired into tests/ui.rs. The parser lane that lands `runtime:` +// adds it and generates the .stderr. +// +// Rule (contract section 7): `runtime` is an arm of the free-order section +// loop, so nothing about its position stops it appearing twice. Two of them +// is a duplicate section, and the message says so rather than silently +// keeping the last. +use worktable::worktable; + +worktable! { + name: RuntimeTwice, + persist: false, + runtime: nagoya(locality), + runtime: tokio, + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/runtime_pinned_and_call_site.rs b/tests/ui/runtime_pinned_and_call_site.rs new file mode 100644 index 00000000..ddefac6a --- /dev/null +++ b/tests/ui/runtime_pinned_and_call_site.rs @@ -0,0 +1,40 @@ +// DRAFT. Not wired into tests/ui.rs. The lane that lands the `Profile` marker +// and the call-site builder adds it and generates the .stderr. +// +// Rule (contract section 7, last row): a query whose section is annotated has +// its runtime pinned by the schema, so a call-site `.runtime()` on it is an +// error. +// +// This is the case the whole harness is for. The rule is implemented as an +// unsatisfiable bound carrying `#[diagnostic::on_unimplemented]`, never by +// omitting the method: omitting it yields "no method named `runtime`", which +// points at the call rather than at the schema line that pinned it. Both +// spellings fail to compile, so a test asserting only on failure cannot tell +// them apart. The .stderr has to contain "already has a runtime pinned by the +// schema". +use worktable::prelude::*; +use worktable::worktable; + +runtimes! { + wide: nagoya(spread), +} + +worktable! { + name: PinnedAndCallSite, + persist: false, + runtime: nagoya(locality), + columns: { + id: u64 primary_key autoincrement, + qty: u64, + }, + queries: { + update runtime wide: { + Fill(qty) by id, + } + }, +} + +fn main() { + let table = PinnedAndCallSiteWorkTable::default(); + let _ = table.fill_query().runtime(wide).execute(); +} diff --git a/tests/ui/runtime_profile_backend_mismatch.rs b/tests/ui/runtime_profile_backend_mismatch.rs new file mode 100644 index 00000000..3baf363b --- /dev/null +++ b/tests/ui/runtime_profile_backend_mismatch.rs @@ -0,0 +1,32 @@ +// DRAFT. Not wired into tests/ui.rs. The lane that lands the `Profile` marker +// adds it and generates the .stderr. +// +// Rule (contract section 6): `.runtime()` and the section annotation take +// `P: Profile`, so naming a `tokio` profile on a +// table declared `runtime: nagoya` fails as a bound that names both backends. +// The message has to carry both names; a bare "trait bound not satisfied" is +// the regression this case exists to catch. +use worktable::prelude::*; +use worktable::worktable; + +runtimes! { + tokio_max: tokio, + fast_local: nagoya(locality), +} + +worktable! { + name: BackendMismatch, + persist: false, + runtime: nagoya(locality), + columns: { + id: u64 primary_key autoincrement, + qty: u64, + }, + queries: { + update runtime tokio_max: { + Fill(qty) by id, + } + }, +} + +fn main() {} diff --git a/tests/ui/runtime_tokio_with_flavor.rs b/tests/ui/runtime_tokio_with_flavor.rs new file mode 100644 index 00000000..4135ce92 --- /dev/null +++ b/tests/ui/runtime_tokio_with_flavor.rs @@ -0,0 +1,20 @@ +// DRAFT. Not wired into tests/ui.rs. The parser lane that lands `runtime:` +// adds it and generates the .stderr. +// +// Rule (contract section 7): `RuntimeBackend::Tokio` carries no flavor, so +// `tokio(spread)` is refused rather than having its parenthesised part +// dropped. Dropping it would accept a declaration that means something the +// table cannot do. +use worktable::worktable; + +worktable! { + name: TokioWithFlavor, + persist: false, + runtime: tokio(spread), + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/runtime_unimplemented_blocking.rs b/tests/ui/runtime_unimplemented_blocking.rs new file mode 100644 index 00000000..94da0a2c --- /dev/null +++ b/tests/ui/runtime_unimplemented_blocking.rs @@ -0,0 +1,23 @@ +// DRAFT. Not wired into tests/ui.rs. The parser lane that lands `runtime:` +// adds it there and blesses the .stderr. Today this fails with "Unexpected +// token `runtime`; expected one of `columns`, `indexes`, `queries`, +// `config`", which is the free-order section loop refusing an arm it does not +// have yet: the right verdict for the wrong reason. +// +// Rule (contract section 7): `blocking` is a name the parser recognises only so it +// can refuse it well. Per PR #58, an inert declaration is an error, so this +// must not be accepted and quietly ignored. The message has to name `blocking`, +// say it is not implemented, and list the backends that are. +use worktable::worktable; + +worktable! { + name: UnimplementedBlocking, + persist: false, + runtime: blocking, + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/runtime_unimplemented_bwos.rs b/tests/ui/runtime_unimplemented_bwos.rs new file mode 100644 index 00000000..5ca3b327 --- /dev/null +++ b/tests/ui/runtime_unimplemented_bwos.rs @@ -0,0 +1,23 @@ +// DRAFT. Not wired into tests/ui.rs. The parser lane that lands `runtime:` +// adds it there and blesses the .stderr. Today this fails with "Unexpected +// token `runtime`; expected one of `columns`, `indexes`, `queries`, +// `config`", which is the free-order section loop refusing an arm it does not +// have yet: the right verdict for the wrong reason. +// +// Rule (contract section 7): `bwos` is a name the parser recognises only so it +// can refuse it well. Per PR #58, an inert declaration is an error, so this +// must not be accepted and quietly ignored. The message has to name `bwos`, +// say it is not implemented, and list the backends that are. +use worktable::worktable; + +worktable! { + name: UnimplementedBwos, + persist: false, + runtime: bwos, + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/runtime_unimplemented_forte.rs b/tests/ui/runtime_unimplemented_forte.rs new file mode 100644 index 00000000..57ded7bc --- /dev/null +++ b/tests/ui/runtime_unimplemented_forte.rs @@ -0,0 +1,23 @@ +// DRAFT. Not wired into tests/ui.rs. The parser lane that lands `runtime:` +// adds it there and blesses the .stderr. Today this fails with "Unexpected +// token `runtime`; expected one of `columns`, `indexes`, `queries`, +// `config`", which is the free-order section loop refusing an arm it does not +// have yet: the right verdict for the wrong reason. +// +// Rule (contract section 7): `forte` is a name the parser recognises only so it +// can refuse it well. Per PR #58, an inert declaration is an error, so this +// must not be accepted and quietly ignored. The message has to name `forte`, +// say it is not implemented, and list the backends that are. +use worktable::worktable; + +worktable! { + name: UnimplementedForte, + persist: false, + runtime: forte, + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/runtime_unknown_flavor.rs b/tests/ui/runtime_unknown_flavor.rs new file mode 100644 index 00000000..603302a0 --- /dev/null +++ b/tests/ui/runtime_unknown_flavor.rs @@ -0,0 +1,19 @@ +// DRAFT. Not wired into tests/ui.rs. The parser lane that lands `runtime:` +// adds it and generates the .stderr. +// +// Rule (contract section 7): nagoya takes `locality`, `spread` or +// `throughput`. An unknown flavor is refused with the three listed, the same +// way `using` lists the four index backends. +use worktable::worktable; + +worktable! { + name: UnknownFlavor, + persist: false, + runtime: nagoya(banana), + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, +} + +fn main() {} diff --git a/tests/ui/runtime_unknown_profile.rs b/tests/ui/runtime_unknown_profile.rs new file mode 100644 index 00000000..9b06c180 --- /dev/null +++ b/tests/ui/runtime_unknown_profile.rs @@ -0,0 +1,31 @@ +// DRAFT. Not wired into tests/ui.rs. The lane that lands section annotations +// adds it and generates the .stderr. +// +// Rule (contract section 2): the token after `runtime` at a section is a +// profile name declared by `runtimes!`, never a backend literal. `nope` is not +// one, and the message names it rather than reporting a parse failure at the +// colon. +use worktable::prelude::*; +use worktable::worktable; + +runtimes! { + tokio_max: tokio, + fast_local: nagoya(locality), +} + +worktable! { + name: UnknownProfile, + persist: false, + runtime: nagoya(locality), + columns: { + id: u64 primary_key autoincrement, + qty: u64, + }, + queries: { + update runtime nope: { + Fill(qty) by id, + } + }, +} + +fn main() {} diff --git a/tests/ui/unknown_index_backend.rs b/tests/ui/unknown_index_backend.rs new file mode 100644 index 00000000..20896230 --- /dev/null +++ b/tests/ui/unknown_index_backend.rs @@ -0,0 +1,18 @@ +// Rule: `using` takes one of the four index backends. A misspelling has to +// name the four rather than fall through to the default, because silently +// defaulting picks a data structure the author did not ask for. +use worktable::worktable; + +worktable! { + name: UnknownBackend, + persist: false, + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, + indexes: { + value_idx: value unique using treap, + }, +} + +fn main() {} diff --git a/tests/ui/unknown_index_backend.stderr b/tests/ui/unknown_index_backend.stderr new file mode 100644 index 00000000..0a537998 --- /dev/null +++ b/tests/ui/unknown_index_backend.stderr @@ -0,0 +1,5 @@ +error: unknown index backend; expected `worktables_index`, `indexset`, `congee`, `fxhash`, or `arctic` + --> tests/ui/unknown_index_backend.rs:14:39 + | +14 | value_idx: value unique using treap, + | ^^^^^ diff --git a/tests/worktable/base.rs b/tests/worktable/base.rs index 77b14c2e..1cd957a9 100644 --- a/tests/worktable/base.rs +++ b/tests/worktable/base.rs @@ -28,6 +28,7 @@ worktable! ( AnotherByExchange(another) by exchange, AnotherByTest(another) by test, AnotherById(another) by id, + ExchangeById(exchange) by id, }, delete: { ByAnother() by another, @@ -109,7 +110,7 @@ async fn iter_with_async() { table.iter_with_async(|_| async move { Ok(()) }).await.unwrap() } -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn update_spawn() { let table = Arc::new(TestWorkTable::default()); let row = TestRow { @@ -137,7 +138,7 @@ async fn update_spawn() { assert!(table.select(2).is_none()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn upsert_spawn() { let table = Arc::new(TestWorkTable::default()); let row = TestRow { @@ -266,12 +267,48 @@ async fn update_parallel() { } h.await.unwrap(); - for (test, val) in i_state.lock_arc().iter() { + for (test, val) in i_state.lock().iter() { let row = table.select_by_test(*test).unwrap(); assert_eq!(row.another, *val) } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn secondary_update_follows_concurrent_row_relocation() { + let table = Arc::new(TestWorkTable::default()); + table.insert(TestRow { + id: 0, + test: 1, + another: 0, + exchange: "initial".into(), + }).await.unwrap(); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let writer_table = table.clone(); + let writer_barrier = barrier.clone(); + let writer = tokio::spawn(async move { + writer_barrier.wait().await; + for revision in 1..=2000 { + writer_table.update_exchange_by_id(ExchangeByIdQuery { + exchange: format!("relocated-{revision}-{}", "x".repeat(revision % 64)), + }, 0).await.unwrap(); + tokio::task::yield_now().await; + } + }); + barrier.wait().await; + for revision in 1..=2000 { + table.update_another_by_test(AnotherByTestQuery { another: revision }, 1) + .await.unwrap(); + tokio::task::yield_now().await; + } + writer.await.unwrap(); + let row = table.select(0).unwrap(); + assert_eq!(row.another, 2000); + assert_eq!(row.exchange, format!("relocated-2000-{}", "x".repeat(2000 % 64))); + assert_eq!(table.select_by_test(1).unwrap(), row); + let indexed = table.select_by_another(2000).execute().unwrap(); + assert_eq!(indexed, vec![row]); +} + #[tokio::test] async fn delete() { let table = TestWorkTable::default(); diff --git a/tests/worktable/bench.rs b/tests/worktable/bench.rs index 8c474e07..d7d24524 100644 --- a/tests/worktable/bench.rs +++ b/tests/worktable/bench.rs @@ -1,8 +1,8 @@ +use nagoya::sync::RwLock; use rand::distr::{Alphanumeric, SampleString}; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; -use tokio::sync::RwLock; use worktable::prelude::*; use worktable_codegen::worktable; diff --git a/tests/worktable/cancel_safety.rs b/tests/worktable/cancel_safety.rs index 8b12c433..0ada7eb6 100644 --- a/tests/worktable/cancel_safety.rs +++ b/tests/worktable/cancel_safety.rs @@ -29,7 +29,7 @@ fn install_blocker(table: &CancelSafetyWorkTable, pk: &CancelSafetyPrimaryKey) - table .0 .lock_manager - .insert(pk.clone(), Arc::new(tokio::sync::RwLock::new(blocker_state))); + .insert(pk.clone(), Arc::new(nagoya::sync::RwLock::new(blocker_state))); blocker } diff --git a/tests/worktable/columnar.rs b/tests/worktable/columnar.rs new file mode 100644 index 00000000..b4ea942a --- /dev/null +++ b/tests/worktable/columnar.rs @@ -0,0 +1,387 @@ +use std::sync::Arc; +use worktable::prelude::*; +use worktable::worktable; + +#[test] +fn columnar_publication_guard_excludes_rebuilds() { + let table = ColumnarMetricsWorkTable::default(); + let guard = table + .0 + .indexes + .row_publication() + .expect("columnar tables need a publication gate"); + assert!(table.0.indexes.columnar_publication.try_write().is_none()); + drop(guard); + assert!(table.0.indexes.columnar_publication.try_write().is_some()); +} + +#[test] +fn persisted_index_capacity_fits_slot_identifiers() { + assert!(get_index_page_size_from_data_length::(4 * 1024 * 1024) <= usize::from(u16::MAX)); +} + +worktable!( + name: ColumnarMetrics, + persist: false, + columns: { + id: u64 primary_key, + host_id: u64 columnar(chunk_rows(2), compression(none)), + timestamp: i64 columnar, + temperature: i64 columnar(chunk_rows(2)), + label: String, + }, + config: { + columnar_slot_id: ColumnSlotId16, + columnar_chunk_rows: 4, + }, + columnar_indexes: { + host_time: { + cluster_by: [host_id, timestamp], + }, + }, + queries: { + update: { + TemperatureById(temperature) by id, + }, + in_place: { + TimestampById(timestamp) by id, + } + }, +); + +worktable!( + name: TinyColumnarIds, + persist: false, + columns: { + id: u16 primary_key, + value: u16 columnar(chunk_rows(32), compression(none)), + }, + config: { + columnar_slot_id: ColumnSlotId8, + columnar_chunk_rows: 32, + }, +); + +// Compile coverage for the persisted derive path. The columnar replica is +// intentionally skipped by the existing index file format and rebuilt from +// authoritative rows after load. +worktable!( + name: PersistedColumnarMetrics, + persist: true, + columns: { + id: u64 primary_key, + host_id: u64 columnar(chunk_rows(4), compression(none)), + timestamp: i64 columnar(chunk_rows(4), compression(none)), + }, + columnar_indexes: { + host_time: { + cluster_by: [host_id, timestamp], + }, + }, +); + +worktable!( + name: CongeeColumnarSideIndex, + persist: false, + columns: { + id: u64 primary_key using congee, + value: u64 columnar, + }, + columnar_indexes: { + value_order: { + cluster_by: [value], + }, + }, +); + +worktable!( + name: ArcticColumnarSideIndex, + persist: false, + columns: { + id: u64 primary_key using arctic, + value: u64 columnar, + }, + columnar_indexes: { + value_order: { + cluster_by: [value], + }, + }, +); + +#[tokio::test] +async fn columnar_fields_and_clustered_index_follow_mutations() { + let table = ColumnarMetricsWorkTable::default(); + table + .insert(ColumnarMetricsRow { + id: 1, + host_id: 2, + timestamp: 20, + temperature: 72, + label: "second".to_string(), + }) + .await + .unwrap(); + table + .insert(ColumnarMetricsRow { + id: 2, + host_id: 1, + timestamp: 10, + temperature: 68, + label: "first".to_string(), + }) + .await + .unwrap(); + + let host_two = table.columnar_select_host_time(2, 20).unwrap(); + assert_eq!(host_two.len(), 1); + assert_eq!(host_two[0].primary_key().0, 1); + assert_eq!(table.columnar_project_temperature(&host_two).unwrap()[0].1, 72); + + let ordered = table.columnar_scan_host_time().unwrap(); + let projected = table.columnar_project_host_id(&ordered).unwrap(); + assert_eq!(projected.iter().map(|(_, value)| *value).collect::>(), [1, 2]); + + table + .update(ColumnarMetricsRow { + id: 1, + host_id: 3, + timestamp: 30, + temperature: 75, + label: "updated".to_string(), + }) + .await + .unwrap(); + + assert!(table.columnar_select_host_time(2, 20).unwrap().is_empty()); + let updated = table.columnar_select_host_time(3, 30).unwrap(); + assert_eq!(updated, host_two, "row identity survives an update"); + assert_eq!(table.columnar_project_temperature(&updated).unwrap()[0].1, 75); + + table + .update_temperature_by_id(TemperatureByIdQuery { temperature: 76 }, 1) + .await + .unwrap(); + assert_eq!(table.columnar_project_temperature(&updated).unwrap()[0].1, 76); + + table + .update_timestamp_by_id_in_place(|value| *value = 40.into(), 1) + .await + .unwrap(); + assert!(table.columnar_is_dirty()); + table.rebuild_columnar().unwrap(); + assert!(!table.columnar_is_dirty()); + assert!(table.columnar_select_host_time(3, 30).unwrap().is_empty()); + assert_eq!(table.columnar_select_host_time(3, 40).unwrap(), updated); + + table.delete(2).await.unwrap(); + assert_eq!(table.columnar_scan_host_id().unwrap().len(), 1); + assert_eq!(table.columnar_scan_host_time().unwrap(), updated); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_reinsert_and_columnar_refresh_preserve_row_identity() { + let table = Arc::new(ColumnarMetricsWorkTable::default()); + table + .insert(ColumnarMetricsRow { + id: 7, + host_id: 1, + timestamp: 0, + temperature: 1, + label: "short".to_string(), + }) + .await + .unwrap(); + let stable_id = table.columnar_select_host_time(1, 0).unwrap()[0].clone(); + + let updater = { + let table = Arc::clone(&table); + tokio::spawn(async move { + for value in 1..=200 { + table + .update(ColumnarMetricsRow { + id: 7, + host_id: 1, + timestamp: value, + temperature: value, + label: if value % 2 == 0 { + "a much longer row value".to_string() + } else { + "tiny".to_string() + }, + }) + .await + .unwrap(); + } + }) + }; + + for _ in 0..200 { + for (row_id, _) in table.columnar_scan_timestamp().unwrap() { + assert_eq!(row_id, stable_id); + } + } + updater.await.unwrap(); + + assert_eq!(table.columnar_select_host_time(1, 200).unwrap(), [stable_id]); +} + +#[tokio::test] +async fn configured_slot_id_capacity_is_checked_and_deleted_slots_are_safe_to_reuse() { + let table = TinyColumnarIdsWorkTable::default(); + for id in 0..=u8::MAX as u16 { + table.insert(TinyColumnarIdsRow { id, value: id }).await.unwrap(); + } + + let stale = table + .columnar_scan_value() + .unwrap() + .into_iter() + .find(|(row_ref, _)| row_ref.primary_key().0 == 7) + .unwrap() + .0; + let error = table + .insert(TinyColumnarIdsRow { id: 256, value: 256 }) + .await + .unwrap_err(); + assert!(matches!(error, WorkTableError::ColumnSlotIdExhausted(8))); + assert!( + table.select(256).is_none(), + "capacity failure rolls back the authoritative row" + ); + + table.delete(7).await.unwrap(); + table.insert(TinyColumnarIdsRow { id: 256, value: 256 }).await.unwrap(); + + let replacement = table + .columnar_scan_value() + .unwrap() + .into_iter() + .find(|(row_ref, _)| row_ref.primary_key().0 == 256) + .unwrap() + .0; + assert!( + table.columnar_project_value(&[stale]).unwrap().is_empty(), + "a recycled slot cannot alias a different primary key" + ); + + table.delete(256).await.unwrap(); + table.insert(TinyColumnarIdsRow { id: 256, value: 999 }).await.unwrap(); + assert!( + table.columnar_project_value(&[replacement]).unwrap().is_empty(), + "delete and reinsert of the same primary key cannot revive a stale row reference" + ); + assert_eq!(table.columnar_slots_in_use(), 256); + assert_eq!(table.columnar_slots_high_water(), 256); +} + +#[tokio::test] +async fn row_refs_are_scoped_to_one_table_incarnation() { + let first = TinyColumnarIdsWorkTable::default(); + first.insert(TinyColumnarIdsRow { id: 1, value: 11 }).await.unwrap(); + let retained = first.columnar_scan_value().unwrap()[0].0.clone(); + + let second = TinyColumnarIdsWorkTable::default(); + second.insert(TinyColumnarIdsRow { id: 1, value: 22 }).await.unwrap(); + + assert!( + second.columnar_project_value(&[retained]).unwrap().is_empty(), + "a ref from another table instance must not alias the same primary key and slot" + ); +} + +#[tokio::test] +async fn columnar_side_indexes_compose_with_congee_and_arctic_using_backends() { + macro_rules! exercise { + ($table:ident, $row:ident) => {{ + let table = $table::default(); + table.insert($row { id: 1, value: 20 }).await.unwrap(); + table.insert($row { id: 2, value: 10 }).await.unwrap(); + + let ordered = table.columnar_scan_value_order().unwrap(); + assert_eq!( + table + .columnar_project_value(&ordered) + .unwrap() + .into_iter() + .map(|(_, value)| value) + .collect::>(), + [10, 20] + ); + + table.update($row { id: 1, value: 5 }).await.unwrap(); + assert_eq!(table.columnar_select_value_order(20).unwrap(), []); + assert_eq!(table.columnar_select_value_order(5).unwrap().len(), 1); + + table.delete(2).await.unwrap(); + assert_eq!(table.columnar_scan_value().unwrap().len(), 1); + }}; + } + + exercise!(CongeeColumnarSideIndexWorkTable, CongeeColumnarSideIndexRow); + exercise!(ArcticColumnarSideIndexWorkTable, ArcticColumnarSideIndexRow); +} + +/// The `ColumnSlotIdExhausted` rollback arms, verified by construction. +/// +/// PR #58 rolled back by re-inserting the old link into the primary index. +/// On this tree the primary index is swung only *after* every index check has +/// passed, so that rollback was dropped from `reinsert` and `reinsert_cdc`. +/// Dropping a rollback in a persistence path is exactly the change that shows +/// up as silent corruption on reload rather than as a failing operation, so +/// what the arms leave behind is asserted here rather than read. +/// +/// Two claims, and the second is the one that was never re-derived: +/// +/// 1. a failed insert leaves **no** primary-index entry for the key, and the +/// key is free for a later insert +/// 2. an update at capacity does not fail at all, because `replace_row` reuses +/// the row's existing slot, so the dropped rollback is on a path an update +/// cannot reach with a live row +#[tokio::test] +async fn a_capacity_failure_never_leaves_a_primary_index_entry_behind() { + let table = TinyColumnarIdsWorkTable::default(); + for id in 0..=u8::MAX as u16 { + table.insert(TinyColumnarIdsRow { id, value: id }).await.unwrap(); + } + assert_eq!(table.columnar_slots_in_use(), 256); + + // Claim 2, and it has to be asserted before the table is disturbed: an + // update of a row that already holds a slot reuses that slot, so a full + // table is not a reason for it to fail. This is what makes the dropped + // rollback unreachable for a live row rather than merely unlikely. + let before = table.select(42).expect("row 42 is present"); + table + .reinsert(before.clone(), TinyColumnarIdsRow { id: 42, value: 4242 }) + .await + .expect("an update at capacity reuses the row's own slot"); + assert_eq!(table.select(42).expect("row 42 survives").value, 4242); + assert_eq!( + table.columnar_slots_in_use(), + 256, + "an update must not consume a second slot" + ); + + // Claim 1: the failed insert. + let error = table + .insert(TinyColumnarIdsRow { id: 300, value: 300 }) + .await + .unwrap_err(); + assert!(matches!(error, WorkTableError::ColumnSlotIdExhausted(8)), "{error:?}"); + assert!( + table.select(300).is_none(), + "a failed insert must leave no primary index entry, or the index points at a row that was never written" + ); + + // The key is free, which is the observable consequence of the index entry + // having actually been removed rather than merely being unreadable. + table.delete(7).await.unwrap(); + table + .insert(TinyColumnarIdsRow { id: 300, value: 300 }) + .await + .expect("the key a failed insert used is free"); + assert_eq!(table.select(300).expect("row 300 is present").value, 300); + assert_eq!(table.columnar_slots_in_use(), 256); + + // And nothing the failure touched disturbed the update above. + assert_eq!(table.select(42).expect("row 42 still present").value, 4242); +} diff --git a/tests/worktable/concurrency.rs b/tests/worktable/concurrency.rs index 51eceb4f..75382664 100644 --- a/tests/worktable/concurrency.rs +++ b/tests/worktable/concurrency.rs @@ -115,7 +115,7 @@ macro_rules! backend_suite { scope.spawn(move || { for n in 0..per_writer { let id = w * per_writer + n; - futures::executor::block_on(table.insert(row(id))).expect("insert"); + nagoya::block_on(table.insert(row(id))).expect("insert"); // Read while the others write, so the scan and // the mutations actually overlap. let _ = table.select(id); @@ -166,7 +166,7 @@ macro_rules! backend_suite { scope.spawn(move || { for n in 0..per_writer { let id = seed + w * per_writer + n; - futures::executor::block_on(table.insert(row(id))).expect("insert"); + nagoya::block_on(table.insert(row(id))).expect("insert"); } }); } @@ -218,11 +218,11 @@ macro_rules! backend_suite { let rows: Vec<_> = (chunk..(chunk + 16).min(per_writer)) .map(|n| row(base + n)) .collect(); - futures::executor::block_on(table.insert_many(rows)).expect("insert_many"); + nagoya::block_on(table.insert_many(rows)).expect("insert_many"); } } else { for n in 0..per_writer { - futures::executor::block_on(table.insert(row(base + n))).expect("insert"); + nagoya::block_on(table.insert(row(base + n))).expect("insert"); } } }); @@ -263,7 +263,7 @@ macro_rules! backend_suite { scope.spawn(move || { // Distinct primary keys, one shared payload. let contended = ConcRow { id: w, payload: 42, bucket: 0 }; - if futures::executor::block_on(table.insert(contended)).is_ok() { + if nagoya::block_on(table.insert(contended)).is_ok() { winners.fetch_add(1, Ordering::Release); } }); @@ -363,8 +363,8 @@ macro_rules! backend_suite { (base + WINDOW, base) }; for i in 0..WINDOW { - futures::executor::block_on(table.delete(from + i)).expect("delete"); - futures::executor::block_on(table.insert(row(to + i))).expect("insert"); + nagoya::block_on(table.delete(from + i)).expect("delete"); + nagoya::block_on(table.insert(row(to + i))).expect("insert"); } } finished.fetch_add(1, Ordering::Release); @@ -452,7 +452,7 @@ macro_rules! backend_suite { let table = Arc::clone(&table); scope.spawn(move || { for n in 0..per_writer { - futures::executor::block_on(table.insert(row(w * per_writer + n))).expect("insert"); + nagoya::block_on(table.insert(row(w * per_writer + n))).expect("insert"); } }); } diff --git a/tests/worktable/in_place.rs b/tests/worktable/in_place.rs index 3a29e7ea..5265e524 100644 --- a/tests/worktable/in_place.rs +++ b/tests/worktable/in_place.rs @@ -267,15 +267,15 @@ async fn test_update_in_place_and_update_sized_multithread() -> eyre::Result<()> h1.await?; h2.await?; - for (id, smth) in i_state.lock_arc().iter() { + for (id, smth) in i_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.something, smth); } - for (id, val) in val2_state.lock_arc().iter() { + for (id, val) in val2_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.val2, val); } - for (id, val) in val_state.lock_arc().iter() { + for (id, val) in val_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.val, val); } @@ -354,12 +354,12 @@ async fn test_update_in_place_and_update_unsized_multithread() -> eyre::Result<( h1.await?; h2.await?; - for (id, smth) in i_state.lock_arc().iter() { + for (id, smth) in i_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.another, smth); } let mut errors = 0; - for (id, val) in val2_state.lock_arc().iter() { + for (id, val) in val2_state.lock().iter() { let row = table.select(*id).unwrap(); if &row.val2 != val { errors += 1; @@ -367,7 +367,7 @@ async fn test_update_in_place_and_update_unsized_multithread() -> eyre::Result<( } assert_eq!(errors, 0); let mut errors = 0; - for (id, val) in val_state.lock_arc().iter() { + for (id, val) in val_state.lock().iter() { let row = table.select(*id).unwrap(); if &row.val != val { errors += 1; diff --git a/tests/worktable/index/insert.rs b/tests/worktable/index/insert.rs index f0ae7c85..b0615d77 100644 --- a/tests/worktable/index/insert.rs +++ b/tests/worktable/index/insert.rs @@ -228,7 +228,7 @@ async fn insert_when_unique_violated() { attr3: 123456789, attr4: row_new_attr_4.clone(), }; - assert!(futures::executor::block_on(shared.insert(row)).is_err()); + assert!(nagoya::block_on(shared.insert(row)).is_err()); } }); @@ -315,7 +315,7 @@ async fn insert_when_pk_violated() { attr3: 123456789, attr4: "Attribute__4".to_string(), }; - assert!(futures::executor::block_on(shared.insert(row)).is_err()); + assert!(nagoya::block_on(shared.insert(row)).is_err()); } }); diff --git a/tests/worktable/index_backends.rs b/tests/worktable/index_backends.rs index 94126653..09064dfc 100644 --- a/tests/worktable/index_backends.rs +++ b/tests/worktable/index_backends.rs @@ -398,7 +398,7 @@ async fn native_art_backends_survive_wal_reload_and_further_mutation() { remove_dir_if_exists(CONGEE_ROOT.to_string()).await; } -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn native_art_backends_recover_concurrent_same_row_updates() { use std::sync::Arc; @@ -462,7 +462,7 @@ async fn native_art_backends_recover_concurrent_same_row_updates() { } #[cfg(feature = "logical-index-persistence")] -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn logical_wti_recovers_concurrent_same_row_updates() { use std::sync::Arc; diff --git a/tests/worktable/key_widths.rs b/tests/worktable/key_widths.rs index f0429fcb..349d0a35 100644 --- a/tests/worktable/key_widths.rs +++ b/tests/worktable/key_widths.rs @@ -66,7 +66,7 @@ macro_rules! width_case { ); // And the entry comes out again. - futures::executor::block_on(table.delete(2u64)).expect("delete"); + nagoya::block_on(table.delete(2u64)).expect("delete"); assert!( table.select_by_key(42 as $key).is_none(), "{}: {} index still resolves a deleted row", diff --git a/tests/worktable/lock_order.rs b/tests/worktable/lock_order.rs index 3c00c6cd..519e58e4 100644 --- a/tests/worktable/lock_order.rs +++ b/tests/worktable/lock_order.rs @@ -90,7 +90,7 @@ async fn multi_row_update_locks_in_primary_key_order_not_index_order() { table .0 .lock_manager - .insert(blocker_pk.clone(), Arc::new(tokio::sync::RwLock::new(blocker_state))); + .insert(blocker_pk.clone(), Arc::new(nagoya::sync::RwLock::new(blocker_state))); let update_table = table.clone(); let update = tokio::spawn(async move { diff --git a/tests/worktable/mod.rs b/tests/worktable/mod.rs index 5bf547e5..b2cf318b 100644 --- a/tests/worktable/mod.rs +++ b/tests/worktable/mod.rs @@ -3,6 +3,7 @@ mod base; mod bench; mod borrowed_primary_key; mod cancel_safety; +mod columnar; mod concurrency; mod config; mod count; @@ -18,12 +19,14 @@ mod key_widths; mod leak_probe; mod lock_order; mod multi_row_deadlock; +mod multi_thread_discipline; mod mutation_gate_deadlock; mod nid; mod nonunique_arctic; mod option; mod partitioned; mod reinsert_visibility; +mod runtime_backends; mod schema_const; mod tuple_primary_key; mod unique_fixed_unsized; @@ -36,5 +39,6 @@ mod uuid; mod vacuum; mod vacuum_invariants; mod vacuum_no_row_loss; +mod vec_table; mod with_enum; mod wrong_row_update; diff --git a/tests/worktable/multi_row_deadlock.rs b/tests/worktable/multi_row_deadlock.rs index a144b775..015c23ac 100644 --- a/tests/worktable/multi_row_deadlock.rs +++ b/tests/worktable/multi_row_deadlock.rs @@ -81,7 +81,7 @@ async fn overlapping_multi_row_updates_do_not_deadlock() { }) }; - tokio::time::timeout(Duration::from_secs(60), async { + tokio::time::timeout(Duration::from_secs(5), async { by_a.await.expect("group_a updater must not panic"); by_b.await.expect("group_b updater must not panic"); }) diff --git a/tests/worktable/multi_thread_discipline.rs b/tests/worktable/multi_thread_discipline.rs new file mode 100644 index 00000000..3dea0710 --- /dev/null +++ b/tests/worktable/multi_thread_discipline.rs @@ -0,0 +1,151 @@ +//! A concurrency test on a current-thread runtime is not a concurrency test. +//! +//! `#[tokio::test]` with no arguments builds a **current-thread** runtime. +//! Tasks spawned inside it interleave only at await points, on one thread, so +//! two writers never overlap and no data race between them can be observed. +//! The test still passes. It simply stops proving what its name says. +//! +//! This repo has been bitten by it. Commit 2702c06 records two tests that +//! shared one table directory and only started failing once the persistence +//! worker moved off `tokio::spawn`: while that worker ran on the test's own +//! current-thread runtime, the two tables' writes never overlapped in time and +//! the corruption stayed invisible. The single-threaded harness was hiding a +//! real defect, and the runtime-backend work is exactly the kind of change +//! that moves work between runtimes again. +//! +//! So this file is the verifier rather than a note in a doc comment: a rule +//! nothing checks is a rule that is quietly false. It scans the test sources +//! for `#[tokio::test]` bodies that call `tokio::spawn` and fails on any that +//! is not already on the list below. +//! +//! `std::thread::spawn` is deliberately not flagged. An OS thread is genuinely +//! parallel whatever the harness runtime is doing, which is why +//! `tests/worktable/concurrency.rs` and `tests/worktable/partitioned.rs` are +//! honest despite their bare `#[tokio::test]` attributes. +//! +//! The fix for a flagged test is one line: +//! +//! ```text +//! #[tokio::test(flavor = "multi_thread", worker_threads = 4)] +//! ``` + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +/// Tests that spawn tokio tasks from a current-thread runtime today. +/// +/// Empty, and meant to stay that way. Every entry was coverage weaker than its +/// name suggested: six had "concurrent" or "races" in the name or doc comment +/// and proved no such thing, and the two `base.rs` ones asserted only that a +/// spawned mutation future is `Send` and joins. +/// +/// The list is retained rather than deleted because the check is two-sided. +/// A new offender fails against the empty list, which is the point, and an +/// entry that stops offending also fails, so re-adding one to silence a +/// failure cannot be done quietly. +const KNOWN_CURRENT_THREAD_SPAWNERS: &[(&str, &str)] = &[]; + +fn tests_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests") +} + +fn rust_sources(dir: &Path, into: &mut Vec) { + for entry in std::fs::read_dir(dir).expect("the tests directory is readable") { + let path = entry.expect("a readable directory entry").path(); + if path.is_dir() { + rust_sources(&path, into); + } else if path.extension().is_some_and(|ext| ext == "rs") { + into.push(path); + } + } +} + +/// The body of the item that follows `lines[start]`, by brace balance. +/// +/// Crude on purpose. A brace inside a string literal would confuse it, and the +/// alternative is a parser dependency for a lint that has to stay cheap enough +/// that nobody is tempted to delete it. A miscount can only mis-scope a body, +/// which shows up as a name this file cannot explain rather than as silence. +fn body_after(lines: &[&str], start: usize) -> String { + let mut depth = 0i32; + let mut opened = false; + let mut body = Vec::new(); + for line in &lines[start + 1..] { + body.push(*line); + depth += line.matches('{').count() as i32; + depth -= line.matches('}').count() as i32; + if line.contains('{') { + opened = true; + } + if opened && depth <= 0 { + break; + } + } + body.join("\n") +} + +fn fn_name(body: &str) -> Option { + let (_, after) = body.split_once("fn ")?; + let name: String = after.chars().take_while(|c| c.is_alphanumeric() || *c == '_').collect(); + (!name.is_empty()).then_some(name) +} + +/// Every `#[tokio::test]` body in the tree that reaches for `tokio::spawn`, +/// keyed by path relative to `tests/` so the entries read like the allowlist. +fn current_thread_spawners() -> BTreeSet<(String, String)> { + let root = tests_root(); + let mut sources = Vec::new(); + rust_sources(&root, &mut sources); + sources.sort(); + + let mut found = BTreeSet::new(); + for path in sources { + let source = std::fs::read_to_string(&path).expect("a readable test source"); + let lines: Vec<&str> = source.lines().collect(); + let relative = path + .strip_prefix(&root) + .expect("every source is under tests/") + .to_string_lossy() + .replace('\\', "/"); + + for (index, line) in lines.iter().enumerate() { + // Exactly the bare attribute. Anything carrying arguments has + // already said what runtime it wants. + if line.trim() != "#[tokio::test]" { + continue; + } + let body = body_after(&lines, index); + if !body.contains("tokio::spawn") { + continue; + } + if let Some(name) = fn_name(&body) { + found.insert((relative.clone(), name)); + } + } + } + found +} + +#[test] +fn no_new_test_spawns_tokio_tasks_from_a_current_thread_runtime() { + let found = current_thread_spawners(); + let known: BTreeSet<(String, String)> = KNOWN_CURRENT_THREAD_SPAWNERS + .iter() + .map(|(file, name)| ((*file).to_owned(), (*name).to_owned())) + .collect(); + + let new: Vec<_> = found.difference(&known).collect(); + assert!( + new.is_empty(), + "these tests spawn tokio tasks on a current-thread runtime, so their tasks \ + never actually overlap and the concurrency they claim to test is not tested: \ + {new:#?}\nUse #[tokio::test(flavor = \"multi_thread\", worker_threads = 4)]." + ); + + let fixed: Vec<_> = known.difference(&found).collect(); + assert!( + fixed.is_empty(), + "these entries no longer spawn from a current-thread runtime, so remove them \ + from KNOWN_CURRENT_THREAD_SPAWNERS: {fixed:#?}" + ); +} diff --git a/tests/worktable/mutation_gate_deadlock.rs b/tests/worktable/mutation_gate_deadlock.rs index aa21af2c..0d716367 100644 --- a/tests/worktable/mutation_gate_deadlock.rs +++ b/tests/worktable/mutation_gate_deadlock.rs @@ -95,7 +95,7 @@ fn concurrent_same_stripe_updates_do_not_deadlock() { ta.await.unwrap(); tb.await.unwrap(); }; - timeout(Duration::from_secs(20), joined) + timeout(Duration::from_secs(5), joined) .await .expect("same-stripe concurrent updates deadlocked (gate held across .await)"); @@ -137,7 +137,7 @@ fn many_same_stripe_updates_do_not_starve_worker_pool() { h.await.unwrap(); } }; - timeout(Duration::from_secs(30), joined) + timeout(Duration::from_secs(5), joined) .await .expect("same-stripe pool starved (gate spin held across .await)"); }); diff --git a/tests/worktable/nonunique_arctic.rs b/tests/worktable/nonunique_arctic.rs index bb9505ec..0267b85d 100644 --- a/tests/worktable/nonunique_arctic.rs +++ b/tests/worktable/nonunique_arctic.rs @@ -165,7 +165,7 @@ async fn concurrent_inserts_and_deletes_keep_the_index_consistent() { for n in 0..per_writer { let edge = (writer as u128) << 64 | n as u128; let source = (n as u128) % keys; - futures::executor::block_on(table.insert(row(&table, source, edge, n))).unwrap(); + nagoya::block_on(table.insert(row(&table, source, edge, n))).unwrap(); // Interleave point reads to race the writers. let _ = table.select_by_source_hash(source).execute().unwrap(); } @@ -192,7 +192,7 @@ async fn concurrent_inserts_and_deletes_keep_the_index_consistent() { } } -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn concurrent_deletes_leave_no_stale_links() { let table = Arc::new(ArcticAdjacencyWorkTable::default()); let mut pks = Vec::new(); @@ -397,7 +397,7 @@ mod persisted { } } - #[tokio::test] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn non_unique_arctic_recovers_concurrent_shared_key_writes() { use tokio::sync::Barrier; diff --git a/tests/worktable/partitioned.rs b/tests/worktable/partitioned.rs index 56df81c2..6522999d 100644 --- a/tests/worktable/partitioned.rs +++ b/tests/worktable/partitioned.rs @@ -6,6 +6,7 @@ use worktable::worktable; worktable!( name: Price, partition_by: symbol_id: u16, + partition_max_size: u64, columns: { exchange_id: u8 primary_key, bid: f64, @@ -19,6 +20,7 @@ worktable!( name: Quote, persist: false, partition_by: venue: u32, + partition_max_size: u64, columns: { id: u64 primary_key autoincrement, tag: u32, @@ -101,7 +103,7 @@ async fn insert_with_a_custom_initialiser_runs_once_per_key() { .partition_or_insert_with(11, || { let t = PriceWorkTable::default(); for e in 0..3u8 { - futures::executor::block_on(t.insert(row(e, e as f64))).unwrap(); + nagoya::block_on(t.insert(row(e, e as f64))).unwrap(); } t }) @@ -177,7 +179,7 @@ fn concurrent_creation_and_reading_is_sound() { let table = prices.partition_or_create(k).unwrap(); // Every thread writes the same row for a key, so whichever // wins the insert the value must match the key. - let _ = futures::executor::block_on(table.insert(row(0, k as f64))); + let _ = nagoya::block_on(table.insert(row(0, k as f64))); let got = prices.partition(k).unwrap().select(0).unwrap(); assert_eq!(got.bid, k as f64, "thread {t} saw a torn partition at {k}"); } @@ -426,7 +428,7 @@ async fn concurrent_writers_on_disjoint_partitions_do_not_interfere() { std::thread::spawn(move || { let table = prices.partition_or_create(t).unwrap(); for e in 0..ROWS { - futures::executor::block_on(table.insert(row(e, t as f64 * 1000.0 + e as f64))).unwrap(); + nagoya::block_on(table.insert(row(e, t as f64 * 1000.0 + e as f64))).unwrap(); } }) }) @@ -490,7 +492,7 @@ async fn readers_survive_partitions_being_removed_under_them() { let t = prices .partition_or_insert_with(k, || { let t = PriceWorkTable::default(); - futures::executor::block_on(t.insert(row(0, k as f64))).unwrap(); + nagoya::block_on(t.insert(row(0, k as f64))).unwrap(); t }) .unwrap(); @@ -509,7 +511,7 @@ async fn readers_survive_partitions_being_removed_under_them() { // Reclamation happened through the shared `Arc` while readers were // running; drain whatever grace period is still open the same way. - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); while prices.retired_len() > 0 && std::time::Instant::now() < deadline { prices.collect(); } @@ -695,3 +697,366 @@ async fn pinned_scopes_work_from_several_threads_at_once() { r.join().unwrap(); } } + +// A `Vec`-backed table is a legal partition payload. +// +// This was refused, on the grounds that "`vec: true` is one contiguous `Vec` +// and has nothing to partition". That reads the relationship backwards. +// Partitioning is what makes the `Vec` shape correct: a `Vec` table is +// single-writer and grows linearly, and cutting the data into many small +// independent ones is exactly how you keep both of those from mattering. +worktable!( + name: Book, + vec: true, + partition_by: symbol_id: u16, + partition_max_size: u64, + columns: { + exchange_id: u8 primary_key, + bid: f64, + ask: f64 + } +); + +#[test] +fn a_vec_table_can_be_partitioned() { + let books = BookPartitions::new(); + + // `insert` on a `vec: true` table takes `&mut self`, and the router hands + // out `Arc`, so a partition is populated before it is handed over rather + // than after. That is the shape the callers wanting this already have: + // every row of a book is known when the book is created. + for symbol in 0u16..4 { + let mut book = BookWorkTable::with_capacity(3); + for exchange_id in 0u8..3 { + book.insert(BookRow { + exchange_id, + bid: f64::from(symbol) + f64::from(exchange_id) / 10.0, + ask: 0.0, + }) + .expect("fresh key"); + } + books + .partition_or_insert_with(symbol, move || book) + .expect("a fresh partition"); + } + + assert_eq!(books.len(), 4); + + let book = books.partition(2).expect("declared above"); + assert_eq!(book.len(), 3); + assert_eq!(book.select(&1).expect("present").bid, 2.1); + + // The keys are per partition, not global: every book has an exchange 0. + for symbol in 0u16..4 { + let book = books.partition(symbol).expect("declared above"); + assert!(book.select(&0).is_some(), "symbol {symbol} has no exchange 0"); + } + + // `used_bytes` is what the router totals, so a Vec payload has to answer + // it. Rows alone are 3 * size_of::() per partition, and the index + // is on top, so the total must exceed the rows and be finite. + let rows_only = 4 * 3 * core::mem::size_of::() as u64; + let total = books.memory_total(); + assert!(total > rows_only, "{total} should exceed the {rows_only} bytes of rows"); + + let by_key = books.memory_by_key(); + assert_eq!(by_key.len(), 4); + assert_eq!(by_key.iter().map(|(_, bytes)| bytes).sum::(), total); +} + +// A narrow `partition_max_size` generates a table with no index at all. +// +// This is the shape the key exists to make declarable: `exchange_id: u8` is not +// looked up, it *is* the row's position, so there is no tree to descend and +// nothing to hash. The router is unchanged; only its payload is. +worktable!( + name: Tick, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { + exchange_id: u8 primary_key, + bid: f64, + ask: f64 + } +); + +#[test] +fn a_narrow_width_generates_a_dense_payload() { + let ticks = TickPartitions::new(); + + // `&self`, straight through the `Arc` the router hands out. This is the + // difference from a `vec: true` payload, whose `insert` needs `&mut self` + // and so has to be populated before it is handed over. + let book = ticks.partition_or_create(7).expect("a fresh partition"); + for exchange_id in 0u8..23 { + book.insert(TickRow { + exchange_id, + bid: f64::from(exchange_id), + ask: f64::from(exchange_id) + 1.0, + }) + .expect("fresh key"); + } + + assert_eq!(book.row_count(), 23); + assert_eq!(book.select(&11).expect("present").bid, 11.0); + assert_eq!(book.slots(), 23, "grown to the keys used, not to the declared 256"); + assert_eq!(TickDenseTable::MAX_ROWS, 256); +} + +#[test] +fn the_declared_width_is_a_bound_at_run_time_too() { + let ticks = TickPartitions::new(); + let book = ticks.partition_or_create(1).expect("a fresh partition"); + + // `exchange_id: u8` counts to 255 and the cap is 256, so nothing a `u8` can + // hold is out of range. What the cap does reject is a duplicate. + book.insert(TickRow { + exchange_id: 3, + bid: 1.0, + ask: 2.0, + }) + .expect("fresh key"); + let again = book + .insert(TickRow { + exchange_id: 3, + bid: 9.0, + ask: 9.0, + }) + .expect_err("3 is taken"); + assert_eq!(again, DenseError::Duplicate { key: 3 }); + assert_eq!( + book.select(&3).expect("present").bid, + 1.0, + "the refusal changed nothing" + ); +} + +#[test] +fn a_column_is_updated_without_cloning_the_row() { + // The method web3.trading's `update_top_price` wants: touch one field of a + // wide row rather than reading it out, editing it and writing it back. + let ticks = TickPartitions::new(); + let book = ticks.partition_or_create(2).expect("a fresh partition"); + book.insert(TickRow { + exchange_id: 4, + bid: 1.0, + ask: 2.0, + }) + .expect("fresh key"); + + assert_eq!(book.update_bid(&4, 1.5), Some(1.0)); + assert_eq!(book.select(&4).expect("present").bid, 1.5); + assert_eq!( + book.select(&4).expect("present").ask, + 2.0, + "the other column is untouched" + ); + + assert_eq!(book.update_bid(&5, 1.0), None, "a key holding no row updates nothing"); +} + +#[test] +fn a_dense_partition_costs_its_rows_and_nothing_else() { + // The measurement the whole shape exists for. A full partition of this + // declaration measured 28,395 bytes empty; this one must be its rows. + let ticks = TickPartitions::new(); + for symbol in 0u16..4 { + let book = ticks.partition_or_create(symbol).expect("a fresh partition"); + for exchange_id in 0u8..23 { + book.insert(TickRow { + exchange_id, + bid: 0.0, + ask: 0.0, + }) + .expect("fresh key"); + } + } + + let rows = 4 * 23 * core::mem::size_of::>() as u64; + assert_eq!(ticks.memory_total(), rows, "there is nothing else to count"); + assert_eq!(ticks.rows_by_key(), (0u16..4).map(|k| (k, 23)).collect::>()); +} + +#[test] +fn deleting_does_not_renumber_the_rows_above_it() { + let ticks = TickPartitions::new(); + let book = ticks.partition_or_create(3).expect("a fresh partition"); + for exchange_id in 0u8..4 { + book.insert(TickRow { + exchange_id, + bid: f64::from(exchange_id), + ask: 0.0, + }) + .expect("fresh key"); + } + + assert_eq!(book.delete(&1).expect("present").bid, 1.0); + assert_eq!(book.select(&1), None); + assert_eq!(book.select(&2).expect("present").bid, 2.0, "key 2 did not become key 1"); + assert_eq!(book.row_count(), 3); + assert_eq!(book.select_all().len(), 3, "select_all skips the hole"); +} + +// The same columns as `Tick`, with the width that keeps the full table, so the +// two shapes can be measured against each other rather than against a +// recollection. +worktable!( + name: FatTick, + partition_by: symbol_id: u16, + partition_max_size: u64, + columns: { + exchange_id: u8 primary_key, + bid: f64, + ask: f64 + } +); + +/// `memory_total` cannot see what the width is worth, and that is worth a test. +/// +/// `used_bytes` is row bytes plus index bytes by definition: it excludes the +/// table's fixed floor, its reserved-but-unused page capacity, the router spine +/// and `Arc` overhead. The fixed floor is precisely what a dense partition +/// deletes, so the router's own reporting shows the two shapes as equal while +/// one of them holds 28 KB per partition that the other does not. +/// +/// The real comparison is in `tests/dense_partition_memory.rs`, which counts +/// what the allocator was actually asked for. This test exists so nobody +/// reaches for `memory_total` to make the claim and concludes the feature does +/// nothing. +#[tokio::test] +async fn memory_total_reports_rows_and_cannot_see_the_apparatus() { + const ROWS: u8 = 23; + + let dense = TickPartitions::new(); + let book = dense.partition_or_create(0).expect("a fresh partition"); + for exchange_id in 0..ROWS { + book.insert(TickRow { + exchange_id, + bid: 0.0, + ask: 0.0, + }) + .expect("fresh key"); + } + + let full = FatTickPartitions::new(); + let fat = full.partition_or_create(0).expect("a fresh partition"); + for exchange_id in 0..ROWS { + fat.insert(FatTickRow { + exchange_id, + bid: 0.0, + ask: 0.0, + }) + .await + .expect("fresh key"); + } + + let payload = u64::from(ROWS) * core::mem::size_of::>() as u64; + assert_eq!( + dense.memory_total(), + payload, + "a dense partition is its rows, and `used_bytes` sees all of it" + ); + assert_eq!( + full.memory_total(), + dense.memory_total(), + "the two shapes report the same used bytes, because the difference between them is \ + entirely in what `used_bytes` excludes. If this ever differs, the definition changed \ + and the note above needs rewriting." + ); + + // Both hold the same rows. The saving is apparatus, not data. + assert_eq!(dense.rows_by_key(), full.rows_by_key()); +} + +/// An empty partition is where the cost lived, so it is where to look. +#[test] +fn an_empty_dense_partition_allocates_nothing() { + let dense = TickPartitions::new(); + dense.partition_or_create(0).expect("a fresh partition"); + assert_eq!(dense.memory_total(), 0, "nothing is allocated until a row arrives"); + assert_eq!( + dense.partition(0).expect("created above").slots(), + 0, + "and no slots either: the declared width is a bound, not a reservation" + ); +} + +// A dense partition carries `queries:`, keyed by position. +// +// This is what decides whether the shape is adoptable: web3.trading's +// `update_top_price` and `update_full` go through declared update queries, and +// a payload that could not carry them would be a payload they cannot use. +worktable!( + name: Quoted, + partition_by: symbol_id: u16, + partition_max_size: u8, + columns: { + exchange_id: u8 primary_key, + bid: f64, + ask: f64, + seq: u64 + }, + queries: { + update: { + TopPrice(bid, ask) by exchange_id, + }, + delete: { + Stale() by exchange_id, + } + } +); + +#[test] +fn a_dense_partition_carries_its_update_queries() { + let quotes = QuotedPartitions::new(); + let book = quotes.partition_or_create(0).expect("a fresh partition"); + book.insert(QuotedRow { + exchange_id: 2, + bid: 1.0, + ask: 2.0, + seq: 7, + }) + .expect("fresh key"); + + // The same method name and the same query struct the paged table generates, + // so the call reads the same. What differs is that there is no `.await`. + assert_eq!( + book.update_top_price(TopPriceQuery { bid: 9.0, ask: 10.0 }, &2), + Some(()) + ); + + let row = book.select(&2).expect("present"); + assert_eq!((row.bid, row.ask), (9.0, 10.0)); + assert_eq!(row.seq, 7, "a column the query does not name is untouched"); + + assert_eq!( + book.update_top_price(TopPriceQuery { bid: 0.0, ask: 0.0 }, &3), + None, + "a key holding no row updates nothing" + ); +} + +#[test] +fn a_dense_partition_carries_its_delete_queries() { + let quotes = QuotedPartitions::new(); + let book = quotes.partition_or_create(0).expect("a fresh partition"); + book.insert(QuotedRow { + exchange_id: 1, + bid: 1.0, + ask: 2.0, + seq: 1, + }) + .expect("fresh key"); + book.insert(QuotedRow { + exchange_id: 2, + bid: 3.0, + ask: 4.0, + seq: 2, + }) + .expect("fresh key"); + + assert_eq!(book.delete_stale(&1).expect("present").seq, 1); + assert_eq!(book.select(&1), None); + assert_eq!(book.select(&2).expect("present").seq, 2, "key 2 did not move"); + assert_eq!(book.delete_stale(&1), None, "deleting twice is not an error"); +} diff --git a/tests/worktable/runtime_backends.rs b/tests/worktable/runtime_backends.rs new file mode 100644 index 00000000..61c6bc95 --- /dev/null +++ b/tests/worktable/runtime_backends.rs @@ -0,0 +1,501 @@ +//! What "this runtime backend is supported" is allowed to mean. +//! +//! The support policy is one sentence: every backend listed compiles, the +//! library and integration suites pass against it, and a persisted table +//! survives open, write, close and reload. Performance belongs to the backend. +//! This file is the half of that sentence a test can hold. It runs one body +//! against `nagoya(locality)`, `nagoya(spread)`, `nagoya(throughput)` and +//! `tokio`, the same way `base_backend_suite!` and `vacuum_backend_suite!` run +//! one body across the index backends. +//! +//! The body covers four things, and each one is here because leaving it out +//! would let a broken backend pass: +//! +//! 1. `insert` / `select` / `update` / `delete` / `in_place`, so a backend that +//! compiles but cannot drive a mutation is caught. +//! 2. A persisted table opened, written, closed and reloaded, so a backend +//! whose spawn or timer never reaches the persistence worker is caught. +//! 3. Concurrent tasks, so the backend's sync primitives are exercised rather +//! than merely named. See the note on runtime flavors below: this is the +//! part that is easiest to write and hardest to write honestly. +//! 4. `wait_for_ops` and `close` bounded by a timeout. A shutdown that does not +//! flush tears the data file, that has happened here before, and swapping +//! the runtime under the persistence worker is exactly the change that would +//! bring it back. A hang is reported as a failed assertion, not as a test +//! that never returns. +//! +//! ## The runtime the harness runs on is not the runtime under test +//! +//! `#[tokio::test]` with no arguments builds a **current-thread** runtime. Tasks +//! spawned inside it interleave only at await points on one thread, so two +//! writers never actually overlap and a data race between them cannot be +//! observed. Any test here that means to exercise concurrency therefore says +//! `flavor = "multi_thread"` explicitly. +//! +//! This is not a hypothetical. Commit 2702c06 on this branch records two tests +//! that shared one table directory and only began failing once the persistence +//! worker moved off `tokio::spawn`: while the worker ran on the test's own +//! current-thread runtime, the two tables' writes never overlapped in time and +//! the corruption stayed hidden. The single-threaded harness was concealing a +//! real defect. +//! +//! Note that the harness runtime and the table's declared runtime are separate +//! things. `tokio::spawn` below drives the *test*; the table drives its own +//! internals on whatever `runtime:` selected. Once the four-backend arms are +//! live, that separation is what lets one body test four backends. +//! +//! ## Directories +//! +//! Every arm gets its own data directory and every test within an arm gets its +//! own subdirectory below that, both derived from the arm's label. Two tests +//! sharing one table directory is the defect 2702c06 fixed: the harness runs +//! tests on parallel threads, so two tables attached to one set of files and +//! filled them at once, and each table's event ids start at zero, so one saw +//! the other's events as corruption. This suite multiplies every test by four +//! backends, which would make that a four-way collision. Deriving the path from +//! the label also means a failure names the arm that failed. +//! +//! ## Status +//! +//! The four-backend arms are behind the `runtime-backends` feature, off by +//! default, because the DSL `runtime:` keyword and the `Runtime` trait are +//! landing separately. The `hardcoded_default` arm has no `runtime:` at all and +//! runs today against the runtime the engine currently hardcodes. That arm is +//! the pre-merge baseline: it proves the body is correct before the body is +//! asked to tell four backends apart. + +/// One arm of the matrix. +/// +/// `$label` names the arm and supplies its data directory, so a failure says +/// which backend failed. The runtime spec is optional and, when present, is +/// re-emitted verbatim into the table declaration. Omitting it is not the same +/// as writing `runtime: nagoya`: it declares nothing, which is what the arm +/// that runs today needs. +macro_rules! runtime_backend_suite { + ($module:ident, $label:literal $(, runtime: $backend:tt $(($flavor:tt))?)?) => { + mod $module { + use std::collections::BTreeSet; + use std::sync::Arc; + use std::time::Duration; + + // The watchdog is deliberately the harness's clock, not the + // table's. A backend whose own timers are broken must not be able + // to break the timeout that is supposed to catch it, so this stays + // `tokio::time` even on the nagoya arms and is aliased so it is + // not confused with `worktable::prelude::timeout`. + use tokio::time::timeout as harness_timeout; + use worktable::prelude::PersistedWorkTable; + use worktable::prelude::*; + use worktable::worktable; + + use crate::remove_dir_if_exists; + + // The in-memory table. Carries an indexed column so `update` and + // `delete` have index maintenance to do, and a plain one so + // `in_place` has somewhere to write that no index watches. + worktable!( + name: RuntimeMatrix, + persist: false, + $(runtime: $backend $(($flavor))?,)? + columns: { + id: u64 primary_key autoincrement, + counter: u64, + bucket: u64, + note: String, + }, + indexes: { + bucket_idx: bucket, + }, + queries: { + update: { + BucketById(bucket) by id, + }, + delete: { + ByBucket() by bucket, + }, + in_place: { + CounterById(counter) by id, + } + } + ); + + // The persisted table. Same shape, no autoincrement, because a + // reload has to compare against keys the test chose rather than + // keys a generator handed out. Its queries carry a `Persist` + // prefix because `worktable!` puts the generated query types at + // module scope, so two tables in one module cannot share a query + // name. + worktable!( + name: RuntimeMatrixPersist, + persist: true, + $(runtime: $backend $(($flavor))?,)? + columns: { + id: u64 primary_key, + counter: u64, + bucket: u64, + }, + indexes: { + bucket_idx: bucket, + }, + queries: { + update: { + PersistBucketById(bucket) by id, + }, + in_place: { + PersistCounterById(counter) by id, + } + } + ); + + /// Names this arm. Used for the data directory, so a torn store + /// says which backend tore it. + const LABEL: &str = $label; + + /// Bounds every drain and shutdown in this file. A backend whose + /// `close` never returns must fail the test, not stall the suite + /// until CI's own timeout kills the run with no attribution. + const SHUTDOWN_BUDGET: Duration = Duration::from_secs(5); + + /// Concurrent writers. Four is enough to have two of them actually + /// running at once on the four-worker harness runtime, and small + /// enough that four arms of this suite stay cheap. + const WRITERS: u64 = 4; + + /// Rows each writer inserts. + const PER_WRITER: u64 = 250; + + /// One directory per test per arm. Never share. + fn data_dir(test: &str) -> String { + format!("tests/data/runtime_backends/{LABEL}/{test}") + } + + /// Bytes the arm actually put on disk. A reload that "survived" + /// without the store growing would mean the assertions below were + /// reading something other than the file, so this is the check + /// that keeps the persistence tests honest. + fn data_file_len(dir: &str) -> u64 { + let path = format!( + "{dir}/{}/{WT_DATA_EXTENSION}", + RuntimeMatrixPersistWorkTable::name_snake_case() + ); + std::fs::metadata(&path) + .unwrap_or_else(|error| panic!("{LABEL}: no store at {path}: {error}")) + .len() + } + + fn config(dir: &str) -> DiskConfig { + DiskConfig::new_with_table_name( + dir, + RuntimeMatrixPersistWorkTable::name_snake_case(), + RuntimeMatrixPersistWorkTable::version(), + ) + } + + async fn open(dir: &str) -> RuntimeMatrixPersistWorkTable { + let engine = RuntimeMatrixPersistPersistenceEngine::new(config(dir)).await.unwrap(); + RuntimeMatrixPersistWorkTable::load(engine).await.unwrap() + } + + fn row(id: u64) -> RuntimeMatrixPersistRow { + RuntimeMatrixPersistRow { + id, + counter: id * 10, + bucket: id % 4, + } + } + + /// Every mutation the table exposes, in one pass, on the runtime + /// the arm declares. The point is coverage of the surface rather + /// than depth in any one operation: a runtime that cannot drive a + /// delete is not supported, whatever else it does well. + #[tokio::test] + async fn every_mutation_runs() { + let table = RuntimeMatrixWorkTable::default(); + + // Raw keys rather than the returned primary-key newtype: the + // newtype is not `Copy`, and every assertion below reuses the + // same key more than once. + let mut ids: Vec = Vec::new(); + for i in 0..16u64 { + let id: u64 = table.get_next_pk().into(); + table + .insert(RuntimeMatrixRow { + id, + counter: 0, + bucket: i % 4, + note: format!("{LABEL}-{i}"), + }) + .await + .unwrap(); + ids.push(id); + } + + // select + let first = table.select(ids[0]).expect("the inserted row must be selectable"); + assert_eq!(first.bucket, 0); + assert_eq!(first.counter, 0); + + // select through a secondary index + let bucket_zero = table.select_by_bucket(0).execute().unwrap(); + assert_eq!(bucket_zero.len(), 4, "{LABEL}: four of sixteen rows are in bucket 0"); + + // update, which also has to move the row between index buckets + table + .update_bucket_by_id(BucketByIdQuery { bucket: 3 }, ids[0]) + .await + .unwrap(); + assert_eq!(table.select(ids[0]).unwrap().bucket, 3); + assert_eq!( + table.select_by_bucket(0).execute().unwrap().len(), + 3, + "{LABEL}: the update must leave the old index bucket" + ); + + // in_place, which mutates the page bytes rather than + // republishing the row + for _ in 0..64 { + table + .update_counter_by_id_in_place(|counter| *counter += 1u64, ids[1]) + .await + .unwrap(); + } + assert_eq!(table.select(ids[1]).unwrap().counter, 64); + + // delete, by a secondary index rather than by primary key + table.delete_by_bucket(1).await.unwrap(); + assert!( + table.select_by_bucket(1).execute().unwrap().is_empty(), + "{LABEL}: bucket 1 must be empty after the delete" + ); + + // and delete by primary key + table.delete(ids[0]).await.unwrap(); + assert!(table.select(ids[0]).is_none()); + } + + /// Writers and readers at once, on a genuinely multi-threaded + /// harness. Under the default `#[tokio::test]` this test would + /// pass without two tasks ever overlapping, which is to say it + /// would prove nothing about the backend's sync primitives. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_tasks_reach_a_consistent_table() { + let table = Arc::new(RuntimeMatrixWorkTable::default()); + + let mut writers = Vec::new(); + for writer in 0..WRITERS { + let table = table.clone(); + writers.push(tokio::spawn(async move { + for i in 0..PER_WRITER { + table + .insert(RuntimeMatrixRow { + id: table.get_next_pk().into(), + counter: writer, + bucket: i % 4, + note: format!("{LABEL}-{writer}-{i}"), + }) + .await + .unwrap(); + } + })); + } + + // Readers run beside the writers rather than after them. A + // select that observes a half-published row is the failure + // this is looking for, and it cannot happen once the writers + // have joined. + let mut readers = Vec::new(); + for _ in 0..2 { + let table = table.clone(); + readers.push(tokio::spawn(async move { + for _ in 0..PER_WRITER { + let rows = table.select_by_bucket(0).execute().unwrap(); + for row in rows { + assert!( + row.counter < WRITERS, + "{LABEL}: a concurrent select observed a row that no writer wrote" + ); + } + worktable::prelude::yield_now().await; + } + })); + } + + for writer in writers { + writer.await.unwrap(); + } + for reader in readers { + reader.await.unwrap(); + } + + let expected = WRITERS * PER_WRITER; + let ids: BTreeSet<_> = table.select_all().execute().unwrap().into_iter().map(|r| r.id).collect(); + assert_eq!( + ids.len() as u64, + expected, + "{LABEL}: {expected} concurrent inserts must produce {expected} distinct rows" + ); + } + + /// Open, write, close, reload, and the data is still there. This is + /// the clause of the support policy that a backend cannot fake: + /// the persistence worker has to have been spawned, driven and + /// drained on the declared runtime. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_persisted_table_survives_a_reload() { + let dir = data_dir("survives_reload"); + remove_dir_if_exists(dir.clone()).await; + + { + let table = open(&dir).await; + for id in 1..=64u64 { + table.insert(row(id)).await.unwrap(); + } + table + .update_persist_bucket_by_id(PersistBucketByIdQuery { bucket: 3 }, 1) + .await + .unwrap(); + table + .update_persist_counter_by_id_in_place(|counter| *counter = 4_242u64.into(), 2) + .await + .unwrap(); + + harness_timeout(SHUTDOWN_BUDGET, table.wait_for_ops()) + .await + .expect("wait_for_ops must not hang") + .unwrap(); + harness_timeout(SHUTDOWN_BUDGET, table.close()) + .await + .expect("close must not hang") + .unwrap(); + } + + assert!( + data_file_len(&dir) > 0, + "{LABEL}: the closed table left an empty store" + ); + + { + let table = open(&dir).await; + for id in 1..=64u64 { + let reloaded = table + .select(id) + .unwrap_or_else(|| panic!("{LABEL}: row {id} did not survive the reload")); + let expected_bucket = if id == 1 { 3 } else { row(id).bucket }; + assert_eq!(reloaded.bucket, expected_bucket, "{LABEL}: row {id} reloaded wrong"); + let expected_counter = if id == 2 { 4_242 } else { row(id).counter }; + assert_eq!( + reloaded.counter, expected_counter, + "{LABEL}: row {id} lost its counter across the reload" + ); + } + + // The secondary index has to come back too, not just the + // rows: a reload that rebuilt the data and dropped the + // index would pass every check above. + let bucket_three: BTreeSet<_> = table + .select_by_bucket(3) + .execute() + .unwrap() + .into_iter() + .map(|r| r.id) + .collect(); + let expected: BTreeSet<_> = (1..=64u64).filter(|id| *id == 1 || id % 4 == 3).collect(); + assert_eq!(bucket_three, expected, "{LABEL}: the secondary index did not survive"); + + harness_timeout(SHUTDOWN_BUDGET, table.close()) + .await + .expect("close must not hang") + .unwrap(); + } + + remove_dir_if_exists(dir).await; + } + + /// Concurrent writers into a persisted table, then a drain and a + /// shutdown, then a reload that has to find every row. + /// + /// This is the one that matters. A shutdown that returns before the + /// persistence worker has flushed leaves a torn `.wt.data`, that + /// has happened in this repo, and moving the worker onto a + /// different runtime is precisely the change that could reintroduce + /// it. Concurrency is here rather than in a separate test because a + /// single-writer drain is the case that works even when the flush + /// is broken. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_concurrent_shutdown_flushes_rather_than_hangs() { + let dir = data_dir("concurrent_shutdown"); + remove_dir_if_exists(dir.clone()).await; + + let written = WRITERS * PER_WRITER; + { + let table = Arc::new(open(&dir).await); + + let mut writers = Vec::new(); + for writer in 0..WRITERS { + let table = table.clone(); + writers.push(tokio::spawn(async move { + for i in 0..PER_WRITER { + table.insert(row(writer * PER_WRITER + i + 1)).await.unwrap(); + } + })); + } + for writer in writers { + writer.await.unwrap(); + } + + harness_timeout(SHUTDOWN_BUDGET, table.wait_for_ops()) + .await + .expect("wait_for_ops must not hang after concurrent writes") + .unwrap(); + + // `close` consumes the table, so the writers' clones have + // to be gone first. If they are not, that is a leaked + // handle and worth failing on rather than working around. + let table = Arc::try_unwrap(table) + .unwrap_or_else(|arc| panic!("{LABEL}: {} table lease(s) outlived the writers", Arc::strong_count(&arc) - 1)); + harness_timeout(SHUTDOWN_BUDGET, table.close()) + .await + .expect("close must not hang after concurrent writes") + .unwrap(); + } + + assert!( + data_file_len(&dir) > 0, + "{LABEL}: the shutdown left an empty store" + ); + + { + let table = open(&dir).await; + let ids: BTreeSet<_> = table.select_all().execute().unwrap().into_iter().map(|r| r.id).collect(); + let expected: BTreeSet<_> = (1..=written).collect(); + assert_eq!( + ids, expected, + "{LABEL}: the shutdown did not flush every concurrent write" + ); + harness_timeout(SHUTDOWN_BUDGET, table.close()) + .await + .expect("close must not hang") + .unwrap(); + } + + remove_dir_if_exists(dir).await; + } + } + }; +} + +// The arm that runs today. No `runtime:` is declared, so the table takes the +// runtime the engine currently hardcodes. This is the pre-merge baseline: it +// proves the body before the body is asked to discriminate between backends. +runtime_backend_suite!(hardcoded_default, "hardcoded_default"); + +// The matrix. Off by default until the DSL `runtime:` keyword and the `Runtime` +// trait land; `cargo test --features runtime-backends` is what turns it on. +#[cfg(feature = "runtime-backends")] +runtime_backend_suite!(nagoya_locality, "nagoya_locality", runtime: nagoya(locality)); +#[cfg(feature = "runtime-backends")] +runtime_backend_suite!(nagoya_spread, "nagoya_spread", runtime: nagoya(spread)); +#[cfg(feature = "runtime-backends")] +runtime_backend_suite!(nagoya_throughput, "nagoya_throughput", runtime: nagoya(throughput)); +#[cfg(feature = "runtime-backends")] +runtime_backend_suite!(tokio_rt, "tokio", runtime: tokio); diff --git a/tests/worktable/unsized_.rs b/tests/worktable/unsized_.rs index f09186bc..80f8faa5 100644 --- a/tests/worktable/unsized_.rs +++ b/tests/worktable/unsized_.rs @@ -303,7 +303,7 @@ async fn update_parallel() { } h.await.unwrap(); - for (test, val) in i_state.lock_arc().iter() { + for (test, val) in i_state.lock().iter() { let row = table.select_by_test(*test).unwrap(); assert_eq!(&row.exchange, val) } @@ -621,11 +621,11 @@ async fn update_parallel_more_strings() { } h.await.unwrap(); - for (id, e) in e_state.lock_arc().iter() { + for (id, e) in e_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.exchange, e) } - for (id, s) in s_state.lock_arc().iter() { + for (id, s) in s_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.some_string, s) } @@ -711,15 +711,15 @@ async fn update_parallel_more_strings_more_threads() { h1.await.unwrap(); h2.await.unwrap(); - for (id, e) in e_state.lock_arc().iter() { + for (id, e) in e_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.exchange, e) } - for (id, s) in s_state.lock_arc().iter() { + for (id, s) in s_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.some_string, s) } - for (id, a) in a_state.lock_arc().iter() { + for (id, a) in a_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.another, a) } @@ -793,11 +793,11 @@ async fn update_parallel_more_strings_with_select_non_unique() { h1.await.unwrap(); h2.await.unwrap(); - for (id, e) in e_state.lock_arc().iter() { + for (id, e) in e_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.exchange, e) } - for (id, a) in a_state.lock_arc().iter() { + for (id, a) in a_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.another, a) } @@ -858,7 +858,7 @@ async fn delete_parallel() { h1.await.unwrap(); h2.await.unwrap(); - for id in deleted_state.lock_arc().iter() { + for id in deleted_state.lock().iter() { let row = table.select(*id); assert!(row.is_none()) } @@ -930,7 +930,7 @@ async fn update_parallel_more_strings_with_select_unique() { h1.await.unwrap(); h2.await.unwrap(); - for (id, e) in e_state.lock_arc().iter() { + for (id, e) in e_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.exchange, e) } @@ -986,7 +986,7 @@ async fn upsert_parallel() { } h1.await.unwrap(); - for (id, e) in e_state.lock_arc().iter() { + for (id, e) in e_state.lock().iter() { let row = table.select(*id).unwrap(); assert_eq!(&row.exchange, e) } diff --git a/tests/worktable/update_delete_race.rs b/tests/worktable/update_delete_race.rs index e34afbfd..7c05ccab 100644 --- a/tests/worktable/update_delete_race.rs +++ b/tests/worktable/update_delete_race.rs @@ -77,7 +77,7 @@ async fn concurrent_update_and_delete_never_panics() { }) }; - tokio::time::timeout(Duration::from_secs(60), async { + tokio::time::timeout(Duration::from_secs(5), async { updater.await.expect("updater must not panic"); deleter.await.expect("deleter must not panic"); }) diff --git a/tests/worktable/upsert.rs b/tests/worktable/upsert.rs index c00f0c83..7dfe9b57 100644 --- a/tests/worktable/upsert.rs +++ b/tests/worktable/upsert.rs @@ -86,14 +86,14 @@ async fn raw_insert_delete_churn_never_panics_or_stalls() { })); } - let (insert_successes, insert_conflicts, delete_successes, delete_misses) = timeout(Duration::from_secs(60), churn) + let (insert_successes, insert_conflicts, delete_successes, delete_misses) = timeout(Duration::from_secs(5), churn) .await .expect("raw insert/delete churn starved") .unwrap(); assert_eq!(insert_successes + insert_conflicts, 5_000); assert_eq!(delete_successes + delete_misses, 5_000); for handle in upserters { - timeout(Duration::from_secs(60), handle) + timeout(Duration::from_secs(5), handle) .await .expect("upserter starved during raw insert/delete churn") .unwrap(); @@ -188,12 +188,12 @@ async fn churn_run(churn_flips: u64, upserts_per_task: u64) { })); } - timeout(Duration::from_secs(60), churn) + timeout(Duration::from_secs(5), churn) .await .expect("churn task starved") .unwrap(); for handle in upserters { - timeout(Duration::from_secs(60), handle) + timeout(Duration::from_secs(5), handle) .await .expect("upserter starved") .unwrap(); diff --git a/tests/worktable/upsert_guard.rs b/tests/worktable/upsert_guard.rs index 6eb99188..2d391ecf 100644 --- a/tests/worktable/upsert_guard.rs +++ b/tests/worktable/upsert_guard.rs @@ -66,7 +66,7 @@ fn inserting_under_a_held_mutation_gate_completes() { /// And the ordinary path still takes the gate, so a caller that is not already /// holding one is still serialised. -#[tokio::test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn upsert_still_serialises_concurrent_writers() { use std::sync::Arc; let table = Arc::new(UpsertGuardWorkTable::default()); diff --git a/tests/worktable/vacuum.rs b/tests/worktable/vacuum.rs index 1bdea8c3..e715c3c7 100644 --- a/tests/worktable/vacuum.rs +++ b/tests/worktable/vacuum.rs @@ -227,6 +227,11 @@ async fn vacuum_parallel_with_upserts() { // which is the damage; this reports which index entry points at storage // holding something else, which is the defect. { + // Pin before obtaining links, just as generated select/iterator + // callsites do. Vacuum is still running: without this guard it can + // reclaim a source page between yielding an index link and reading + // its bytes, making the oracle itself report an invalid link. + let _read_guard = table.0.data.read_guard(); let stale: Vec<_> = table .0 .indexes @@ -323,7 +328,7 @@ async fn vacuum_loop_test() { } task.await.unwrap(); - vacuum_task.abort(); + vacuum_task.cancel(); } } diff --git a/tests/worktable/vacuum_invariants.rs b/tests/worktable/vacuum_invariants.rs index 0c4c3a33..c8c791f8 100644 --- a/tests/worktable/vacuum_invariants.rs +++ b/tests/worktable/vacuum_invariants.rs @@ -203,7 +203,7 @@ macro_rules! vacuum_invariant_suite { // Let vacuum run once more against the wreckage, then stop it // so the check reads a still table. tokio::time::sleep(Duration::from_millis(60)).await; - vacuum_task.abort(); + vacuum_task.cancel(); tokio::time::sleep(Duration::from_millis(20)).await; assert_indexes_resolve_to_their_own_rows(&table, "after churn"); diff --git a/tests/worktable/vacuum_no_row_loss.rs b/tests/worktable/vacuum_no_row_loss.rs index 8b9bf9e4..c4ffc785 100644 --- a/tests/worktable/vacuum_no_row_loss.rs +++ b/tests/worktable/vacuum_no_row_loss.rs @@ -95,7 +95,7 @@ async fn vacuum_never_loses_surviving_rows() { reader.await.unwrap(); // Let vacuum run a few more cycles, then stop it and let grace periods drain. tokio::time::sleep(Duration::from_millis(200)).await; - handle.abort(); + handle.cancel(); tokio::time::sleep(Duration::from_millis(100)).await; // FULL AUDIT: every survivor must still be present and correct, by pk and diff --git a/tests/worktable/vec_table.rs b/tests/worktable/vec_table.rs new file mode 100644 index 00000000..c59e6062 --- /dev/null +++ b/tests/worktable/vec_table.rs @@ -0,0 +1,1834 @@ +//! `worktable_vec!` behaves, and costs what a `Vec` costs. +//! +//! The second half is the point. A Vec-backed table that is materially slower +//! than the `Vec` it wraps has no reason to exist: the caller would write the +//! `Vec`. So the comparison is against the thing it replaces, not against +//! `worktable!`. + +use std::collections::BTreeMap; +use std::time::Instant; + +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: Point, + vec: true, + columns: { + id: u64 primary_key, + value: u64, + tag: u64, + }, + indexes: { + tag_idx: tag, + }, +); + +#[test] +fn it_behaves_like_a_table() { + let mut table = PointWorkTable::new(); + + table + .insert(PointRow { + id: 1, + value: 10, + tag: 7, + }) + .expect("fresh"); + table + .insert(PointRow { + id: 2, + value: 20, + tag: 7, + }) + .expect("fresh"); + assert!( + table + .insert(PointRow { + id: 1, + value: 99, + tag: 9 + }) + .is_err(), + "duplicate key" + ); + + assert_eq!(table.select(&1).expect("present").value, 10); + assert_eq!(table.len(), 2); + assert_eq!(table.select_all().count(), 2); + + // A non-unique index returns every row, in insertion order. + let tagged = table.select_by_tag(&7); + assert_eq!(tagged.len(), 2); + assert_eq!(tagged[0].id, 1); + + table.upsert(PointRow { + id: 1, + value: 11, + tag: 7, + }); + assert_eq!(table.select(&1).expect("present").value, 11, "upsert replaces"); + assert_eq!(table.len(), 2, "upsert does not grow the table"); + + let removed = table.delete(&1).expect("present"); + assert_eq!(removed.value, 11); + assert_eq!(table.len(), 1); + assert!(table.select(&1).is_none()); + // The survivor keeps its position; only the dead row left the indexes. + assert_eq!(table.select(&2).expect("present").value, 20); + assert_eq!(table.select_by_tag(&7).len(), 1); +} + +/// The generated table must not be categorically slower than a plain `Vec`. +/// +/// The baseline is what an application writes when it has no table: a `Vec` of +/// rows and a `BTreeMap` from key to position. +/// +/// **This is not the parity measurement, and cannot be.** It used to be: the +/// generated table also held a `BTreeMap`, so the two were the same data +/// structures and a gap was the macro. The default backend is arctic now, so +/// the arms differ in the index as well, and the two disagree about which way. +/// Optimized, the generated table runs 0.64x the baseline. Unoptimized, which +/// is how `cargo test` runs it, it runs 1.77x, because an ART's generics are a +/// pile of uninlined calls until the optimizer sees them and `BTreeMap` suffers +/// far less. A tight bound here would encode whichever build happened to be +/// used to pick it. +/// +/// Parity is measured in `perf-benchmarks`, in `benchmarks/wt-vec-generated.rs`, +/// against `worktable-vec`'s own `ArcticTable` and `IndexedTable`, optimized +/// and interleaved. What is left here is the check that survives a debug +/// build: that the table still does a map lookup and not a linear scan. +#[test] +#[ignore = "manual timing guard; run on a quiet host with an optimized benchmark for release evidence"] +fn it_costs_what_a_vec_costs() { + const ROWS: u64 = 50_000; + + struct Baseline { + rows: Vec<(u64, u64, u64)>, + by_pk: BTreeMap, + } + + let started = Instant::now(); + let mut baseline = Baseline { + rows: Vec::new(), + by_pk: BTreeMap::new(), + }; + for id in 0..ROWS { + baseline.by_pk.insert(id, baseline.rows.len()); + baseline.rows.push((id, id * 2, id % 64)); + } + let mut sum = 0u64; + for id in 0..ROWS { + if let Some(at) = baseline.by_pk.get(&id) { + sum += baseline.rows[*at].1; + } + } + let vec_time = started.elapsed(); + + let started = Instant::now(); + let mut table = PointWorkTable::new(); + for id in 0..ROWS { + table + .insert(PointRow { + id, + value: id * 2, + tag: id % 64, + }) + .expect("fresh"); + } + let mut table_sum = 0u64; + for id in 0..ROWS { + if let Some(row) = table.select(&id) { + table_sum += row.value; + } + } + let table_time = started.elapsed(); + + assert_eq!(sum, table_sum, "the two must do the same work"); + + let ratio = table_time.as_secs_f64() / vec_time.as_secs_f64(); + eprintln!("VEC-COST vec={:?} table={:?} ratio={ratio:.2}x", vec_time, table_time); + assert!( + ratio < 4.0, + "the generated table took {ratio:.2}x the hand-written Vec plus BTreeMap. \ + It maintains one extra index and a different backend, so it is not \ + expected to tie in either direction, but this much means it is scanning \ + where the baseline looks up. See the doc comment for where parity is \ + actually measured." + ); +} + +worktable!( + name: Ordered, + vec: true, + columns: { + id: u64 primary_key using indexset, + value: u64, + tag: u64, + }, + indexes: { + tag_idx: tag using indexset, + }, +); + +worktable!( + name: Named, + vec: true, + columns: { + key: String primary_key, + value: u64, + }, +); + +/// Deleting from the middle leaves every other row where it was, on whichever +/// backend is holding the indexes. +/// +/// A delete now ghosts the slot, so nothing above the hole moves and no index +/// entry but the dead row's is touched. That is the cheap half; the expensive +/// half is `compact`, which is tested next to this. What this covers is the +/// state in between, where the vector is sparse and every lookup still has to +/// be right: a non-unique index whose posting list straddles the hole is what +/// makes an off-by-one visible, because a row reachable by the wrong key +/// rather than by none is something `select_all` alone would not catch. +#[test] +fn deleting_from_the_middle_reindexes_both_backends() { + macro_rules! check { + ($table:ty, $row:ident) => {{ + let mut table = <$table>::new(); + for id in 0..6u64 { + table + .insert($row { + id, + value: id * 10, + tag: id % 2, + }) + .expect("fresh"); + } + + assert_eq!(table.delete(&2).expect("present").value, 20); + + // Every survivor still answers to its own key, with its own value. + for id in [0u64, 1, 3, 4, 5] { + let row = table.select(&id).unwrap_or_else(|| panic!("{id} should survive")); + assert_eq!(row.value, id * 10, "{id} came back as another row"); + } + assert!(table.select(&2).is_none()); + assert_eq!(table.len(), 5); + + // Insertion order survives the hole. + let ids: Vec = table.select_all().map(|row| row.id).collect(); + assert_eq!(ids, vec![0, 1, 3, 4, 5]); + + // The non-unique index straddled the hole: tag 0 held 0, 2 and 4. + let even: Vec = table.select_by_tag(&0).iter().map(|row| row.id).collect(); + assert_eq!(even, vec![0, 4], "tag 0 kept a deleted row or lost a live one"); + let odd: Vec = table.select_by_tag(&1).iter().map(|row| row.id).collect(); + assert_eq!(odd, vec![1, 3, 5]); + + // And the table still takes writes afterwards. + table + .insert($row { + id: 9, + value: 90, + tag: 1, + }) + .expect("fresh"); + assert_eq!(table.select(&9).expect("present").value, 90); + let odd: Vec = table.select_by_tag(&1).iter().map(|row| row.id).collect(); + assert_eq!(odd, vec![1, 3, 5, 9]); + }}; + } + + check!(PointWorkTable, PointRow); + check!(OrderedWorkTable, OrderedRow); +} + +/// Arctic takes a `String` key, so the macro does not have to refuse one. +/// +/// This is here because the refusal test next to it uses `bool`, and the two +/// together say where the line actually is. A reader who sees only the refusal +/// would reasonably assume every non-integer key is out. +#[test] +fn a_string_keyed_table_works() { + let mut table = NamedWorkTable::new(); + table + .insert(NamedRow { + key: "beta".to_string(), + value: 2, + }) + .expect("fresh"); + table + .insert(NamedRow { + key: "alpha".to_string(), + value: 1, + }) + .expect("fresh"); + assert!( + table + .insert(NamedRow { + key: "alpha".to_string(), + value: 9 + }) + .is_err() + ); + + assert_eq!(table.select(&"alpha".to_string()).expect("present").value, 1); + assert_eq!(table.delete(&"beta".to_string()).expect("present").value, 2); + assert_eq!(table.select(&"alpha".to_string()).expect("present").value, 1); + assert_eq!(table.len(), 1); +} + +worktable!( + name: Congeed, + vec: true, + columns: { + id: u64 primary_key using congee, + value: u64, + }, +); + +worktable!( + name: Wtid, + vec: true, + columns: { + id: u64 primary_key using worktables_index, + value: u64, + code: u64, + }, + indexes: { + code_idx: code unique, + }, +); + +/// The two backends without a multimap still index a table, and still delete. +/// +/// Congee is here because it was refused outright for a while: `worktable!` +/// demands an explicit `persist` before accepting it, and this macro inherited +/// the rule without inheriting the reason. There is no persistence here for +/// the author to declare, so there was never a question to answer. +/// +/// The delete goes through the middle for the same reason as the arctic test: +/// it is the reinsert-every-position path, which neither of these backends can +/// do in place. +#[test] +fn the_backends_without_a_multimap_still_work() { + let mut congee = CongeedWorkTable::new(); + for id in 1..=5u64 { + congee.insert(CongeedRow { id, value: id * 10 }).expect("fresh"); + } + assert!(congee.insert(CongeedRow { id: 3, value: 99 }).is_err(), "duplicate key"); + assert_eq!(congee.delete(&3).expect("present").value, 30); + for id in [1u64, 2, 4, 5] { + assert_eq!(congee.select(&id).unwrap_or_else(|| panic!("{id} gone")).value, id * 10); + } + assert!(congee.select(&3).is_none()); + assert_eq!( + congee.select_all().map(|row| row.id).collect::>(), + vec![1, 2, 4, 5] + ); + + let mut wti = WtidWorkTable::new(); + for id in 1..=5u64 { + wti.insert(WtidRow { + id, + value: id * 10, + code: id + 100, + }) + .expect("fresh"); + } + // The unique secondary refuses independently of the primary key. + assert!( + wti.insert(WtidRow { + id: 6, + value: 60, + code: 103 + }) + .is_err(), + "duplicate code should be refused even though the id is fresh" + ); + // ...and refusing it must not have left the fresh id behind. + assert!(wti.select(&6).is_none(), "a rejected insert half-landed"); + assert_eq!(wti.len(), 5); + + assert_eq!(wti.select_by_code(&103).expect("present").id, 3); + assert_eq!(wti.delete(&3).expect("present").value, 30); + assert!(wti.select_by_code(&103).is_none(), "the secondary kept a deleted row"); + assert_eq!(wti.select_by_code(&104).expect("present").value, 40); +} + +worktable!( + name: Saved, + vec: true, + columns: { + id: u64 primary_key, + label: String, + tag: u64, + }, + indexes: { + tag_idx: tag, + }, +); + +/// Rows out as pages and back, with the indexes rebuilt rather than stored. +/// +/// The indexes are positions into the row vector, so they are cheaper to +/// rebuild on load than to write, validate and keep consistent with the rows. +/// This checks the rebuild rather than only the rows: a `load` that restored +/// `select_all` and left `select` empty would look correct to any assertion +/// that only walked the rows. +#[test] +fn a_table_survives_a_round_trip_through_pages() { + let mut table = SavedWorkTable::new(); + for id in 0..200u64 { + table + .insert(SavedRow { + id, + label: format!("row-{id}"), + tag: id % 8, + }) + .expect("fresh"); + } + table.delete(&7).expect("present"); + + let bytes = table.unload().expect("rows fit a page"); + assert_eq!(bytes.len() % 16384, 0, "whole pages only"); + + let loaded = SavedWorkTable::load(&bytes).expect("its own bytes"); + assert_eq!(loaded.len(), 199); + assert_eq!(loaded.select_all().count(), 199); + assert!(loaded.select(&7).is_none(), "the deleted row came back"); + + // Every key still finds its own row through the rebuilt primary index. + for id in (0..200u64).filter(|id| *id != 7) { + let row = loaded.select(&id).unwrap_or_else(|| panic!("{id} missing after load")); + assert_eq!(row.label, format!("row-{id}")); + } + // And the secondary index was rebuilt too, minus the deleted row. + assert_eq!( + loaded.select_by_tag(&7).len(), + 24, + "tag 7 held 25 rows before the delete" + ); + assert_eq!(loaded.select_by_tag(&0).len(), 25); + + // Insertion order survives, which is what makes `select_all` meaningful. + let ids: Vec = loaded.select_all().map(|row| row.id).collect(); + let expected: Vec = (0..200u64).filter(|id| *id != 7).collect(); + assert_eq!(ids, expected); +} + +/// An empty table still writes a page, and loads back empty. +/// +/// A zero byte file is indistinguishable from a missing one, so a load has to +/// be able to tell "no rows" from "nothing landed". +#[test] +fn an_empty_table_round_trips_as_one_page() { + let bytes = SavedWorkTable::new().unload().expect("nothing to overflow"); + assert_eq!(bytes.len(), 16384, "one page, not zero bytes"); + assert!(SavedWorkTable::load(&bytes).expect("its own bytes").is_empty()); +} + +/// A flipped bit inside a row is caught, which is the whole reason for the CRC. +/// +/// rkyv validates that an archive is structurally sound. It cannot tell that a +/// `u64` holds a different number than the one written, because the altered +/// archive is still perfectly well formed. Only the checksum sees it. +#[test] +fn a_flipped_bit_is_refused_rather_than_read() { + let mut table = SavedWorkTable::new(); + table + .insert(SavedRow { + id: 1, + label: "one".into(), + tag: 0, + }) + .expect("fresh"); + let mut bytes = table.unload().expect("fits"); + + // Into the body, which the header's last `u32` gives the length of. A + // fixed offset is not good enough: one small row archives to well under a + // hundred bytes, so byte 64 landed in the page's zero padding, outside + // what the checksum covers, and the file loaded cleanly. + let body = u32::from_le_bytes(bytes[24..28].try_into().expect("four bytes")) as usize; + assert!(body > 0, "a one-row page has a body"); + bytes[28 + body / 2] ^= 0b0000_0001; + + match SavedWorkTable::load(&bytes) { + Err(LoadError::Corrupt { page, .. }) => assert_eq!(page, 0), + other => panic!("a corrupted page loaded or failed some other way: {other:?}"), + } +} + +/// A truncated file is refused before any page is read. +#[test] +fn a_partial_page_is_refused() { + let mut table = SavedWorkTable::new(); + table + .insert(SavedRow { + id: 1, + label: "one".into(), + tag: 0, + }) + .expect("fresh"); + let bytes = table.unload().expect("fits"); + + match SavedWorkTable::load(&bytes[..bytes.len() - 1]) { + Err(LoadError::NotWholePages { found }) => assert_eq!(found, bytes.len() - 1), + other => panic!("a torn file loaded: {other:?}"), + } + match SavedWorkTable::load(&[]) { + Err(LoadError::NotWholePages { found }) => assert_eq!(found, 0), + other => panic!("an empty file loaded: {other:?}"), + } +} + +worktable!( + name: Other, + vec: true, + columns: { + id: u64 primary_key, + label: String, + tag: u64, + }, +); + +/// Another row type's file is refused, not reinterpreted. +/// +/// `OtherRow` has the same fields in the same order as `SavedRow`, so its +/// archive deserializes without complaint. Nothing but the fingerprint stands +/// between a caller and a table full of another table's rows. +#[test] +fn another_row_types_pages_are_refused() { + let mut other = OtherWorkTable::new(); + other + .insert(OtherRow { + id: 1, + label: "one".into(), + tag: 0, + }) + .expect("fresh"); + let bytes = other.unload().expect("fits"); + + match SavedWorkTable::load(&bytes) { + Err(LoadError::ForeignRows { found, expected }) => assert_ne!(found, expected), + other => panic!("another row type's file loaded: {other:?}"), + } +} + +/// Rows spanning many pages come back in order. +/// +/// One page holds 16 KiB, so this is several of them, and the page-boundary +/// arithmetic is what the test is for: a row dropped at a boundary, or a page +/// whose rows are appended twice, shows up as a length or an order mismatch. +#[test] +fn rows_across_many_pages_come_back_in_order() { + let mut table = SavedWorkTable::new(); + for id in 0..5_000u64 { + table + .insert(SavedRow { + id, + label: format!("a fairly long label for row {id}"), + tag: id % 8, + }) + .expect("fresh"); + } + let bytes = table.unload().expect("no single row is oversized"); + assert!( + bytes.len() / 16384 > 1, + "this needs to span pages to be testing anything" + ); + + let loaded = SavedWorkTable::load(&bytes).expect("its own bytes"); + assert_eq!(loaded.len(), 5_000); + let ids: Vec = loaded.select_all().map(|row| row.id).collect(); + assert_eq!(ids, (0..5_000u64).collect::>()); + assert_eq!( + loaded.select(&4_999).expect("last row").label, + "a fairly long label for row 4999" + ); +} + +/// `update` edits in place and repairs every index the edit moved the row +/// under. +/// +/// The reason it takes a closure rather than handing out `&mut Row`: a caller +/// with `&mut Row` can change an indexed column, and the index then points at +/// a key the row no longer has. That is silent, and the row is unfindable by +/// either key. The closure lets the table compare before against after. +#[test] +fn update_edits_in_place_and_repairs_the_indexes() { + let mut table = PointWorkTable::new(); + for id in 0..4u64 { + table + .insert(PointRow { + id, + value: id * 10, + tag: id % 2, + }) + .expect("fresh"); + } + + // An unindexed column: nothing to repair, and nothing should move. + assert!(table.update(&2, |row| row.value = 999)); + assert_eq!(table.select(&2).expect("present").value, 999); + assert_eq!(table.select_by_tag(&0).len(), 2); + + // An indexed column: the row has to leave one posting list and join another. + assert!(table.update(&2, |row| row.tag = 1)); + let evens: Vec = table.select_by_tag(&0).iter().map(|row| row.id).collect(); + assert_eq!(evens, vec![0], "row 2 stayed in its old posting list"); + let odds: Vec = table.select_by_tag(&1).iter().map(|row| row.id).collect(); + assert_eq!(odds, vec![1, 2, 3], "row 2 never joined its new one"); + + // The primary key itself: findable under the new key, gone from the old. + assert!(table.update(&2, |row| row.id = 42)); + assert!(table.select(&2).is_none(), "the old key still resolves"); + assert_eq!(table.select(&42).expect("present").value, 999); + assert_eq!(table.len(), 4, "a re-key is not an insert"); + + // A key that does not exist changes nothing. + assert!(!table.update(&1000, |row| row.value = 1)); +} + +/// A re-key onto an occupied key is refused, and refused without damage. +#[test] +#[should_panic(expected = "primary key another row already holds")] +fn update_refuses_to_collide_two_rows_onto_one_key() { + let mut table = PointWorkTable::new(); + table + .insert(PointRow { + id: 1, + value: 10, + tag: 0, + }) + .expect("fresh"); + table + .insert(PointRow { + id: 2, + value: 20, + tag: 0, + }) + .expect("fresh"); + table.update(&1, |row| row.id = 2); +} + +/// Sizing the row vector up front, and handing the rows back out. +#[test] +fn capacity_and_into_rows() { + let mut table = PointWorkTable::with_capacity(64); + assert!(table.capacity() >= 64); + table.reserve(256); + assert!(table.capacity() >= 256); + + for id in 0..3u64 { + table.insert(PointRow { id, value: id, tag: 0 }).expect("fresh"); + } + assert_eq!(table.iter().count(), 3); + assert_eq!(table.iter().map(|row| row.id).collect::>(), vec![0, 1, 2]); + + let rows = table.into_rows(); + assert_eq!(rows.len(), 3); + assert_eq!(rows[2].id, 2); +} + +// The WTI leaf width, set at the call site. +// +// It is a call-site parameter and not grammar because the right width depends +// on the workload rather than the schema: measured at a million shuffled keys, +// 256 is 1.56x faster than the 1,024 default on insert and 3.7% slower on +// lookup, so the same declaration wants different widths in a write-heavy +// process and a read-heavy one. A declaration can only say one thing. +worktable!( + name: Tuned, + vec: true, + columns: { + id: u64 primary_key using worktables_index, + value: u64, + } +); + +// No WTI index anywhere, so no knob should be generated for it. +worktable!( + name: Untuned, + vec: true, + columns: { + id: u64 primary_key, + value: u64, + } +); + +#[test] +fn the_node_size_is_a_call_site_parameter() { + let mut table = TunedWorkTable::with_node_size(256); + for id in 0..1_000u64 { + table.insert(TunedRow { id, value: id * 2 }).expect("fresh key"); + } + assert_eq!(table.len(), 1_000); + assert_eq!(table.select(&500).expect("present").value, 1_000); +} + +#[test] +fn a_narrow_node_size_caps_nothing() { + // The width is the leaf size a node splits at, not a limit on rows. A tree + // built with a width of 2 must hold thousands of rows by adding nodes, + // exactly as it does at the default. This is the "what happens when it + // needs to grow" question, answered: it grows. + let mut table = TunedWorkTable::with_node_size(2); + for id in 0..5_000u64 { + table.insert(TunedRow { id, value: id }).expect("fresh key"); + } + assert_eq!(table.len(), 5_000); + for id in (0..5_000u64).step_by(97) { + assert_eq!(table.select(&id).expect("present").value, id, "row {id} went missing"); + } + + // And the same table at an absurdly wide leaf holds exactly the same rows. + let mut wide = TunedWorkTable::with_node_size(1 << 20); + for id in 0..5_000u64 { + wide.insert(TunedRow { id, value: id }).expect("fresh key"); + } + assert_eq!(wide.len(), 5_000); + assert_eq!(wide.select(&4_999).expect("present").value, 4_999); +} + +#[test] +fn both_knobs_compose() { + let table = TunedWorkTable::with_capacity_and_node_size(4_096, 256); + assert!(table.capacity() >= 4_096, "the row vector was sized"); + assert_eq!(table.len(), 0); +} + +#[test] +fn a_table_with_no_wti_index_gets_no_node_size_knob() { + // Asserted by compiling: `UntunedWorkTable::with_node_size` does not exist, + // because arctic has no node-size concept and a constructor that accepted + // one would be a silent no-op. The table still works. + let mut table = UntunedWorkTable::with_capacity(16); + table.insert(UntunedRow { id: 1, value: 2 }).expect("fresh key"); + assert_eq!(table.select(&1).expect("present").value, 2); +} + +// --------------------------------------------------------------------------- +// Ghosts, and the compaction that reclaims them. + +/// A delete costs a bit and a slot, and moves nothing. +/// +/// This is the whole claim, so it is asserted on structure rather than on +/// behaviour: `slots` does not fall, `len` does, and the surviving rows keep +/// the positions they had. A `delete` that quietly went back to closing the +/// hole would still pass every lookup assertion in this file, because closing +/// the hole correctly is what the old implementation did. +#[test] +fn a_delete_leaves_a_ghost_and_nothing_moves() { + let mut table = PointWorkTable::new(); + for id in 0..6u64 { + table + .insert(PointRow { + id, + value: id * 10, + tag: id % 2, + }) + .expect("fresh"); + } + assert_eq!(table.slots(), 6); + assert_eq!(table.ghost_count(), 0); + + table.delete(&2).expect("present"); + + assert_eq!(table.len(), 5, "one fewer live row"); + assert_eq!(table.slots(), 6, "the slot was kept, not closed"); + assert_eq!(table.ghost_count(), 1); + assert!(!table.is_empty()); + + // Deleting every row leaves six ghosts and an empty table. + for id in [0u64, 1, 3, 4, 5] { + table.delete(&id).expect("present"); + } + assert!(table.is_empty()); + assert_eq!(table.len(), 0); + assert_eq!(table.slots(), 6); + assert_eq!(table.ghost_count(), 6); + assert_eq!(table.select_all().count(), 0); + + // And an insert after that appends rather than reusing a ghost, which is + // what keeps `select_all` in insertion order. + table + .insert(PointRow { + id: 42, + value: 420, + tag: 0, + }) + .expect("fresh"); + assert_eq!(table.slots(), 7); + assert_eq!(table.select(&42).expect("present").value, 420); +} + +/// Compaction closes every hole and leaves every index pointing at the row it +/// named before. +/// +/// Run on each backend, because renumbering is the one operation whose +/// implementation genuinely differs between them: a `BTreeMap` rewrites values +/// in place, an ART cannot and has to reinsert, and the non-unique arm moves +/// pairs. Deleting from the middle of a straddling posting list is what makes +/// an off-by-one visible, for the same reason the delete test does it. +#[test] +fn compaction_reclaims_the_ghosts_and_repairs_every_index() { + macro_rules! check { + ($table:ty, $row:ident) => {{ + let mut table = <$table>::new(); + for id in 0..8u64 { + table + .insert($row { + id, + value: id * 10, + tag: id % 2, + }) + .expect("fresh"); + } + for id in [1u64, 2, 5] { + table.delete(&id).expect("present"); + } + assert_eq!(table.ghost_count(), 3); + + assert_eq!(table.compact(), 3, "three slots were reclaimed"); + assert_eq!(table.ghost_count(), 0); + assert_eq!(table.slots(), 5); + assert_eq!(table.len(), 5); + assert_eq!(table.compact(), 0, "a second pass has nothing to do"); + + // Every survivor answers to its own key, with its own value. A + // renumbering that was off by one would hand back a neighbour. + for id in [0u64, 3, 4, 6, 7] { + let row = table.select(&id).unwrap_or_else(|| panic!("{id} lost by compaction")); + assert_eq!(row.value, id * 10, "{id} came back as another row"); + } + assert!(table.select(&2).is_none()); + + // Insertion order survived. + let ids: Vec = table.select_all().map(|row| row.id).collect(); + assert_eq!(ids, vec![0, 3, 4, 6, 7]); + + // The non-unique index straddled all three holes. + let even: Vec = table.select_by_tag(&0).iter().map(|row| row.id).collect(); + assert_eq!(even, vec![0, 4, 6], "tag 0 lost a row or kept a dead one"); + let odd: Vec = table.select_by_tag(&1).iter().map(|row| row.id).collect(); + assert_eq!(odd, vec![3, 7]); + + // And the table still takes writes at the new positions. + table + .insert($row { + id: 9, + value: 90, + tag: 1, + }) + .expect("fresh"); + assert_eq!(table.select(&9).expect("present").value, 90); + assert_eq!(table.select(&0).expect("present").value, 0); + let odd: Vec = table.select_by_tag(&1).iter().map(|row| row.id).collect(); + assert_eq!(odd, vec![3, 7, 9]); + }}; + } + + check!(PointWorkTable, PointRow); + check!(OrderedWorkTable, OrderedRow); +} + +/// The backends with no multimap compact too, including a unique secondary. +#[test] +fn compaction_repairs_the_multimapless_backends() { + let mut congee = CongeedWorkTable::new(); + for id in 1..=6u64 { + congee.insert(CongeedRow { id, value: id * 10 }).expect("fresh"); + } + congee.delete(&2).expect("present"); + congee.delete(&3).expect("present"); + assert_eq!(congee.compact(), 2); + for id in [1u64, 4, 5, 6] { + assert_eq!(congee.select(&id).unwrap_or_else(|| panic!("{id} gone")).value, id * 10); + } + assert_eq!(congee.slots(), 4); + + let mut wti = WtidWorkTable::new(); + for id in 1..=6u64 { + wti.insert(WtidRow { + id, + value: id * 10, + code: id + 100, + }) + .expect("fresh"); + } + wti.delete(&2).expect("present"); + wti.delete(&5).expect("present"); + assert_eq!(wti.compact(), 2); + for id in [1u64, 3, 4, 6] { + assert_eq!(wti.select(&id).unwrap_or_else(|| panic!("{id} gone")).value, id * 10); + // The unique secondary was renumbered alongside the primary. + assert_eq!( + wti.select_by_code(&(id + 100)) + .unwrap_or_else(|| panic!("{id} code gone")) + .id, + id, + "the code index points at the wrong row after compaction" + ); + } + assert!(wti.select_by_code(&102).is_none(), "a deleted row kept its code entry"); +} + +/// Compaction keeps the leaf width the call site asked for. +/// +/// Rebuilding the indexes from `Default::default()` would be the obvious way +/// to renumber and would silently throw away `with_node_size`, which is the +/// kind of failure nothing else here would catch: the table would still be +/// correct and only slower. Asserted by continuing to work at a width of 2, +/// where a reset to the 1,024 default changes the tree's shape entirely. +#[test] +fn compaction_keeps_the_node_size_the_caller_asked_for() { + let mut table = TunedWorkTable::with_node_size(2); + for id in 0..64u64 { + table.insert(TunedRow { id, value: id }).expect("fresh"); + } + for id in (0..64u64).step_by(2) { + table.delete(&id).expect("present"); + } + assert_eq!(table.compact(), 32); + assert_eq!(table.slots(), 32); + for id in (1..64u64).step_by(2) { + assert_eq!(table.select(&id).unwrap_or_else(|| panic!("{id} gone")).value, id); + } +} + +/// Ghosts are slots, and `shrink_to_fit` is the only thing that hands them +/// back to the allocator. +#[test] +fn compaction_keeps_capacity_and_shrinking_gives_it_back() { + let mut table = PointWorkTable::with_capacity(256); + for id in 0..128u64 { + table.insert(PointRow { id, value: id, tag: 0 }).expect("fresh"); + } + for id in 0..120u64 { + table.delete(&id).expect("present"); + } + table.compact(); + assert!(table.capacity() >= 256, "compaction kept the capacity for reuse"); + table.shrink_to_fit(); + assert!(table.capacity() < 256, "shrinking did not give it back"); + assert_eq!(table.len(), 8); +} + +/// A ghost is not written out, so a reload does not resurrect it. +#[test] +fn unload_does_not_write_a_ghost() { + let mut table = SavedWorkTable::new(); + for id in 0..40u64 { + table + .insert(SavedRow { + id, + label: format!("row-{id}"), + tag: id % 4, + }) + .expect("fresh"); + } + for id in [3u64, 11, 29] { + table.delete(&id).expect("present"); + } + assert_eq!(table.ghost_count(), 3, "unload is being asked to skip real ghosts"); + + let loaded = SavedWorkTable::load(&table.unload().expect("rows fit")).expect("its own bytes"); + assert_eq!(loaded.len(), 37); + assert_eq!(loaded.slots(), 37, "the ghosts were written as rows"); + for id in [3u64, 11, 29] { + assert!(loaded.select(&id).is_none(), "{id} came back from the dead"); + } +} + +// --------------------------------------------------------------------------- +// Ranges, which the index was always able to answer. + +/// The primary-key range walks in key order, in both directions, on every +/// bound shape. +/// +/// Keys are inserted out of order on purpose: an implementation that walked +/// the row vector instead of the index would return insertion order and pass +/// any test whose rows went in sorted. +#[test] +fn a_range_walks_the_keys_in_order() { + let mut table = PointWorkTable::new(); + for id in [5u64, 1, 9, 3, 7, 2, 8, 4, 6] { + table + .insert(PointRow { + id, + value: id * 10, + tag: id % 2, + }) + .expect("fresh"); + } + + let ids = |rows: Vec<&PointRow>| rows.into_iter().map(|row| row.id).collect::>(); + + assert_eq!(ids(table.range(3..7).collect()), vec![3, 4, 5, 6]); + assert_eq!(ids(table.range(3..=7).collect()), vec![3, 4, 5, 6, 7]); + assert_eq!(ids(table.range(..3).collect()), vec![1, 2]); + assert_eq!(ids(table.range(7..).collect()), vec![7, 8, 9]); + assert_eq!(ids(table.range(..).collect()), vec![1, 2, 3, 4, 5, 6, 7, 8, 9]); + assert_eq!(ids(table.range(100..200).collect()), Vec::::new()); + + // The rows are the right rows, not just the right keys. + for row in table.range(..) { + assert_eq!(row.value, row.id * 10); + } + + // Backwards, which is what makes this a `DoubleEndedIterator` rather than + // an iterator that happens to arrive sorted. + assert_eq!(ids(table.range(..).rev().collect()), vec![9, 8, 7, 6, 5, 4, 3, 2, 1]); + assert_eq!(ids(table.range(3..7).rev().collect()), vec![6, 5, 4, 3]); +} + +/// A deleted row leaves no index entry, so a range never has to look at a +/// ghost. +/// +/// This is what lets `range` call `row_at` and expect a row: if a delete left +/// its entry behind, the range would walk into an empty slot and panic, which +/// is a far better failure than silently returning a stale row and is still a +/// failure. Asserted so the invariant is checked rather than assumed. +#[test] +fn a_range_skips_a_deleted_row() { + let mut table = PointWorkTable::new(); + for id in 0..10u64 { + table + .insert(PointRow { + id, + value: id * 10, + tag: id % 2, + }) + .expect("fresh"); + } + for id in [4u64, 5, 6] { + table.delete(&id).expect("present"); + } + + let ids: Vec = table.range(2..9).map(|row| row.id).collect(); + assert_eq!(ids, vec![2, 3, 7, 8], "a range walked into a ghost"); + assert_eq!(table.range(..).count(), 7); + + // And compaction does not change the answer, only where the rows live. + table.compact(); + let ids: Vec = table.range(2..9).map(|row| row.id).collect(); + assert_eq!(ids, vec![2, 3, 7, 8]); +} + +/// Every backend answers a range, because every backend the `using` clause can +/// name is an ordered tree. +/// +/// Congee is the one worth naming: it is an adaptive radix tree with a native +/// range scan, and it was the backend most likely to have been given a range +/// that silently returned everything. +#[test] +fn every_backend_answers_a_range() { + let mut ordered = OrderedWorkTable::new(); + let mut congee = CongeedWorkTable::new(); + let mut wti = WtidWorkTable::new(); + for id in [7u64, 2, 9, 4, 1, 6, 3, 8, 5] { + ordered + .insert(OrderedRow { + id, + value: id * 10, + tag: id % 2, + }) + .expect("fresh"); + congee.insert(CongeedRow { id, value: id * 10 }).expect("fresh"); + wti.insert(WtidRow { + id, + value: id * 10, + code: id + 100, + }) + .expect("fresh"); + } + + assert_eq!( + ordered.range(3..7).map(|row| row.id).collect::>(), + vec![3, 4, 5, 6] + ); + assert_eq!( + congee.range(3..7).map(|row| row.id).collect::>(), + vec![3, 4, 5, 6] + ); + assert_eq!(wti.range(3..7).map(|row| row.id).collect::>(), vec![3, 4, 5, 6]); + + // A unique secondary index is an ordered tree too, and ranges on its own + // column rather than on the primary key. + assert_eq!( + wti.range_by_code(&103..&106).map(|row| row.id).collect::>(), + vec![3, 4, 5], + "the secondary range answered on the wrong column" + ); +} + +/// A `String` key ranges lexicographically, which is the index's order and not +/// the vector's. +#[test] +fn a_string_key_ranges_lexicographically() { + let mut table = NamedWorkTable::new(); + for key in ["delta", "alpha", "charlie", "bravo", "echo"] { + table + .insert(NamedRow { + key: key.to_string(), + value: key.len() as u64, + }) + .expect("fresh"); + } + let keys: Vec<&str> = table.range(..).map(|row| row.key.as_str()).collect(); + assert_eq!(keys, vec!["alpha", "bravo", "charlie", "delta", "echo"]); + + let keys: Vec<&str> = table + .range("bravo".to_string().."delta".to_string()) + .map(|row| row.key.as_str()) + .collect(); + assert_eq!(keys, vec!["bravo", "charlie"]); +} + +// --------------------------------------------------------------------------- +// `using fxhash`: point operations only, and the range methods are absent. + +worktable!( + name: Hashed, + vec: true, + columns: { + id: u64 primary_key using fxhash, + value: u64, + tag: u64, + }, + indexes: { + tag_idx: tag using fxhash, + code_idx: value unique using fxhash, + }, +); + +/// A hash-backed table is a table: everything but ordering still works. +/// +/// Deliberately exercises the whole surface rather than a lookup, because the +/// hash arm reaches a different branch in every one of `insert`, `upsert`, +/// `update`, `delete` and `compact` — it uses inherent map methods where the +/// ARTs use the `UniqueIndex` trait, and its entry type is `HashMapEntry` +/// rather than `BTreeMapEntry`. +#[test] +fn a_hash_backed_table_does_everything_but_order() { + let mut table = HashedWorkTable::new(); + for id in 0..8u64 { + table + .insert(HashedRow { + id, + value: id * 10, + tag: id % 2, + }) + .expect("fresh"); + } + + // Duplicate primary key, refused in one traversal through the entry API. + assert!( + table + .insert(HashedRow { + id: 3, + value: 999, + tag: 0 + }) + .is_err(), + "duplicate primary key" + ); + // Duplicate unique secondary, refused independently of the primary key. + assert!( + table + .insert(HashedRow { + id: 99, + value: 30, + tag: 0 + }) + .is_err(), + "duplicate unique secondary" + ); + + assert_eq!(table.select(&3).expect("present").value, 30); + assert_eq!(table.select_by_value(&40).expect("present").id, 4); + let even: Vec = table.select_by_tag(&0).iter().map(|row| row.id).collect(); + assert_eq!(even, vec![0, 2, 4, 6]); + + // Update, including a key move, which repairs three maps. + assert!(table.update(&5, |row| { + row.value = 555; + row.tag = 0; + })); + assert_eq!(table.select(&5).expect("present").value, 555); + assert_eq!(table.select_by_value(&555).expect("present").id, 5); + assert!(table.select_by_value(&50).is_none(), "the old value kept its entry"); + + // Upsert replaces in place. + table.upsert(HashedRow { + id: 5, + value: 5_555, + tag: 1, + }); + assert_eq!(table.select(&5).expect("present").value, 5_555); + assert_eq!(table.len(), 8, "upsert did not grow the table"); + + // Delete ghosts, and compaction renumbers every hash map. + assert_eq!(table.delete(&2).expect("present").value, 20); + assert_eq!(table.delete(&6).expect("present").value, 60); + assert_eq!(table.ghost_count(), 2); + assert_eq!(table.compact(), 2); + assert_eq!(table.slots(), 6); + for id in [0u64, 1, 3, 4, 5, 7] { + assert_eq!( + table.select(&id).unwrap_or_else(|| panic!("{id} lost")).id, + id, + "compaction pointed the primary index at the wrong row" + ); + } + assert_eq!( + table.select_by_value(&70).expect("present").id, + 7, + "compaction pointed the unique secondary at the wrong row" + ); + let even: Vec = table.select_by_tag(&0).iter().map(|row| row.id).collect(); + assert_eq!(even, vec![0, 4], "compaction lost or misplaced a posting list entry"); +} + +/// Ranges are not emitted for a hash backend, and that is checked by compiling. +/// +/// `HashedWorkTable::range` and `range_by_value` do not exist. There is nothing +/// to call here, which is the assertion: a method that existed and panicked, or +/// returned insertion order and called it key order, is the failure mode this +/// design avoids. The ordered tables next to this one have both methods and are +/// tested for them. +#[test] +fn a_hash_backed_table_has_no_range() { + let mut table = HashedWorkTable::new(); + table + .insert(HashedRow { + id: 1, + value: 1, + tag: 1, + }) + .expect("fresh"); + // Still walkable in insertion order, which needs no index at all. + assert_eq!(table.select_all().count(), 1); + assert_eq!(table.iter().count(), 1); +} + +worktable!( + name: HashedSaved, + vec: true, + columns: { + id: u64 primary_key using fxhash, + label: String, + code: u64, + }, + indexes: { + code_idx: code unique using fxhash, + tag_idx: label using fxhash, + }, +); + +/// A hash-backed table round-trips through pages, indexes and all. +/// +/// This is the question `persist: true` makes people ask about `fxhash` and +/// answers wrongly. A **paged** table cannot take a hash index because a +/// persisted index's on-disk form *is* sorted pages, rebuilt with +/// `attach_nodes`. A `vec: true` table stores **no index at all**: `unload` +/// writes rows and `load` rebuilds every index by re-inserting them. So the +/// thing that blocks the paged table does not exist here, and manual +/// flush-and-hydrate works on a hash backend exactly as it does on a tree. +/// +/// Asserted on the indexes rather than on the rows, because rows surviving is +/// the easy half: a `load` that restored `select_all` and left `select` empty +/// would pass any assertion that only walked the table. +#[test] +fn a_hash_backed_table_round_trips_through_pages() { + let mut table = HashedSavedWorkTable::with_capacity(500); + for id in 0..500u64 { + table + .insert(HashedSavedRow { + id, + label: format!("row-{}", id % 8), + code: id + 10_000, + }) + .expect("fresh"); + } + // Ghosts too, so the round trip is exercised on a table that has deleted. + for id in [3u64, 111, 499] { + table.delete(&id).expect("present"); + } + assert_eq!(table.ghost_count(), 3); + + let bytes = table.unload().expect("rows fit a page"); + assert_eq!(bytes.len() % 16_384, 0, "whole pages only"); + + let loaded = HashedSavedWorkTable::load(&bytes).expect("its own bytes"); + assert_eq!(loaded.len(), 497); + assert_eq!(loaded.slots(), 497, "a ghost was written as a row"); + + // The primary hash index was rebuilt. + for id in (0..500u64).filter(|id| ![3, 111, 499].contains(id)) { + let row = loaded.select(&id).unwrap_or_else(|| panic!("{id} missing after load")); + assert_eq!(row.code, id + 10_000, "{id} came back as another row"); + } + for id in [3u64, 111, 499] { + assert!(loaded.select(&id).is_none(), "{id} came back from the dead"); + } + + // And both secondary hash indexes, unique and non-unique. + assert_eq!( + loaded.select_by_code(&10_042).expect("present").id, + 42, + "the unique secondary was not rebuilt" + ); + assert!(loaded.select_by_code(&10_003).is_none(), "a deleted row kept its code"); + // The deleted ids are 3, 111 and 499, which are 3, 7 and 3 mod 8, so + // lost two and lost one. lost none, which is why it is not + // the tag asserted on: a posting list that never changed proves nothing + // about whether a delete reached the index. + let intact = loaded.select_by_label(&"row-0".to_string()); + assert_eq!(intact.len(), 63, "row-0 lost a row it never had deleted"); + let lost_two = loaded.select_by_label(&"row-3".to_string()); + assert_eq!(lost_two.len(), 61, "row-3 held 63 and lost ids 3 and 499"); + let lost_one = loaded.select_by_label(&"row-7".to_string()); + assert_eq!(lost_one.len(), 61, "row-7 held 62 (ids 7..=495 step 8) and lost id 111"); + + // Insertion order survives, which is what makes `select_all` meaningful. + let ids: Vec = loaded.select_all().map(|row| row.id).collect(); + let expected: Vec = (0..500u64).filter(|id| ![3, 111, 499].contains(id)).collect(); + assert_eq!(ids, expected); + + // The reloaded table still takes writes, which proves the rebuilt index is + // a working map and not just a populated one. + let mut loaded = loaded; + assert!( + loaded + .insert(HashedSavedRow { + id: 42, + label: "dup".into(), + code: 1 + }) + .is_err() + ); + loaded + .insert(HashedSavedRow { + id: 3, + label: "back".into(), + code: 3, + }) + .expect("the deleted key is free again"); + assert_eq!(loaded.select(&3).expect("present").code, 3); +} + +/// Two unloads concatenated load as one table, so a flush can append. +/// +/// `unload` writes the whole table, so writing it to a file is a clobber and +/// there is no incremental form of it. But a page is self-describing — its own +/// header, CRC, row directory and row-type fingerprint — and `from_pages` walks +/// `chunks_exact(PAGE_SIZE)` in order without any global header or trailer. So +/// the bytes of two unloads concatenated are a valid file, and a caller that +/// keeps new rows in a second table can append rather than rewrite. +/// +/// What append cannot express is a delete. `load` applies rows in order and +/// keeps the first of any duplicate key, so a later segment cannot remove or +/// replace an earlier row. Both halves are asserted, because the second is the +/// one that decides whether this is a usable strategy or a trap. +#[test] +fn two_unloads_concatenate_into_one_table() { + let mut first = HashedSavedWorkTable::with_capacity(64); + for id in 0..64u64 { + first + .insert(HashedSavedRow { + id, + label: "a".into(), + code: id, + }) + .expect("fresh"); + } + let mut second = HashedSavedWorkTable::with_capacity(64); + for id in 64..128u64 { + second + .insert(HashedSavedRow { + id, + label: "b".into(), + code: id, + }) + .expect("fresh"); + } + + let mut appended = first.unload().expect("rows fit"); + appended.extend_from_slice(&second.unload().expect("rows fit")); + assert_eq!(appended.len() % 16_384, 0, "still whole pages"); + + let loaded = HashedSavedWorkTable::load(&appended).expect("a concatenation of its own pages"); + assert_eq!(loaded.len(), 128, "the append lost a segment"); + for id in 0..128u64 { + assert_eq!( + loaded.select(&id).unwrap_or_else(|| panic!("{id} missing")).code, + id, + "{id} came back as another row" + ); + } + // Order is segment order, which is what makes this an append rather than a + // merge: the second file's rows follow the first file's. + let ids: Vec = loaded.select_all().map(|row| row.id).collect(); + assert_eq!(ids, (0..128u64).collect::>()); + + // And the limit. A later segment cannot replace an earlier row: `load` + // keeps the first of a duplicate key, so an append-only log of these needs + // a full rewrite to express an update or a delete. + let mut shadow = HashedSavedWorkTable::with_capacity(1); + shadow + .insert(HashedSavedRow { + id: 7, + label: "newer".into(), + code: 9_999, + }) + .expect("fresh"); + let mut with_shadow = first.unload().expect("rows fit"); + with_shadow.extend_from_slice(&shadow.unload().expect("rows fit")); + let reloaded = HashedSavedWorkTable::load(&with_shadow).expect("valid pages"); + assert_eq!( + reloaded.select(&7).expect("present").code, + 7, + "a later segment overwrote an earlier row; it must not, and if this ever \ + changes then append becomes a way to silently lose the newer value \ + instead of the older one" + ); + assert_eq!(reloaded.len(), 64, "the duplicate was counted as a new row"); +} + +#[test] +fn append_callsite_writes_only_new_live_rows_and_rebuilds_indexes() { + let mut table = HashedSavedWorkTable::new(); + for id in 0..100u64 { + table + .insert(HashedSavedRow { + id, + label: "first".into(), + code: id, + }) + .unwrap(); + } + table.delete(&3).unwrap(); + let first = table.len(); + let mut bytes = table.unload().unwrap(); + let before = bytes.clone(); + for id in 100..200u64 { + table + .insert(HashedSavedRow { + id, + label: "second".into(), + code: id, + }) + .unwrap(); + } + let pages_before = u32::try_from(bytes.len() / worktable::vec_hydrate::PAGE_SIZE).unwrap(); + let appended = table.unload_appending(first, pages_before).unwrap(); + assert_eq!(HashedSavedWorkTable::load(&appended).unwrap().len(), 100); + bytes.extend_from_slice(&appended); + assert_eq!(&bytes[..before.len()], &before); + let loaded = HashedSavedWorkTable::load(&bytes).unwrap(); + assert_eq!(loaded.len(), 199); + assert!(loaded.select(&3).is_none()); + assert_eq!(loaded.select_by_label(&"second".into()).len(), 100); + assert_eq!(loaded.select_by_code(&199).unwrap().id, 199); +} + +worktable!( + name: Level, + vec: true, + columns: { + exchange: u64 primary_key, + bid: f64, + ask: f64, + size: u64, + }, +); + +worktable!( + name: Labelled, + vec: true, + columns: { + id: u64 primary_key, + label: String, + }, +); + +/// A fixed-width row's page layout does not move when a value changes. +/// +/// This is the property the whole in-place persistence question turns on. An +/// orderbook updates a price: an `f64` becomes another `f64`, never an `f80`. +/// If the archive of a page is the same length before and after, then row K +/// lives at the same byte offset forever, a page can be written back in place, +/// and none of the append, segment, last-wins or tombstone machinery is needed +/// for that shape. +/// +/// `rows_per_page` searches rather than computing, so stability is a property +/// of the data and not an obvious one: it is asserted here rather than assumed +/// anywhere that relies on it. +/// +/// The `String` table is the control. Without it this test would pass on a +/// format that simply never varies, and prove nothing about the format's +/// ability to vary. +#[test] +fn a_fixed_width_rows_pages_are_byte_stable_under_update() { + let mut table = LevelWorkTable::with_capacity(5_000); + for exchange in 0..5_000u64 { + table + .insert(LevelRow { + exchange, + bid: 100.0, + ask: 101.0, + size: 10, + }) + .expect("fresh key"); + } + let before = table.unload().expect("rows fit"); + + // Every value changes, and every value stays the same width. + for exchange in 0..5_000u64 { + assert!(table.update(&exchange, |row| { + row.bid = (exchange % 997) as f64 + 0.5; + row.ask = f64::MAX; + row.size = u64::MAX; + })); + } + let after = table.unload().expect("rows fit"); + + assert_eq!( + before.len(), + after.len(), + "a fixed-width row changed its page count by changing its values" + ); + // Stronger than equal length: every page boundary is where it was, so the + // row at a given offset is still the row that was there. + assert_eq!(before.len() % 16_384, 0); + let pages = before.len() / 16_384; + for page in 0..pages { + let at = page * 16_384; + // The header carries the page index and the body length. Both must be + // unchanged; only the body bytes may differ. + assert_eq!( + before[at..at + 32], + after[at..at + 32], + "page {page}'s header moved, so a row's home is not stable" + ); + } + assert_ne!(before, after, "the values did not actually change"); + + // The control: a variable-width row is not stable, which is what makes the + // assertion above a real property rather than a description of the format. + let mut labelled = LabelledWorkTable::with_capacity(5_000); + for id in 0..5_000u64 { + labelled + .insert(LabelledRow { + id, + label: "x".to_string(), + }) + .expect("fresh key"); + } + let short = labelled.unload().expect("rows fit"); + for id in 0..5_000u64 { + assert!(labelled.update(&id, |row| { + row.label = "x".repeat(64); + })); + } + let long = labelled.unload().expect("rows fit"); + assert!( + long.len() > short.len(), + "a String column grew by 63 bytes a row and the file did not grow, so \ + this test is not measuring what it claims to" + ); +} + +worktable!( + name: Mixed, + vec: true, + columns: { + id: u64 primary_key using fxhash, + venue: u64, + seq: u64, + }, + indexes: { + venue_idx: venue using arctic, + seq_idx: seq unique using arctic, + }, +); + +/// A backend is chosen per index, so a hash primary key does not cost the +/// secondaries their ordering. +/// +/// The primary key here cannot answer a range and the secondaries can, which is +/// the whole point: `range` is absent from this table and `range_by_seq` is +/// present on it. If capability were decided per table rather than per index, +/// one of those two facts would be wrong. +#[test] +fn a_hash_primary_key_leaves_an_arctic_secondary_ordered() { + let mut table = MixedWorkTable::with_capacity(64); + // Inserted out of key order so an implementation that walked the row vector + // instead of the index would return insertion order and be caught. + for id in [5u64, 1, 9, 3, 7, 2, 8, 4, 6] { + table + .insert(MixedRow { + id, + venue: id % 3, + seq: 100 + id, + }) + .expect("fresh key"); + } + + // The hash primary key does point lookups, and refuses a duplicate. + assert_eq!(table.select(&7).expect("present").seq, 107); + assert!( + table + .insert(MixedRow { + id: 7, + venue: 0, + seq: 999 + }) + .is_err() + ); + + // The unique arctic secondary ranges, in its own column's order. + let ranged: Vec = table.range_by_seq(&103..&107).map(|row| row.id).collect(); + assert_eq!(ranged, vec![3, 4, 5, 6], "the arctic secondary lost its order"); + let backwards: Vec = table.range_by_seq(&103..&107).rev().map(|row| row.id).collect(); + assert_eq!(backwards, vec![6, 5, 4, 3]); + + // The non-unique arctic secondary still groups. + let venue0: Vec = table.select_by_venue(&0).iter().map(|row| row.id).collect(); + assert_eq!( + venue0, + vec![9, 3, 6], + "insertion order within a venue: 9, 3 and 6 are the ids with venue 0" + ); + + // And all of it survives a delete and a compaction, which renumber the + // hash map and both ARTs by different code paths. + table.delete(&5).expect("present"); + assert_eq!(table.compact(), 1); + let ranged: Vec = table.range_by_seq(&103..&108).map(|row| row.id).collect(); + assert_eq!(ranged, vec![3, 4, 6, 7], "compaction broke the secondary range"); + assert_eq!(table.select(&7).expect("present").seq, 107); +} + +worktable!( + name: Ticket, + vec: true, + columns: { + id: u64 primary_key using fxhash, + owner: u64, + state: u8, + amount: u64, + }, + indexes: { + owner_idx: owner using fxhash, + amount_idx: amount unique using arctic, + }, + queries: { + update: { + StateById(state) by id, + StateByOwner(state) by owner, + AmountById(amount) by id, + }, + delete: { + ById() by id, + ByOwner() by owner, + }, + in_place: { + Status(state) by id, + }, + }, +); + +/// Declared queries work on a `vec: true` table, and a hash index serves them. +/// +/// Every declared query is an *equality* lookup, which is the shape a hash +/// index is best at. That is why these are emitted whatever the `using` clause +/// says, while `range` and `range_by_` are not: the restriction is ordering, +/// not the query machinery. +/// +/// This table deliberately mixes backends — a `fxhash` primary key, a `fxhash` +/// non-unique secondary, and an `arctic` unique secondary — so a query keyed by +/// each kind runs against a different implementation. +#[test] +fn declared_queries_run_on_a_vec_table() { + let mut table = TicketWorkTable::new(); + for id in 0..6u64 { + table + .insert(TicketRow { + id, + owner: id % 2, + state: 0, + amount: 100 + id, + }) + .expect("fresh"); + } + + // Keyed by the hash primary key: one row. + assert_eq!(table.update_state_by_id(StateByIdQuery { state: 7 }, &3), 1); + assert_eq!(table.select(&3).expect("present").state, 7); + assert_eq!(table.select(&2).expect("present").state, 0, "only one row moved"); + + // Keyed by a non-unique hash secondary: every row it names. + assert_eq!( + table.update_state_by_owner(StateByOwnerQuery { state: 5 }, &1), + 3, + "owner 1 holds ids 1, 3 and 5" + ); + for id in [1u64, 3, 5] { + let row = table.select(&id).expect("present"); + assert_eq!(row.state, 5); + assert_eq!(row.amount, 100 + id); + } + assert_eq!(table.select(&0).expect("present").amount, 100, "owner 0 untouched"); + + assert_eq!(table.update_amount_by_id(AmountByIdQuery { amount: 999 }, &1), 1); + // The unique arctic secondary was repaired without stealing another row's key. + assert!( + table.select_by_amount(&101).is_none(), + "the old amount kept its entry after an update moved the row" + ); + assert_eq!( + table.select_by_amount(&999).expect("present").owner, + 1, + "the updated row owns amount 999" + ); + + // in_place edits one column through a closure. + assert_eq!(table.update_status_in_place(|s| *s = 42, &0), 1); + assert_eq!(table.select(&0).expect("present").state, 42); + + // Deletes, by the key and by a non-unique secondary. + assert_eq!(table.delete_by_id(&0), 1); + assert!(table.select(&0).is_none()); + assert_eq!(table.delete_by_owner(&1), 3, "owner 1 had three rows left"); + assert_eq!(table.len(), 2, "ids 2 and 4 survive"); + assert_eq!(table.ghost_count(), 4, "deletes ghost rather than close the hole"); +} + +#[test] +fn vec_unique_collisions_and_panicking_edits_leave_rows_and_indexes_unchanged() { + use std::panic::{AssertUnwindSafe, catch_unwind}; + + let mut table = HashedSavedWorkTable::new(); + for id in 1..=3u64 { + table + .insert(HashedSavedRow { + id, + code: id * 10, + label: format!("row-{id}"), + }) + .unwrap(); + } + let before = table.unload().unwrap(); + assert!( + catch_unwind(AssertUnwindSafe(|| { + table.update(&1, |row| { + row.id = 2; + row.code = 99; + row.label = "changed".into(); + }); + })) + .is_err() + ); + assert_eq!(table.unload().unwrap(), before); + assert!( + catch_unwind(AssertUnwindSafe(|| { + table.update(&1, |row| { + row.code = 20; + row.label = "changed".into(); + }); + })) + .is_err() + ); + assert_eq!(table.unload().unwrap(), before); + assert!( + catch_unwind(AssertUnwindSafe(|| { + table.upsert(HashedSavedRow { + id: 1, + code: 20, + label: "changed".into(), + }); + })) + .is_err() + ); + assert_eq!(table.unload().unwrap(), before); + assert!( + catch_unwind(AssertUnwindSafe(|| { + table.update(&1, |row| { + row.id = 4; + row.code = 40; + panic!("caller failed"); + }); + })) + .is_err() + ); + assert_eq!(table.unload().unwrap(), before); + for id in 1..=3u64 { + assert_eq!(table.select_by_code(&(id * 10)).unwrap().id, id); + assert_eq!(table.select_by_label(&format!("row-{id}")).len(), 1); + } + table.delete(&1).unwrap(); + table.compact(); + assert_eq!(table.select_by_code(&20).unwrap().id, 2); + assert_eq!(table.select_by_code(&30).unwrap().id, 3); +} + +#[test] +fn vec_declared_query_cannot_steal_another_rows_unique_key() { + let mut table = TicketWorkTable::new(); + for id in 0..2u64 { + table + .insert(TicketRow { + id, + owner: id, + state: 0, + amount: 100 + id, + }) + .unwrap(); + } + let before = table.unload().unwrap(); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + table.update_amount_by_id(AmountByIdQuery { amount: 101 }, &0); + })) + .is_err() + ); + assert_eq!(table.unload().unwrap(), before); + assert_eq!(table.select_by_amount(&100).unwrap().id, 0); + assert_eq!(table.select_by_amount(&101).unwrap().id, 1); +} + +#[test] +fn vec_secondary_key_churn_does_not_retain_empty_posting_lists() { + let mut table = HashedSavedWorkTable::new(); + table + .insert(HashedSavedRow { + id: 1, + code: 1, + label: "initial".into(), + }) + .unwrap(); + table + .insert(HashedSavedRow { + id: 2, + code: 2, + label: "stable".into(), + }) + .unwrap(); + for revision in 0..100 { + assert!(table.update(&1, |row| row.label = format!("edited-{revision}"))); + table.upsert(HashedSavedRow { + id: 1, + code: 1, + label: format!("replaced-{revision}"), + }); + assert_eq!(table.label_map.len(), 2, "secondary index must contain only live keys"); + } + table.delete(&1).unwrap(); + assert_eq!(table.label_map.len(), 1); + assert_eq!(table.select_by_label(&"stable".into())[0].id, 2); + table.delete(&2).unwrap(); + assert!(table.label_map.is_empty()); +} diff --git a/tests/worktable/wrong_row_update.rs b/tests/worktable/wrong_row_update.rs index d29764c1..a8ddf35f 100644 --- a/tests/worktable/wrong_row_update.rs +++ b/tests/worktable/wrong_row_update.rs @@ -49,7 +49,7 @@ async fn unique_update_does_not_mutate_a_row_that_stole_the_value() { table .0 .lock_manager - .insert(pk.clone(), Arc::new(tokio::sync::RwLock::new(blocker_state))); + .insert(pk.clone(), Arc::new(nagoya::sync::RwLock::new(blocker_state))); let update = { let table = table.clone(); diff --git a/tests/wt_default_runtime.rs b/tests/wt_default_runtime.rs new file mode 100644 index 00000000..3d061f75 --- /dev/null +++ b/tests/wt_default_runtime.rs @@ -0,0 +1,54 @@ +//! `WT_DEFAULT_RUNTIME`, end to end. +//! +//! **One test in this file, deliberately.** The selection is cached in a +//! `OnceLock` so that reading it costs one acquire load rather than an +//! allocating `std::env::var` on every spawn, which means the first caller in +//! the process fixes the answer for all of them. A second test here would +//! either race for that slot or silently observe the first one's value, and +//! either way it would be testing the cache rather than the resolution. +//! +//! An integration test is its own binary, so this one owns its process. + +use worktable::prelude::{Flavor, engine_flavor, env_override, parse_selection}; + +#[test] +fn the_environment_selects_the_pool_the_engine_spawns_on() { + // Before anything has read it. Set here rather than through the test + // harness so the test is self-contained and the value is visible in the + // source that asserts on it. + // + // SAFETY: this is the first statement of the only test in this binary, so + // no other thread of this process exists yet to observe the environment + // concurrently. + unsafe { + std::env::set_var("WT_DEFAULT_RUNTIME", "nagoya(spread)"); + } + + assert_eq!(env_override(), Some(Flavor::Spread)); + + // The engine's own background work follows the process-level selection, + // not the locality default it used to be pinned to. A benchmark that + // moved only its client tasks would otherwise leave the flush loop and + // the vacuum sweep on a different scheduler, and the arm would describe a + // stack nobody would ship. + assert_eq!(engine_flavor(), Flavor::Spread); + + // Reading it again cannot change the answer, which is what makes it safe + // to call from the hot path. + assert_eq!(env_override(), Some(Flavor::Spread)); + + // And the value really is cached rather than re-read: changing the + // variable now must not move the selection, or a benchmark could have its + // arm changed underneath it mid-run. + // + // SAFETY: as above, still single-threaded with respect to the environment. + unsafe { + std::env::set_var("WT_DEFAULT_RUNTIME", "nagoya(throughput)"); + } + assert_eq!(env_override(), Some(Flavor::Spread), "the selection is read once"); + + // The parser itself is pure and can be exercised freely. + assert_eq!(parse_selection("nagoya(throughput)").unwrap(), Flavor::Throughput); + assert_eq!(parse_selection("low_latency").unwrap(), Flavor::LowLatency); + assert!(parse_selection("nagoya(banana)").is_err()); +}