diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ecb2281e..695ace57 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -13,6 +13,18 @@ permissions: contents: read jobs: + # `scripts/ci-local.sh` runs `cargo fmt --all --check` as its first step. CI + # did not, which made the script stricter than CI instead of equal to it, and + # formatting drift reached master unnoticed. Same command, same arguments. + fmt: + name: Formatting + runs-on: ubicloud-standard-2 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - name: Formatting (cargo fmt --all --check) + run: cargo fmt --all --check + build: name: Build and test (${{ matrix.name }}) runs-on: ubicloud-standard-2 diff --git a/CHANGELOG.md b/CHANGELOG.md index b37f3eed..98d36fda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,697 @@ Change Log - Persisted primary/secondary index reconstruction and validation failures that could otherwise expose missing, duplicate, or mismatched rows. +## [1.0.0-beta.17] + +### Added + +- `worktable_dsl`, a standalone crate holding the schema language. A schema can + now be read as data and written back, two schemas can be compared and the + cost of the difference reported, and declarations can be found across a + source tree. +- Every generated table embeds its own declaration, so the schema is + recoverable from the code the macro produced. + +### Changed + +- Dependency requirements on the index and reclamation crates are carets rather + than exact pins, and `ps-reclaim` moved to 0.1.1 taken from the registry. +- Retirement runs through the reclamation domain rather than through the guard. + +## [1.0.0-beta.16] + +### Changed + +- A batch pins its reclamation domain once instead of once per row, and + reclamation goes through `ps-reclaim`. +- Requires data_bucket 0.5.5. + +## [1.0.0-beta.15] + +### BC Breaks + +- Non-unique index entries are identified and ordered by their `(key, value)` + pair, and the discriminator is gone. This is a persisted format change. An + index file written by beta.14 or earlier orders entries within a key by + discriminator, so it must be reindexed rather than loaded. + +### Fixed + +- Inserting into a non-unique index no longer scans every entry sharing the + key. On a table that puts a whole generation under one key, a one-file update + measured 698 ms on beta.13 and 15.1 s on beta.14; the per-row cost is back + from 330 us to roughly 9 us. +- Index pages reconstruct in order of their minimum rather than their node id, + so a page that merely ends late no longer sorts ahead of one that starts + earlier. + +## [1.0.0-beta.14] + +### Added + +- `insert_many` with all-or-nothing semantics and CDC batch operations, and + `reserve_pks` for atomic primary key range reservation, both generated on + in-memory and persisted tables. +- Per-table epoch pin domains. The global reader counter is replaced by + epoch-based retirement reclamation, and removed partitions are reclaimed + through the shared router under the same grace period. +- Non-unique Arctic indexes for fixed-width integer keys, generated for + in-memory and persisted tables, with a pair-list checkpoint and WAL. + +### Changed + +- The persistence queue takes batches on a single wakeup, deduplicates page + queries when collecting a multi-row batch, and caps rows per group id so the + analyzer drain stays linear. +- The table-global page barrier is narrowed to one barrier per page. +- Requires WorkTablesIndex 0.0.8 and data_bucket 0.5.4. + +### Fixed + +- Mutation stripes are acquired as a batch without deadlocking. +- A unique-collision unwind on a persisted table survives reload. + +## [1.0.0-beta.13] + +The audit-fix release. Most of it is durability and concurrency correctness +rather than new surface. + +### BC Breaks + +- Persisted tables reject any `page_size` other than 16384 instead of writing a + file that cannot be read back. +- A torn table-of-contents page 1 fails loudly instead of silently starting + from an empty table. +- In-place update is rejected on indexed columns rather than leaving the index + stale. +- An exhausted autoincrement generator panics instead of wrapping around and + handing out keys that are already in use. + +### Fixed + +- A failed data write no longer leaves published index keys behind. Insert, + update and delete each roll their index changes back. +- Index pages are written before the table of contents that references them, + and table-of-contents key updates are guarded against segment overflow. +- Data-file accounting: the u32 page-offset wrap when writing the last page's + data length, files whose length is an exact page multiple failing to reopen, + and non-extending writes being counted into the last page's length. +- Vacuum no longer panics on a failed row move, no longer counts its scratch + pages in `pages_freed`, and never reports a source page fully moved when a + row was skipped. +- A cancelled lock wait releases its registered op-lock, and a row that + vanishes mid-update returns `NotFound` instead of panicking. +- The persistence worker refuses new operations once `Drop` has aborted it, + propagates `insert_cdc` serialization failure instead of panicking, and keeps + surviving data-only writes when event removal empties a batch. +- ART checkpoints are atomic and clean up stale temporaries. +- Arctic returns an empty range for `Excluded` bounds with no neighbour. +- Row counts include inserts and deletes that reused a slot, and the + `PageIsFull` page switch is serialized against racing inserters. +- A misplaced `persist` or `partition_by` in a declaration now names the + position it belongs in. + +## [1.0.0-beta.12] + +### Added + +- `partition_by`: one declared table type, many routed instances, with + `partition_ref` for borrowing a partition rather than cloning it. + +### Changed + +- `system_info` no longer copies every data page, and partition metrics scan + and allocate once instead of three times. + +### Fixed + +- A use-after-free in partition removal. +- `close` reported success having persisted nothing. +- One panic inside the router no longer disables the router. + +## [1.0.0-beta.11] + +### Changed + +- Requires the reviewed ART backend releases. + +## [1.0.0-beta.10] + +### Fixed + +- Table-of-contents inserts carry across persisted segments, reload insertion + stays on the fast path, and the insert API keeps its previous shape. + +## [1.0.0-beta.9] + +### Fixed + +- Persistence health is preserved across page splits. + +## [1.0.0-beta.8] + +### Fixed + +- Multi-row persistence order is preserved, and overlapping durable row writes + are ordered against each other. + +## [1.0.0-beta.7] + +### Fixed + +- The sized indexed update path is preserved. + +## [1.0.0-beta.6] + +### Changed + +- Fixed-width updates stay in place. + +### Fixed + +- Vacuum revalidates links after row locking. + +## [1.0.0-beta.5] + +### Added + +- Checked offline recovery load. + +### Changed + +- WorkTablesIndex structural persistence moved off the mutation path. +- Logical WTI mutation stripes hash with FxHash, reusable data ranges are + subtracted in one pass, and full-row updates generate distinct paths. + +### Fixed + +- Same-size unsized updates apply in place instead of going through reinsert. +- Full-table scans re-resolve stale links. +- Vacuumed pages are reusable after reload. +- Cancelled lock acquirers are cleaned up. +- A panic while loading a persisted table is contained instead of unwinding + into the caller. + +## [1.0.0-beta.4] + +### Fixed + +- Release hardening and torn-store refusal are consolidated, so a torn store + refuses cleanly. + +## [1.0.0-beta.3] + +### Changed + +- Depends on the published index dependency chain rather than git revisions. +- Row publication is concurrency-safe by default. +- Persistence failures are terminal instead of leaving the table in a state + that looks usable. + +### Fixed + +- Synchronous insert is serialized against row mutations. +- Page and link reclamation no longer overlap, and vacuum page reuse is + deferred through the read grace period. +- Upsert retry backoff is bounded and its shift is capped, so same-key churn + cannot livelock. +- Fragmented unsized index pages are compacted. +- A stale multimap removal lookup is avoided. + +## [1.0.0-beta.2] + +### Added + +- Native ART index backends persist. + +### Changed + +- The temporary rusty-s3 fork is retired in favour of the published crate. +- Stable index reads use the specialized path by default. + +### Fixed + +- Same-key upserts linearize. +- Bounded retry for transient index misses is gated rather than always on. +- Reused persistence slots coalesce. + +## [1.0.0-beta.1] + +### Added + +- Per-index backend selection in the `worktable!` declaration, with unique-index + adapters for Arctic, Congee and a parallel upstream indexset. Persistence is + preserved across indexset providers. + +## [0.9.4] + +### Changed + +- Requires data_bucket 0.4.1, and the temporary git patch is retired. + +### Fixed + +- A torn store refuses cleanly instead of terminating the process by signal. + +## [0.9.3] + +### Fixed + +- `worktable_version!` stays read-only when the primary key is unsized. + +## [0.9.2] + +### Fixed + +- Duplicate-key secondary indexes reconstruct correctly on reload. +- Nodes sharing a maximum key order correctly, and pages are no longer re-sorted + on reload. +- Space files flush before an operation reports done. + +## [0.9.1] + +### Changed + +- The proc-macro crate is published as `worktable_codegen` again, after a brief + release under the name `worktable_macros`. + +## [0.9.0] + +### Changed + +- Moves to WorkTablesIndex 0.0.1 and data_bucket 0.4.0. +- The unsound lock-free persistence queue is replaced with a mutexed + `VecDeque`. + +### Fixed + +- Row lock acquisition and vacuum no longer race between check and act. +- `wait_for_ops` no longer returns while a popped operation is still in flight. +- Upsert retries an existence flip instead of surfacing it to the caller. +- A multi-row update locks one validated snapshot, predicate included, and + delete by non-unique index snapshots validated primary keys. +- Gapped event streams are never force-applied to the on-disk index, and the + whole batch is scanned for event-id gaps rather than the last thirty events. +- A failed batch sub-operation is reported without cancelling the rest of the + work. +- `save_batch_data` tracks the real maximum created page id. +- Vacuum persists row moves through CDC, so persisted tables survive + defragmentation. + +## [0.9.0-beta0.2.3] + +### Fixed + +- Primary key generator state is preserved across migration reinserts. + +## [0.9.0-beta0.2.2] + +### Changed + +- Range ordering query logic reworked. + +## [0.9.0-beta0.2.1] + +### Changed + +- Update locks spin before returning a `Pending` state. + +## [0.9.0-beta0.2.0] + +### Added + +- Migrations. + +## [0.9.0-beta0.1.4] + +### Fixed + +- Page-not-found bug in the table of contents. + +## [0.9.0-beta0.1.1] + +### Fixed + +- Persistence bug affecting operations that fail. + +## [0.9.0-alpha8] + +### Changed + +- S3 integration moves to a different client crate. + +## [0.9.0-alpha7] + +### Fixed + +- S3 integration bug. + +## [0.9.0-alpha6] + +### Changed + +- Moves to rustls. + +## [0.9.0-alpha5] + +### Added + +- nanoid support for primary keys. + +## [0.9.0-alpha4] + +### Fixed + +- Vacuum logic. + +## [0.9.0-alpha3] + +### Fixed + +- The S3 macro. + +## [0.9.0-alpha2] + +### Added + +- S3 sync feature. + +## [0.9.0-alpha1] + +### Changed + +- Persistence is moved behind separate traits. + +## [0.8.23] + +### Changed + +- `Lock`s are reworked around RAII guards. + +## [0.8.22] + +### Added + +- `MemStat` derive on the generated primary key type. + +### Changed + +- `DataPages` select is generic over the input link type. + +## [0.8.21] + +### Changed + +- `delete` is generic, matching `insert` and `update`. + +## [0.8.20] + +### Added + +- Vacuum. + +## [0.8.19] + +### Fixed + +- Optional fields in persisted tables. + +## [0.8.18] + +### Fixed + +- Persisted table code failed to compile when the declaration used `optional` + fields. + +## [0.8.17] + +### Changed + +- Updated `indexset`. + +## [0.8.16] + +### Changed + +- Dependencies are pinned to exact versions. + +## [0.8.15] + +### Fixed + +- Empty link registry. + +## [0.8.13] + +### Changed + +- Bumped `indexset`. + +## [0.8.12] + +### Changed + +- Bumped `data_bucket` to 0.3.5 and `wt-indexset` to 0.12.11, and the crate now + declares its repository. + +## [0.8.11] + +### Changed + +- Bumped `indexset`. + +## [0.8.10] + +### Changed + +- Bumped `data_bucket` to 0.3.3 and `wt-indexset` to 0.12.9. + +## [0.8.9] + +### Fixed + +- Empty node bug. + +## [0.8.8] + +### Added + +- Every `AtomicU*` and `AtomicI*` type is usable as a primary key. + +## [0.8.7] + +### Changed + +- Dependency bumps. + +## [0.8.6] + +### Fixed + +- An `update`-related bug. + +## [0.8.5] + +### Fixed + +- Another `update`-related bug. + +## [0.8.4] + +### Changed + +- Codegen version bump. + +## [0.8.3] + +### Fixed + +- `delete` queries on a table whose primary key is not named `id`. +- An update bug, by way of an `indexset` update. + +## [0.8.1] + +### Added + +- The macro reports an error when an index names a column that does not exist, + and declaration errors are raised as `syn::Error`s with usable messages. + +### Fixed + +- `UnsizedNode` split. + +## [0.8.0] + +### Fixed + +- Unsized node bug. + +## [0.7.2] + +### Fixed + +- A further `update` bug. + +## [0.7.1] + +### Fixed + +- An update violation. + +## [0.7.0] + +### Fixed + +- Reinsert bug. + +## [0.6.14] + +### Added + +- Ghost inserts. A row is staged invisible and becomes visible only once its + index entries are in place, so a concurrent reader never observes a + half-inserted row. + +## [0.6.13] + +### Fixed + +- Concurrency bugs in `select`. + +## [0.6.12] + +### Fixed + +- A further locking bug. + +## [0.6.11] + +### Fixed + +- Locking bugs for unsized types, and an `UnsizedNode` bug on `update`. + +### Changed + +- Dependency bumps. + +## [0.6.10] + +### Changed + +- Republished against `worktable_codegen` 0.6.9. No library change. + +## [0.6.9] + +### Fixed + +- Concurrent persistence issues. + +## [0.6.8] + +### Fixed + +- `wait_for_ops` logic. + +## [0.6.7] + +### Fixed + +- `delete` on persisted tables. + +## [0.6.5] + +### Added + +- Custom derives can be attached to the generated row type. + +## [0.6.4] + +### Fixed + +- `uuid` usage. + +## [0.6.3] + +### Fixed + +- A debug `println!` on the persistence batch path no longer writes to stdout. + +## [0.6.2] + +### Fixed + +- Table-of-contents corrections. + +## [0.6.1] + +### Added + +- `update_in_place`. + +### Changed + +- The persistence queue is optimized. + +### Fixed + +- `insert` with an already-existing key. +- A `use rkyv::Archive` import was required for some declarations. +- `wait_for_ops`. + +## [0.5.6] + +### Changed + +- Updated `indexset`. + +## [0.5.5] + +### Fixed + +- Array-typed fields. + +### Changed + +- Moves to the newer Rust edition. + +## [0.5.4] + +### Added + +- Unsized index space, so index keys are no longer limited to fixed-width + types. +- `SystemInfo` for the table and its indexes. +- `where_by` on `SelectBuilder` for any column, indexed or not. +- Float columns are usable in indexes, including ranges. + +### Fixed + +- Re-reading a table from file. +- Index difference logic for `update` queries. + +## [0.5.1] + +### Changed + +- Persistence I/O is asynchronous. + +## [0.5.0] + +### Added + +- `select_where_{field}` queries for selecting data ranges. +- `count` on the table. +- Persist sync logic. + +### Changed + +- Non-unique indexes are backed by `IndexMultiMap`. + +### Fixed + +- Secondary index left inconsistent after an update. +- Diff logic for a full-row update. + ## [0.4.1] ### Added diff --git a/Cargo.toml b/Cargo.toml index e32eb50c..c08a4e56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,17 @@ keywords = ["database", "embedded", "in-memory", "index", "storage"] categories = ["database-implementations", "data-structures", "caching"] [features] -default = ["wti-predictable-search"] +default = ["std", "wti-predictable-search", "vanilla-index"] +# `futures/std` belongs here rather than on the dependency +# line: written as `features = ["std"]` beside the version, a +# `--no-default-features` build of this crate would still turn them on. The +# s3 tests reach `futures::io`, whose traits live behind that feature. +std = ["nagoya/std", "ps-reclaim/std", "congee/std", "indexset/std", "futures/std"] +# The upstream IndexSet backend, selectable per index with `using indexset`. +# Optional because it reaches `crossbeam-utils`, `parking_lot` and `serde`, +# none of which build without `std`, and because it is one backend of four +# rather than something the table needs. +vanilla-index = ["dep:vanilla_indexset"] perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"] s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktable_codegen/s3-support"] # Moves unique WorkTablesIndex structural CDC work out of the table mutation @@ -35,36 +45,52 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"] [dependencies] # Read-mostly snapshots own page Arcs while the fixed directory supplies a # pointer-only fast path. Publication is append-only and asserted at each swap. -arc-swap = "1" +arc-swap = { version = "1", default-features = false } async-trait = "0.1" -arctic = { package = "arctic-wt", version = "^0.1, >=0.1.9", default-features = false, features = ["smr-ps-reclaim"] } -congee = { package = "congee-wt", version = "^0.4, >=0.4.4" } -convert_case = "0.6" -crc32fast = "1" -data_bucket = { version = "^0.5, >=0.5.7" } -derive_more = { version = "2", features = ["from", "error", "display", "debug", "into"] } +arctic = { package = "arctic-wt", version = "^0.1, >=0.1.11", default-features = false, features = ["smr-ps-reclaim"] } +# `default-features = false`, or its `std` turns `ps-reclaim/std` on and this +# crate links the standard library through a dependency. That matters for +# EKOPathRS bootstrapping, which is what `no_std` here is for, not for any +# bare-metal target: we ship on Linux, macOS and Windows. +congee = { package = "congee-wt", version = "^0.4, >=0.4.4", default-features = false } +convert_case = { version = "0.6", default-features = false } +crc32fast = { version = "1", default-features = false } +# 0.6 because the page stride is a parameter now and the error type is +# concrete: `^0.5` cannot resolve the crate this depends on. +data_bucket = { version = "^0.6" } +derive_more = { version = "2", default-features = false, features = ["from", "error", "display", "debug", "into"] } eyre = "0.6" fastrand = "2" -futures = "0.3" +hashbrown = "0.15" +futures = { version = "0.3", default-features = false, features = ["alloc"] } +# The file traits and the host implementation. `std` here; the crate takes it +# without default features wherever the filesystem is not involved. +nagoya = { version = "0.1", default-features = false } indexset = { package = "WorkTablesIndex", version = "^0.0, >=0.0.12", default-features = false, features = ["concurrent", "cdc", "multimap"] } -vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"] } +# Pulls `serde` and `serde_core` into every build, and nothing here uses them. +# They are not reachable from this line: `indexset` 0.15 declares `ftree` with +# `features = ["serde"]` unconditionally, so no `default-features = false` here +# removes them and forking `ftree` would not either. This is one of four +# selectable index backends and making it optional is the fix. +vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"], optional = true } # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "^0.12", features = ["concurrent", "cdc", "multimap"] } -log = "0.4" -ordered-float = "5" +log = { version = "0.4", default-features = false } +ordered-float = { version = "5", default-features = false } parking_lot = "0.12" performance_measurement = { path = "performance_measurement", version = "^0.1", optional = true } performance_measurement_codegen = { path = "performance_measurement/codegen", version = "^0.1", optional = true } -prettytable-rs = "0.10" psc-nanoid = { version = "3", features = ["rkyv", "packed"] } -rkyv = { version = "0.8", features = ["uuid-1"] } +rkyv = { version = "0.8", default-features = false, features = ["alloc", "bytecheck", "uuid-1"] } reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] } -ps-reclaim = { version = "^0.1, >=0.1.3" } -rustc-hash = "2" +# `spin` rather than `std`: without it this crate reaches for `thread_local!`, +# which needs a platform. See its `slot` module for what it falls back to. +ps-reclaim = { version = "^0.1, >=0.1.4", default-features = false, features = ["spin"] } +rustc-hash = { version = "2", default-features = false } rusty-s3 = { version = "0.10", optional = true } smart-default = "0.7" tokio = { version = "1", features = ["full"] } -tracing = "0.1" +tracing = { version = "0.1", default-features = false } url = { version = "2", optional = true } uuid = { version = "1", features = ["v4", "v7"] } walkdir = { version = "2", optional = true } @@ -95,3 +121,16 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(wt_loom)'] } [[bench]] name = "worktable_benchmarks" harness = false + + +# None of these three are published yet, and `[patch]` only takes effect from +# the workspace root, so the whole chain has to be named here: a dependency's +# own patches do not reach its consumer. Without this the branch resolves +# nowhere, for a reviewer and for CI alike. +# +# Delete as each publishes: ps-st3 0.5.1, then nagoya 0.1.0, then data_bucket +# 0.6.0. +[patch.crates-io] +data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "feat/tunable-page-stride" } +nagoya = { git = "https://github.com/pathscale/nagoya", branch = "feat/timers" } +ps-st3 = { git = "https://github.com/pathscale/ps-st3", branch = "perf/skip-the-wake-when-nobody-sleeps" } diff --git a/benches/cases/full_featured.rs b/benches/cases/full_featured.rs index 0e5b222d..69c95bfa 100644 --- a/benches/cases/full_featured.rs +++ b/benches/cases/full_featured.rs @@ -19,7 +19,7 @@ fn insert(c: &mut Criterion) { another: format!("another_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(black_box(row))) + nagoya::block_on(table.insert(black_box(row))) }, BatchSize::SmallInput, ) @@ -37,7 +37,7 @@ fn select_by_pk(c: &mut Criterion) { another: format!("another_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -60,7 +60,7 @@ fn select_by_unique_index(c: &mut Criterion) { another: format!("another_{}", i), something: i, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("full_featured_select_by_val1", |b| { @@ -82,7 +82,7 @@ fn select_by_non_unique_index(c: &mut Criterion) { another: format!("cat_{}", i % 10), something: i, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("full_featured_select_by_another", |b| { @@ -196,7 +196,7 @@ fn in_place_update(c: &mut Criterion) { another: "test".to_string(), something: 0, }; - futures::executor::block_on(table.insert(row)).unwrap().into() + nagoya::block_on(table.insert(row)).unwrap().into() }; c.bench_function("full_featured_in_place_update_val", |b| { @@ -219,7 +219,7 @@ fn delete(c: &mut Criterion) { another: format!("temp_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: FullFeaturedPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -242,7 +242,7 @@ fn delete_by_index_query(c: &mut Criterion) { another: another.clone(), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); another }, |another: String| rt.block_on(async { table.delete_by_another(another).await.unwrap() }), @@ -319,7 +319,7 @@ fn batch_insert(c: &mut Criterion) { another: format!("another_{}", i), something: i as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -347,7 +347,7 @@ fn batch_select_pk(c: &mut Criterion) { another: format!("another_{}", fastrand::u64(..)), something: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/benches/cases/non_unique_index.rs b/benches/cases/non_unique_index.rs index 7e5f2038..c2589030 100644 --- a/benches/cases/non_unique_index.rs +++ b/benches/cases/non_unique_index.rs @@ -33,7 +33,7 @@ fn select_by_pk(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -54,7 +54,7 @@ fn select_by_non_unique_index(c: &mut Criterion) { value: fastrand::u64(..), category: i % 10, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("non_unique_index_select_by_category", |b| { @@ -108,7 +108,7 @@ fn delete(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: NonUniqueIndexPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -176,7 +176,7 @@ fn batch_insert(c: &mut Criterion) { value: i as u64, category: (i % 10) as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -202,7 +202,7 @@ fn batch_select_pk(c: &mut Criterion) { value: fastrand::u64(..), category: fastrand::u64(0..10), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/benches/cases/nonunique_arctic_vs_wti.rs b/benches/cases/nonunique_arctic_vs_wti.rs index 6ea1d8a7..9d4e19ed 100644 --- a/benches/cases/nonunique_arctic_vs_wti.rs +++ b/benches/cases/nonunique_arctic_vs_wti.rs @@ -84,7 +84,7 @@ fn select_by_key(c: &mut Criterion) { for (fan_out, keys) in SHAPES { group.throughput(Throughput::Elements(fan_out)); - let table = futures::executor::block_on(populated_wti(fan_out, keys)); + let table = nagoya::block_on(populated_wti(fan_out, keys)); group.bench_with_input(BenchmarkId::new("wti_string", fan_out), &fan_out, |b, _| { b.iter(|| { let key = string_key(fastrand::u64(0..keys)); @@ -92,7 +92,7 @@ fn select_by_key(c: &mut Criterion) { }) }); - let table = futures::executor::block_on(populated_arctic(fan_out, keys)); + let table = nagoya::block_on(populated_arctic(fan_out, keys)); group.bench_with_input(BenchmarkId::new("arctic_u128", fan_out), &fan_out, |b, _| { b.iter(|| { let key = hash_of(fastrand::u64(0..keys)); @@ -108,7 +108,7 @@ fn insert(c: &mut Criterion) { for (fan_out, keys) in SHAPES { // Steady state: the table already holds `fan_out` rows per key and // each measured insert lands on an existing key. - let table = futures::executor::block_on(populated_wti(fan_out, keys)); + let table = nagoya::block_on(populated_wti(fan_out, keys)); group.bench_with_input(BenchmarkId::new("wti_string", fan_out), &fan_out, |b, _| { b.iter_batched( || WtiStringAdjacencyRow { @@ -116,12 +116,12 @@ fn insert(c: &mut Criterion) { source: string_key(fastrand::u64(0..keys)), payload: u64::MAX, }, - |row| futures::executor::block_on(table.insert(black_box(row))).unwrap(), + |row| nagoya::block_on(table.insert(black_box(row))).unwrap(), BatchSize::SmallInput, ) }); - let table = futures::executor::block_on(populated_arctic(fan_out, keys)); + let table = nagoya::block_on(populated_arctic(fan_out, keys)); group.bench_with_input(BenchmarkId::new("arctic_u128", fan_out), &fan_out, |b, _| { b.iter_batched( || ArcticHashAdjacencyRow { @@ -129,7 +129,7 @@ fn insert(c: &mut Criterion) { source: hash_of(fastrand::u64(0..keys)), payload: u64::MAX, }, - |row| futures::executor::block_on(table.insert(black_box(row))).unwrap(), + |row| nagoya::block_on(table.insert(black_box(row))).unwrap(), BatchSize::SmallInput, ) }); diff --git a/benches/cases/partition_routing.rs b/benches/cases/partition_routing.rs index 7d59b03a..a207634d 100644 --- a/benches/cases/partition_routing.rs +++ b/benches/cases/partition_routing.rs @@ -46,7 +46,7 @@ async fn populated() -> RoutePartitions { /// The four ways to reach a partition, single threaded, one hot key. fn lookup(c: &mut Criterion) { - let routes = futures::executor::block_on(populated()); + let routes = nagoya::block_on(populated()); let cached = routes.partition(7).unwrap(); let mut group = c.benchmark_group("partition_lookup"); @@ -87,7 +87,7 @@ fn contended(c: &mut Criterion, name: &str, same_key: bool) { for api in ["partition_ref", "pinned_get", "partition_arc"] { group.bench_with_input(BenchmarkId::new(api, threads), &threads, |b, &threads| { b.iter_custom(|iters| { - let routes = Arc::new(futures::executor::block_on(populated())); + let routes = Arc::new(nagoya::block_on(populated())); let go = Arc::new(AtomicBool::new(false)); let workers: Vec<_> = (0..threads) @@ -151,7 +151,7 @@ fn distinct_key_readers(c: &mut Criterion) { /// Accounting over 500 partitions. These were routed through `system_info`, /// which copied every data page, and through `keys` then `partition` per key. fn metrics(c: &mut Criterion) { - let routes = futures::executor::block_on(populated()); + let routes = nagoya::block_on(populated()); let mut group = c.benchmark_group("partition_metrics"); group.bench_function("memory_total", |b| b.iter(|| black_box(routes.memory_total()))); group.bench_function("rows_by_key", |b| b.iter(|| black_box(routes.rows_by_key()))); diff --git a/benches/cases/simple.rs b/benches/cases/simple.rs index a035b53a..afbcc353 100644 --- a/benches/cases/simple.rs +++ b/benches/cases/simple.rs @@ -15,7 +15,7 @@ fn insert(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(black_box(row))) + nagoya::block_on(table.insert(black_box(row))) }, BatchSize::SmallInput, ) @@ -30,7 +30,7 @@ fn select_by_pk(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -82,7 +82,7 @@ fn delete(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: SimplePrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -146,7 +146,7 @@ fn batch_insert(c: &mut Criterion) { id: table.get_next_pk().into(), value: i as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -171,7 +171,7 @@ fn batch_select_pk(c: &mut Criterion) { id: table.get_next_pk().into(), value: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/benches/cases/unique_index.rs b/benches/cases/unique_index.rs index 7509ae63..73144ff2 100644 --- a/benches/cases/unique_index.rs +++ b/benches/cases/unique_index.rs @@ -52,7 +52,7 @@ fn select_by_pk(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); @@ -73,7 +73,7 @@ fn select_by_unique_index(c: &mut Criterion) { test: i, another: i as u64, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("unique_index_select_by_test", |b| { @@ -93,7 +93,7 @@ fn select_by_unique_index_range(c: &mut Criterion) { test: i, another: i as u64, }; - futures::executor::block_on(table.insert(row)).unwrap(); + nagoya::block_on(table.insert(row)).unwrap(); } c.bench_function("unique_index_select_by_test_range", |b| { @@ -109,8 +109,8 @@ fn art_primary_key_ranges(c: &mut Criterion) { let congee = CongeeRangeBenchmarkWorkTable::default(); let arctic = ArcticRangeBenchmarkWorkTable::default(); for id in 0..ROWS { - futures::executor::block_on(congee.insert(CongeeRangeBenchmarkRow { id, value: id })).unwrap(); - futures::executor::block_on(arctic.insert(ArcticRangeBenchmarkRow { id, value: id })).unwrap(); + nagoya::block_on(congee.insert(CongeeRangeBenchmarkRow { id, value: id })).unwrap(); + nagoya::block_on(arctic.insert(ArcticRangeBenchmarkRow { id, value: id })).unwrap(); } let mut group = c.benchmark_group("art_primary_key_single_row_range"); @@ -173,7 +173,7 @@ fn delete(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }, |pk: UniqueIndexPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }), BatchSize::SmallInput, @@ -241,7 +241,7 @@ fn batch_insert(c: &mut Criterion) { test: i as i64, another: i as u64, }; - futures::executor::block_on(table.insert(black_box(row))).unwrap(); + nagoya::block_on(table.insert(black_box(row))).unwrap(); } black_box(table) }, @@ -267,7 +267,7 @@ fn batch_select_pk(c: &mut Criterion) { test: fastrand::i64(..), another: fastrand::u64(..), }; - futures::executor::block_on(table.insert(row)).unwrap() + nagoya::block_on(table.insert(row)).unwrap() }) .collect(); diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index cab40835..5057c7f4 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -24,9 +24,17 @@ proc-macro = true # policy. beta.18.1 is the first DSL artifact that exports every validator this # code generator calls; beta.18 was published before that API landed. worktable_dsl = { path = "../dsl", version = "^1.0.0-beta.18.1" } -rkyv = { version = "0.8" } +# Test-only. As a normal dependency this proc-macro crate put `rkyv` with its +# default features into the graph, which turned on `rkyv/std` for the target +# build too and dragged `ptr_meta` with it. The generated code names `rkyv` +# paths through `quote!`, which needs no dependency here at all. +# +# See the `[dev-dependencies]` section below. syn = { version = "2", features = ["full"] } quote = "1" proc-macro2 = "1" convert_case = "0.6" indexmap = "2" + +[dev-dependencies] +rkyv = { version = "0.8" } diff --git a/codegen/src/generators/in_memory/index/cdc.rs b/codegen/src/generators/in_memory/index/cdc.rs index ed85e087..3394f3f5 100644 --- a/codegen/src/generators/in_memory/index/cdc.rs +++ b/codegen/src/generators/in_memory/index/cdc.rs @@ -311,7 +311,7 @@ impl InMemoryGenerator { fn process_difference_remove_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { #(#process_difference_rows)* (#events_ident { @@ -381,7 +381,7 @@ impl InMemoryGenerator { fn process_difference_insert_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; let mut partial_events = #events_ident::default(); diff --git a/codegen/src/generators/in_memory/index/usual.rs b/codegen/src/generators/in_memory/index/usual.rs index 2c90abc7..5329a29e 100644 --- a/codegen/src/generators/in_memory/index/usual.rs +++ b/codegen/src/generators/in_memory/index/usual.rs @@ -245,7 +245,7 @@ impl InMemoryGenerator { fn process_difference_remove( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { #(#process_difference_remove_rows)* core::result::Result::Ok(()) @@ -304,7 +304,7 @@ impl InMemoryGenerator { fn process_difference_insert( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { let mut inserted_indexes: Vec<#avt_index_ident> = vec![]; #(#process_difference_insert_rows)* diff --git a/codegen/src/generators/in_memory/locks.rs b/codegen/src/generators/in_memory/locks.rs index 78b40aec..1153c65c 100644 --- a/codegen/src/generators/in_memory/locks.rs +++ b/codegen/src/generators/in_memory/locks.rs @@ -24,7 +24,7 @@ impl InMemoryGenerator { .keys() .map(|i| { let name = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); - quote! { #name: Option>, } + quote! { #name: Option>, } }) .collect(); @@ -124,8 +124,8 @@ impl InMemoryGenerator { .collect(); quote! { - fn with_lock(id: u16) -> (Self, std::sync::Arc) { - let lock = std::sync::Arc::new(Lock::new(id)); + fn with_lock(id: u16) -> (Self, worktable::prelude::Arc) { + let lock = worktable::prelude::Arc::new(Lock::new(id)); ( Self { #(#rows),* @@ -154,9 +154,9 @@ impl InMemoryGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let lock = std::sync::Arc::new(Lock::new(id)); + fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* (set, lock) @@ -186,8 +186,8 @@ impl InMemoryGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> std::collections::HashSet> { - let mut set = std::collections::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { + let mut set = worktable::prelude::HashSet::new(); #(#rows)* set } diff --git a/codegen/src/generators/in_memory/primary_key.rs b/codegen/src/generators/in_memory/primary_key.rs index 8d58b84f..ac4db3e9 100644 --- a/codegen/src/generators/in_memory/primary_key.rs +++ b/codegen/src/generators/in_memory/primary_key.rs @@ -140,14 +140,14 @@ impl InMemoryGenerator { /// atomic of primitive. fn get_generator_from_type(type_: &TokenStream, i: &Ident) -> syn::Result { Ok(match type_.to_string().as_str() { - "u8" => quote! { std::sync::atomic::AtomicU8 }, - "u16" => quote! { std::sync::atomic::AtomicU16 }, - "u32" => quote! { std::sync::atomic::AtomicU32 }, - "u64" => quote! { std::sync::atomic::AtomicU64 }, - "i8" => quote! { std::sync::atomic::AtomicI8 }, - "i16" => quote! { std::sync::atomic::AtomicI16 }, - "i32" => quote! { std::sync::atomic::AtomicI32 }, - "i64" => quote! { std::sync::atomic::AtomicI64 }, + "u8" => quote! { core::sync::atomic::AtomicU8 }, + "u16" => quote! { core::sync::atomic::AtomicU16 }, + "u32" => quote! { core::sync::atomic::AtomicU32 }, + "u64" => quote! { core::sync::atomic::AtomicU64 }, + "i8" => quote! { core::sync::atomic::AtomicI8 }, + "i16" => quote! { core::sync::atomic::AtomicI16 }, + "i32" => quote! { core::sync::atomic::AtomicI32 }, + "i64" => quote! { core::sync::atomic::AtomicI64 }, // The accepted set is `worktable_dsl::AUTOINCREMENT_TYPES`, and the // arms above must stay equal to it. `check` uses that list to // answer "would the macro accept this", so a second copy drifting diff --git a/codegen/src/generators/in_memory/queries/delete.rs b/codegen/src/generators/in_memory/queries/delete.rs index 911f7e9a..efcde879 100644 --- a/codegen/src/generators/in_memory/queries/delete.rs +++ b/codegen/src/generators/in_memory/queries/delete.rs @@ -186,7 +186,7 @@ impl InMemoryGenerator { quote! { pub async fn #name(&self, by: #type_) -> core::result::Result<(), WorkTableError> { let _bulk_mutation = self.0.lock_manager.bulk_mutation_guard(); - let pks = std::cell::RefCell::new(Vec::new()); + let pks = core::cell::RefCell::new(Vec::new()); self.iter_with(|row| { if row.#field == by { pks.borrow_mut().push(row.get_primary_key()); diff --git a/codegen/src/generators/in_memory/queries/locks.rs b/codegen/src/generators/in_memory/queries/locks.rs index cf80019f..11e15c77 100644 --- a/codegen/src/generators/in_memory/queries/locks.rs +++ b/codegen/src/generators/in_memory/queries/locks.rs @@ -95,9 +95,9 @@ impl InMemoryGenerator { quote! { #[allow(clippy::mutable_key_type)] - pub fn #ident(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let new_lock = std::sync::Arc::new(Lock::new(id)); + pub fn #ident(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let new_lock = worktable::prelude::Arc::new(Lock::new(id)); #(#inner)* (set, new_lock) } diff --git a/codegen/src/generators/in_memory/queries/select.rs b/codegen/src/generators/in_memory/queries/select.rs index 42139128..744cd0ea 100644 --- a/codegen/src/generators/in_memory/queries/select.rs +++ b/codegen/src/generators/in_memory/queries/select.rs @@ -32,7 +32,7 @@ impl InMemoryGenerator { // Acquire the grace-period guard only when iteration starts. // Merely constructing and retaining a query builder must not // stall retired-link reclamation. - let iter = std::iter::once_with(move || { + let iter = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .iter_values() @@ -55,7 +55,7 @@ impl InMemoryGenerator { return None; } current_link = replacement; - std::hint::spin_loop(); + core::hint::spin_loop(); } None }) diff --git a/codegen/src/generators/in_memory/queries/update.rs b/codegen/src/generators/in_memory/queries/update.rs index 04af1554..a10c90bf 100644 --- a/codegen/src/generators/in_memory/queries/update.rs +++ b/codegen/src/generators/in_memory/queries/update.rs @@ -42,7 +42,7 @@ impl InMemoryGenerator { .keys() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -273,8 +273,8 @@ impl InMemoryGenerator { let avt_type_ident = name_generator.get_available_type_ident(); quote! { if let core::result::Result::Err(e) = #write { - let mut reversed_diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = - std::collections::HashMap::new(); + let mut reversed_diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = + worktable::prelude::HashMap::new(); for (key, diff) in diffs { reversed_diffs.insert(key, Difference { old: diff.new, new: diff.old }); } @@ -460,7 +460,7 @@ impl InMemoryGenerator { let row_old = self.0.data.select_non_ghosted(link)?; let row_new = row.clone(); let updated_bytes: Vec = vec![]; - let mut diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = std::collections::HashMap::new(); + let mut diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = worktable::prelude::HashMap::new(); } } else { quote! { @@ -605,7 +605,7 @@ impl InMemoryGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -680,7 +680,7 @@ impl InMemoryGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -830,7 +830,7 @@ impl InMemoryGenerator { pks.sort_unstable(); pks.dedup(); - let mut guards: std::collections::HashMap<_, _> = std::collections::HashMap::new(); + let mut guards: worktable::prelude::HashMap<_, _> = worktable::prelude::HashMap::new(); // Full-row locks, not per-column custom locks: each row's // unsized reinsert path mutates the whole row under these // guards, and one uniform lock kind keeps every concurrent @@ -902,7 +902,7 @@ impl InMemoryGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index 198e390a..9d4554de 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -109,7 +109,7 @@ impl InMemoryGenerator { #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds + 'a, + R: core::ops::RangeBounds + 'a, Pk: Clone + 'a, { let converted_range = ( @@ -118,7 +118,7 @@ impl InMemoryGenerator { ); // Delay the grace-period guard until the returned iterator is // consumed so an idle query builder cannot pin reclamation. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .range_links(converted_range) @@ -302,7 +302,7 @@ impl InMemoryGenerator { let exponent = core::cmp::min(backoff_spins - 8, 8); let micros = core::cmp::min(1u64 << exponent, 256); backoff_spins = backoff_spins.saturating_add(1); - tokio::time::sleep(std::time::Duration::from_micros(micros)).await; + tokio::time::sleep(core::time::Duration::from_micros(micros)).await; } } } @@ -343,7 +343,7 @@ impl InMemoryGenerator { /// assigned contiguous keys before `insert_many`. Interleaved /// `get_next_pk` calls keep working and never overlap a /// reservation. - pub fn reserve_pks(&self, count: usize) -> std::ops::Range<#pk_inner_type> { + pub fn reserve_pks(&self, count: usize) -> core::ops::Range<#pk_inner_type> { self.0.reserve_pks(count) } } @@ -375,7 +375,7 @@ impl InMemoryGenerator { quote! { pub async fn iter_with_async< F: Fn(#row_type) -> Fut, - Fut: std::future::Future> + Fut: core::future::Future> >(&self, f: F) -> core::result::Result<(), WorkTableError> { #inner } @@ -436,8 +436,8 @@ impl InMemoryGenerator { let lock_type = name_generator.get_lock_type_ident(); quote! { - pub fn vacuum(&self) -> std::sync::Arc { - std::sync::Arc::new(EmptyDataVacuum::< + pub fn vacuum(&self) -> worktable::prelude::Arc { + worktable::prelude::Arc::new(EmptyDataVacuum::< _, _, _, @@ -448,10 +448,10 @@ impl InMemoryGenerator { _ >::new( #table_name, - std::sync::Arc::clone(&self.0.data), - std::sync::Arc::clone(&self.0.lock_manager), - std::sync::Arc::clone(&self.0.primary_index), - std::sync::Arc::clone(&self.0.indexes), + worktable::prelude::Arc::clone(&self.0.data), + worktable::prelude::Arc::clone(&self.0.lock_manager), + worktable::prelude::Arc::clone(&self.0.primary_index), + worktable::prelude::Arc::clone(&self.0.indexes), )) } } diff --git a/codegen/src/generators/in_memory/table/index_fns.rs b/codegen/src/generators/in_memory/table/index_fns.rs index 35d51744..1317e50c 100644 --- a/codegen/src/generators/in_memory/table/index_fns.rs +++ b/codegen/src/generators/in_memory/table/index_fns.rs @@ -99,7 +99,7 @@ impl InMemoryGenerator { if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None }; @@ -140,7 +140,7 @@ impl InMemoryGenerator { #column_range_type, #row_fields_ident> { - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.indexes.#field_ident .get(#by) @@ -173,7 +173,7 @@ impl InMemoryGenerator { let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { ( predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), @@ -183,7 +183,7 @@ impl InMemoryGenerator { ) } else { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { predicate_range.clone() }, ) }; @@ -216,7 +216,7 @@ impl InMemoryGenerator { }; let predicate_filter = quote! { .filter(move |row| { - std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + core::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) }) }; @@ -231,7 +231,7 @@ impl InMemoryGenerator { #predicate_setup // Query construction is not an active read. Pin the grace // period on the first row lookup instead. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); #index_range .filter_map(#select_row) diff --git a/codegen/src/generators/in_memory/table/mod.rs b/codegen/src/generators/in_memory/table/mod.rs index d5bd9e39..10d3768c 100644 --- a/codegen/src/generators/in_memory/table/mod.rs +++ b/codegen/src/generators/in_memory/table/mod.rs @@ -126,7 +126,7 @@ impl InMemoryGenerator { #index_type, #lock_ident, <#primary_key_type as TablePrimaryKey>::Generator, - { INNER_PAGE_SIZE }, + { #inner_const_name }, #node_type > ); diff --git a/codegen/src/generators/in_memory/table/select_executor.rs b/codegen/src/generators/in_memory/table/select_executor.rs index 0dd2560b..4538de0e 100644 --- a/codegen/src/generators/in_memory/table/select_executor.rs +++ b/codegen/src/generators/in_memory/table/select_executor.rs @@ -44,7 +44,7 @@ impl InMemoryGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - #variant_ident(std::ops::#range_ident<#ty_ident>), + #variant_ident(core::ops::#range_ident<#ty_ident>), } }) .collect(); @@ -65,8 +65,8 @@ impl InMemoryGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - impl From> for #column_range_type { - fn from(range: std::ops::#range_ident<#ty_ident>) -> Self { + impl From> for #column_range_type { + fn from(range: core::ops::#range_ident<#ty_ident>) -> Self { Self::#variant_ident(range) } } @@ -100,8 +100,8 @@ impl InMemoryGenerator { let col_ident = Ident::new(&column.to_string(), Span::call_site()); quote! { #row_fields_ident::#column_variant => { - let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(std::cmp::Ordering::Equal); - if cmp != std::cmp::Ordering::Equal { + let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(core::cmp::Ordering::Equal); + if cmp != core::cmp::Ordering::Equal { return match order { Order::Asc => cmp, Order::Desc => cmp.reverse(), @@ -165,7 +165,7 @@ impl InMemoryGenerator { _ => continue, } } - std::cmp::Ordering::Equal + core::cmp::Ordering::Equal }); iter = Box::new(items.into_iter()); }; diff --git a/codegen/src/generators/partitions.rs b/codegen/src/generators/partitions.rs index 5cf07732..5fed340e 100644 --- a/codegen/src/generators/partitions.rs +++ b/codegen/src/generators/partitions.rs @@ -38,7 +38,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok pub fn partition_or_create( &self, #key_name: #key_ty, - ) -> Result, worktable::partition::PartitionError> { + ) -> Result, worktable::partition::PartitionError> { self.inner.get_or_create(#key_name as u64, <#table as Default>::default) } } @@ -82,7 +82,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok /// The partition routed to by `#key_name`, if it exists. #[inline] - pub fn partition(&self, #key_name: #key_ty) -> Option> { + pub fn partition(&self, #key_name: #key_ty) -> Option> { self.inner.partition(#key_name as u64) } @@ -94,7 +94,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok &self, #key_name: #key_ty, make: F, - ) -> Result, worktable::partition::PartitionError> + ) -> Result, worktable::partition::PartitionError> where F: FnOnce() -> #table, { @@ -146,7 +146,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok /// grace period). Removal and creation reclaim opportunistically, /// so a router shared behind an `Arc` does not accumulate removed /// partitions; `collect` is available for removal-only phases. - pub fn remove(&self, #key_name: #key_ty) -> Option> { + pub fn remove(&self, #key_name: #key_ty) -> Option> { self.inner.remove(#key_name as u64) } @@ -156,7 +156,7 @@ pub fn expand(name: &Ident, key: &PartitionKey, persistence: Persistence) -> Tok } /// Every live partition with its key. - pub fn iter(&self) -> Vec<(#key_ty, std::sync::Arc<#table>)> { + pub fn iter(&self) -> Vec<(#key_ty, worktable::prelude::Arc<#table>)> { self.inner .iter() .into_iter() diff --git a/codegen/src/generators/persist/index/cdc.rs b/codegen/src/generators/persist/index/cdc.rs index 8aee8d6a..69bde2a4 100644 --- a/codegen/src/generators/persist/index/cdc.rs +++ b/codegen/src/generators/persist/index/cdc.rs @@ -338,7 +338,7 @@ impl PersistGenerator { fn process_difference_remove_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { #(#process_difference_rows)* (#events_ident { @@ -408,7 +408,7 @@ impl PersistGenerator { fn process_difference_insert_cdc( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> (#events_ident, Result<(), IndexError<#available_index_ident>>) { let mut inserted_indexes: Vec<#available_index_ident> = vec![]; let mut partial_events = #events_ident::default(); diff --git a/codegen/src/generators/persist/index/usual.rs b/codegen/src/generators/persist/index/usual.rs index e8629cc8..19663c79 100644 --- a/codegen/src/generators/persist/index/usual.rs +++ b/codegen/src/generators/persist/index/usual.rs @@ -236,7 +236,7 @@ impl PersistGenerator { fn process_difference_remove( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { #(#process_difference_remove_rows)* core::result::Result::Ok(()) @@ -293,7 +293,7 @@ impl PersistGenerator { fn process_difference_insert( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { let mut inserted_indexes: Vec<#avt_index_ident> = vec![]; #(#process_difference_insert_rows)* diff --git a/codegen/src/generators/persist/locks.rs b/codegen/src/generators/persist/locks.rs index 92a88a0c..86b55b60 100644 --- a/codegen/src/generators/persist/locks.rs +++ b/codegen/src/generators/persist/locks.rs @@ -24,7 +24,7 @@ impl PersistGenerator { .keys() .map(|i| { let name = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); - quote! { #name: Option>, } + quote! { #name: Option>, } }) .collect(); @@ -124,8 +124,8 @@ impl PersistGenerator { .collect(); quote! { - fn with_lock(id: u16) -> (Self, std::sync::Arc) { - let lock = std::sync::Arc::new(Lock::new(id)); + fn with_lock(id: u16) -> (Self, worktable::prelude::Arc) { + let lock = worktable::prelude::Arc::new(Lock::new(id)); ( Self { #(#rows),* @@ -154,9 +154,9 @@ impl PersistGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let lock = std::sync::Arc::new(Lock::new(id)); + fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* (set, lock) @@ -186,8 +186,8 @@ impl PersistGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> std::collections::HashSet> { - let mut set = std::collections::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { + let mut set = worktable::prelude::HashSet::new(); #(#rows)* set } diff --git a/codegen/src/generators/persist/primary_key.rs b/codegen/src/generators/persist/primary_key.rs index 59b002b9..47f3192b 100644 --- a/codegen/src/generators/persist/primary_key.rs +++ b/codegen/src/generators/persist/primary_key.rs @@ -133,14 +133,14 @@ impl PersistGenerator { fn get_generator_from_type(type_: &TokenStream, i: &Ident) -> syn::Result { Ok(match type_.to_string().as_str() { - "u8" => quote! { std::sync::atomic::AtomicU8 }, - "u16" => quote! { std::sync::atomic::AtomicU16 }, - "u32" => quote! { std::sync::atomic::AtomicU32 }, - "u64" => quote! { std::sync::atomic::AtomicU64 }, - "i8" => quote! { std::sync::atomic::AtomicI8 }, - "i16" => quote! { std::sync::atomic::AtomicI16 }, - "i32" => quote! { std::sync::atomic::AtomicI32 }, - "i64" => quote! { std::sync::atomic::AtomicI64 }, + "u8" => quote! { core::sync::atomic::AtomicU8 }, + "u16" => quote! { core::sync::atomic::AtomicU16 }, + "u32" => quote! { core::sync::atomic::AtomicU32 }, + "u64" => quote! { core::sync::atomic::AtomicU64 }, + "i8" => quote! { core::sync::atomic::AtomicI8 }, + "i16" => quote! { core::sync::atomic::AtomicI16 }, + "i32" => quote! { core::sync::atomic::AtomicI32 }, + "i64" => quote! { core::sync::atomic::AtomicI64 }, // The accepted set is `worktable_dsl::AUTOINCREMENT_TYPES`, and the // arms above must stay equal to it. `check` uses that list to // answer "would the macro accept this", so a second copy drifting diff --git a/codegen/src/generators/persist/queries/delete.rs b/codegen/src/generators/persist/queries/delete.rs index c2efa53c..8aca38f3 100644 --- a/codegen/src/generators/persist/queries/delete.rs +++ b/codegen/src/generators/persist/queries/delete.rs @@ -98,7 +98,23 @@ impl PersistGenerator { row, link, ); - res?; + // `delete_row_cdc` produces events whether or not it succeeds, and + // the index has already assigned their ids. Propagating the error + // without queueing them leaves a hole the persistence stream can + // never fill, exactly as the restore path below is careful not to. + if let core::result::Result::Err(e) = res { + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events, + }); + self.1.apply_operation(ack_op)?; + return core::result::Result::Err(e.into()); + } let (_, primary_key_events) = self.0.primary_index.remove_cdc(pk.clone(), link); if let core::result::Result::Err(e) = self.0.data.delete(link) { let mut secondary_keys_events = secondary_keys_events; @@ -219,7 +235,7 @@ impl PersistGenerator { quote! { pub async fn #name(&self, by: #type_) -> core::result::Result<(), WorkTableError> { let _bulk_mutation = self.0.lock_manager.bulk_mutation_guard(); - let pks = std::cell::RefCell::new(Vec::new()); + let pks = core::cell::RefCell::new(Vec::new()); self.iter_with(|row| { if row.#field == by { pks.borrow_mut().push(row.get_primary_key()); @@ -387,5 +403,21 @@ mod tests { emitted.contains("Operation :: Acknowledge"), "acknowledge op missing:\n{emitted}" ); + + // A failed secondary removal used to propagate through a bare `res?`, + // dropping the events `delete_row_cdc` had already produced. Their ids + // are assigned when the index produces them, so the persistence stream + // gapped permanently and the stall named a range rather than a cause. + let secondary = emitted.find("delete_row_cdc").expect("secondary removal emitted"); + let tail = &emitted[secondary..]; + let ack = tail + .find("Operation :: Acknowledge") + .expect("a failed secondary removal must acknowledge its events"); + assert!( + ack < tail + .find("remove_cdc (pk . clone () , link)") + .expect("primary removal emitted"), + "the secondary removal propagates before acknowledging, which gaps the stream:\n{emitted}" + ); } } diff --git a/codegen/src/generators/persist/queries/locks.rs b/codegen/src/generators/persist/queries/locks.rs index c6f0c3ee..3b685c39 100644 --- a/codegen/src/generators/persist/queries/locks.rs +++ b/codegen/src/generators/persist/queries/locks.rs @@ -95,9 +95,9 @@ impl PersistGenerator { quote! { #[allow(clippy::mutable_key_type)] - pub fn #ident(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let new_lock = std::sync::Arc::new(Lock::new(id)); + pub fn #ident(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let new_lock = worktable::prelude::Arc::new(Lock::new(id)); #(#inner)* (set, new_lock) } diff --git a/codegen/src/generators/persist/queries/select.rs b/codegen/src/generators/persist/queries/select.rs index 7627a6fb..581c4fd6 100644 --- a/codegen/src/generators/persist/queries/select.rs +++ b/codegen/src/generators/persist/queries/select.rs @@ -32,7 +32,7 @@ impl PersistGenerator { // Acquire the grace-period guard only when iteration starts. // Merely constructing and retaining a query builder must not // stall retired-link reclamation. - let iter = std::iter::once_with(move || { + let iter = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .iter_values() @@ -55,7 +55,7 @@ impl PersistGenerator { return None; } current_link = replacement; - std::hint::spin_loop(); + core::hint::spin_loop(); } None }) diff --git a/codegen/src/generators/persist/queries/update.rs b/codegen/src/generators/persist/queries/update.rs index ffd41658..f91b65d8 100644 --- a/codegen/src/generators/persist/queries/update.rs +++ b/codegen/src/generators/persist/queries/update.rs @@ -42,7 +42,7 @@ impl PersistGenerator { .keys() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -311,8 +311,8 @@ impl PersistGenerator { // compensation above). let mut merged_events = secondary_keys_events; if row_holds_old_values { - let mut reversed_diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = - std::collections::HashMap::new(); + let mut reversed_diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = + worktable::prelude::HashMap::new(); for (key, diff) in diffs { reversed_diffs.insert(key, Difference { old: diff.new, new: diff.old }); } @@ -501,13 +501,14 @@ impl PersistGenerator { fn gen_process_diffs_insert_on_index(&self, idents: &[Ident], idx_idents: Option<&Vec>) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let avt_type_ident = name_generator.get_available_type_ident(); + let pk_ident = name_generator.get_primary_key_type_ident(); // `updated_bytes` is bound by gen_data_write_and_fetch, which captures // the real row bytes right after the data write. let diff_container = if idx_idents.is_some() { quote! { let row_old = self.0.data.select_non_ghosted(link)?; let row_new = row.clone(); - let mut diffs: std::collections::HashMap<&str, Difference<#avt_type_ident>> = std::collections::HashMap::new(); + let mut diffs: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> = worktable::prelude::HashMap::new(); } } else { quote! {} @@ -567,7 +568,27 @@ impl PersistGenerator { Err(WorkTableError::AlreadyExists(at.to_string_value())) } - IndexError::NotFound => Err(WorkTableError::NotFound), + IndexError::NotFound => { + // The insert side produced events before it + // failed, and the index has already assigned + // their ids. Returning without queueing them + // leaves a hole the persistence stream can + // never fill, which is what the sibling arm + // above avoids and what this arm used to + // cause. + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events: secondary_events.clone(), + }); + self.1.apply_operation(ack_op)?; + + Err(WorkTableError::NotFound) + } }; } let mut secondary_keys_events = secondary_events; @@ -587,10 +608,29 @@ impl PersistGenerator { } fn gen_process_diffs_remove_on_index(&self, idx_idents: Option<&Vec>) -> TokenStream { + let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); + let pk_ident = name_generator.get_primary_key_type_ident(); + let secondary_events_ident = name_generator.get_space_secondary_index_events_ident(); if idx_idents.is_some() { quote! { let (secondary_keys_events_remove, res) = self.0.indexes.process_difference_remove_cdc(link, diffs); - res?; + // The removal produced events whether or not it succeeded, and + // their ids are already assigned. Propagating the error without + // queueing them gaps the stream permanently, so acknowledge + // them first and then propagate unchanged. + if let core::result::Result::Err(e) = res { + let ack_op: Operation< + <<#pk_ident as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State, + #pk_ident, + #secondary_events_ident + > = Operation::Acknowledge(AcknowledgeOperation { + id: OperationId::Single(uuid::Uuid::now_v7()), + primary_key_events: vec![], + secondary_keys_events: secondary_keys_events_remove, + }); + self.1.apply_operation(ack_op)?; + return core::result::Result::Err(e.into()); + } op.extend_secondary_key_events(secondary_keys_events_remove); } } else { @@ -615,7 +655,7 @@ impl PersistGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -693,7 +733,7 @@ impl PersistGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -860,7 +900,7 @@ impl PersistGenerator { pks.sort_unstable(); pks.dedup(); - let mut guards: std::collections::HashMap<_, _> = std::collections::HashMap::new(); + let mut guards: worktable::prelude::HashMap<_, _> = worktable::prelude::HashMap::new(); // Full-row locks, not per-column custom locks: each row's // unsized reinsert path mutates the whole row under these // guards, and one uniform lock kind keeps every concurrent @@ -933,7 +973,7 @@ impl PersistGenerator { .iter() .map(|i| { quote! { - std::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); + core::mem::swap(&mut archived.inner.#i, &mut archived_row.#i); } }) .collect::>(); @@ -1143,5 +1183,45 @@ mod tests { .find("Operation :: Update (UpdateOperation") .expect("update op emitted"); assert!(insert < write && write < op_build, "emission order broken:\n{emitted}"); + + // Every event the index produced must reach the persistence stream, + // including on the paths that fail. The index assigns an event id at + // the moment it produces the event, so a path that returns without + // queueing one leaves a hole `BatchOperation::validate` will refuse + // forever, and the stall it causes names a range rather than a cause. + // + // The `NotFound` arm used to be exactly that: its sibling + // `AlreadyExists` arm built an Acknowledge and it did not. + let not_found = emitted + .find("IndexError :: NotFound =>") + .expect("not-found arm emitted"); + let tail = &emitted[not_found..]; + let ack = tail + .find("Operation :: Acknowledge") + .expect("not-found arm must acknowledge its events"); + let returns = tail + .find("Err (WorkTableError :: NotFound)") + .expect("not-found arm returns"); + assert!( + ack < returns, + "the not-found arm returns before acknowledging, which gaps the stream:\n{emitted}" + ); + + // Same for the removal side, where the events were dropped by a bare + // `res?` before the extend that would have carried them. + let removal = emitted + .find("process_difference_remove_cdc (link , diffs)") + .expect("old-key removal emitted"); + let tail = &emitted[removal..]; + let ack = tail + .find("Operation :: Acknowledge") + .expect("a failed removal must acknowledge its events"); + let extend = tail + .find("op . extend_secondary_key_events") + .expect("successful removal extends the operation"); + assert!( + ack < extend, + "a failed removal propagates before acknowledging, which gaps the stream:\n{emitted}" + ); } } diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index 39e453b2..80a1ef93 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -254,13 +254,13 @@ impl PersistGenerator { }; let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( PersistentArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); } } else if pk_types_unsized { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name) )); } @@ -268,19 +268,19 @@ impl PersistGenerator { match self.columns.primary_index_backend { crate::common::model::IndexBackend::WorktablesIndex => quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( #wti_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) )); }, crate::common::model::IndexBackend::Indexset => quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( UpstreamIndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) )); }, crate::common::model::IndexBackend::Arctic => unreachable!("handled before variable-size dispatch"), crate::common::model::IndexBackend::Congee => quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( PersistentCongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); }, @@ -409,7 +409,7 @@ impl PersistGenerator { #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds + 'a, + R: core::ops::RangeBounds + 'a, Pk: Clone + 'a, { let converted_range = ( @@ -418,7 +418,7 @@ impl PersistGenerator { ); // Delay the grace-period guard until the returned iterator is // consumed so an idle query builder cannot pin reclamation. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .range_links(converted_range) @@ -648,7 +648,7 @@ impl PersistGenerator { let exponent = core::cmp::min(backoff_spins - 8, 8); let micros = core::cmp::min(1u64 << exponent, 256); backoff_spins = backoff_spins.saturating_add(1); - tokio::time::sleep(std::time::Duration::from_micros(micros)).await; + tokio::time::sleep(core::time::Duration::from_micros(micros)).await; } } } @@ -689,7 +689,7 @@ impl PersistGenerator { /// assigned contiguous keys before `insert_many`. Interleaved /// `get_next_pk` calls keep working and never overlap a /// reservation. - pub fn reserve_pks(&self, count: usize) -> std::ops::Range<#pk_inner_type> { + pub fn reserve_pks(&self, count: usize) -> core::ops::Range<#pk_inner_type> { self.0.reserve_pks(count) } } @@ -732,7 +732,7 @@ impl PersistGenerator { quote! { pub async fn iter_with_async< F: Fn(#row_type) -> Fut, - Fut: std::future::Future> + Fut: core::future::Future> >(&self, f: F) -> core::result::Result<(), WorkTableError> { #inner } @@ -794,8 +794,8 @@ impl PersistGenerator { let lock_type = name_generator.get_lock_type_ident(); quote! { - pub fn vacuum(&self) -> std::sync::Arc { - std::sync::Arc::new(EmptyDataVacuum::< + pub fn vacuum(&self) -> worktable::prelude::Arc { + worktable::prelude::Arc::new(EmptyDataVacuum::< _, _, _, @@ -807,10 +807,10 @@ impl PersistGenerator { #secondary_index_events >::new( #table_name, - std::sync::Arc::clone(&self.0.data), - std::sync::Arc::clone(&self.0.lock_manager), - std::sync::Arc::clone(&self.0.primary_index), - std::sync::Arc::clone(&self.0.indexes), + worktable::prelude::Arc::clone(&self.0.data), + worktable::prelude::Arc::clone(&self.0.lock_manager), + worktable::prelude::Arc::clone(&self.0.primary_index), + worktable::prelude::Arc::clone(&self.0.indexes), ).with_persistence(self.1.vacuum_sink())) } } diff --git a/codegen/src/generators/persist/table/index_fns.rs b/codegen/src/generators/persist/table/index_fns.rs index e6712995..20f6f5e2 100644 --- a/codegen/src/generators/persist/table/index_fns.rs +++ b/codegen/src/generators/persist/table/index_fns.rs @@ -99,7 +99,7 @@ impl PersistGenerator { if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None }; @@ -140,7 +140,7 @@ impl PersistGenerator { #column_range_type, #row_fields_ident> { - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.indexes.#field_ident .get(#by) @@ -173,7 +173,7 @@ impl PersistGenerator { let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { ( predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), @@ -183,7 +183,7 @@ impl PersistGenerator { ) } else { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { predicate_range.clone() }, ) }; @@ -216,7 +216,7 @@ impl PersistGenerator { }; let predicate_filter = quote! { .filter(move |row| { - std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + core::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) }) }; @@ -231,7 +231,7 @@ impl PersistGenerator { #predicate_setup // Query construction is not an active read. Pin the grace // period on the first row lookup instead. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); #index_range .filter_map(#select_row) diff --git a/codegen/src/generators/persist/table/select_executor.rs b/codegen/src/generators/persist/table/select_executor.rs index 1499d250..bec09452 100644 --- a/codegen/src/generators/persist/table/select_executor.rs +++ b/codegen/src/generators/persist/table/select_executor.rs @@ -44,7 +44,7 @@ impl PersistGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - #variant_ident(std::ops::#range_ident<#ty_ident>), + #variant_ident(core::ops::#range_ident<#ty_ident>), } }) .collect(); @@ -65,8 +65,8 @@ impl PersistGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - impl From> for #column_range_type { - fn from(range: std::ops::#range_ident<#ty_ident>) -> Self { + impl From> for #column_range_type { + fn from(range: core::ops::#range_ident<#ty_ident>) -> Self { Self::#variant_ident(range) } } @@ -100,8 +100,8 @@ impl PersistGenerator { let col_ident = Ident::new(&column.to_string(), Span::call_site()); quote! { #row_fields_ident::#column_variant => { - let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(std::cmp::Ordering::Equal); - if cmp != std::cmp::Ordering::Equal { + let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(core::cmp::Ordering::Equal); + if cmp != core::cmp::Ordering::Equal { return match order { Order::Asc => cmp, Order::Desc => cmp.reverse(), @@ -165,7 +165,7 @@ impl PersistGenerator { _ => continue, } } - std::cmp::Ordering::Equal + core::cmp::Ordering::Equal }); iter = Box::new(items.into_iter()); }; diff --git a/codegen/src/generators/read_only/index/usual.rs b/codegen/src/generators/read_only/index/usual.rs index 0af30608..9739b3bc 100644 --- a/codegen/src/generators/read_only/index/usual.rs +++ b/codegen/src/generators/read_only/index/usual.rs @@ -236,7 +236,7 @@ impl ReadOnlyGenerator { fn process_difference_remove( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { #(#process_difference_remove_rows)* core::result::Result::Ok(()) @@ -293,7 +293,7 @@ impl ReadOnlyGenerator { fn process_difference_insert( &self, link: Link, - difference: std::collections::HashMap<&str, Difference<#avt_type_ident>> + difference: worktable::prelude::HashMap<&str, Difference<#avt_type_ident>> ) -> core::result::Result<(), IndexError<#avt_index_ident>> { let mut inserted_indexes: Vec<#avt_index_ident> = vec![]; #(#process_difference_insert_rows)* diff --git a/codegen/src/generators/read_only/locks.rs b/codegen/src/generators/read_only/locks.rs index 280afd28..ffa50040 100644 --- a/codegen/src/generators/read_only/locks.rs +++ b/codegen/src/generators/read_only/locks.rs @@ -24,7 +24,7 @@ impl ReadOnlyGenerator { .keys() .map(|i| { let name = Ident::new(format!("{i}_lock").as_str(), Span::mixed_site()); - quote! { #name: Option>, } + quote! { #name: Option>, } }) .collect(); @@ -124,8 +124,8 @@ impl ReadOnlyGenerator { .collect(); quote! { - fn with_lock(id: u16) -> (Self, std::sync::Arc) { - let lock = std::sync::Arc::new(Lock::new(id)); + fn with_lock(id: u16) -> (Self, worktable::prelude::Arc) { + let lock = worktable::prelude::Arc::new(Lock::new(id)); ( Self { #(#rows),* @@ -154,9 +154,9 @@ impl ReadOnlyGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn lock(&mut self, id: u16) -> (std::collections::HashSet>, std::sync::Arc) { - let mut set = std::collections::HashSet::new(); - let lock = std::sync::Arc::new(Lock::new(id)); + fn lock(&mut self, id: u16) -> (worktable::prelude::HashSet>, worktable::prelude::Arc) { + let mut set = worktable::prelude::HashSet::new(); + let lock = worktable::prelude::Arc::new(Lock::new(id)); #(#rows)* (set, lock) @@ -186,8 +186,8 @@ impl ReadOnlyGenerator { quote! { #[allow(clippy::mutable_key_type)] - fn merge(&mut self, other: &mut Self) -> std::collections::HashSet> { - let mut set = std::collections::HashSet::new(); + fn merge(&mut self, other: &mut Self) -> worktable::prelude::HashSet> { + let mut set = worktable::prelude::HashSet::new(); #(#rows)* set } diff --git a/codegen/src/generators/read_only/primary_key.rs b/codegen/src/generators/read_only/primary_key.rs index 192a2e44..8c8274c7 100644 --- a/codegen/src/generators/read_only/primary_key.rs +++ b/codegen/src/generators/read_only/primary_key.rs @@ -133,14 +133,14 @@ impl ReadOnlyGenerator { fn get_generator_from_type(type_: &TokenStream, i: &Ident) -> syn::Result { Ok(match type_.to_string().as_str() { - "u8" => quote! { std::sync::atomic::AtomicU8 }, - "u16" => quote! { std::sync::atomic::AtomicU16 }, - "u32" => quote! { std::sync::atomic::AtomicU32 }, - "u64" => quote! { std::sync::atomic::AtomicU64 }, - "i8" => quote! { std::sync::atomic::AtomicI8 }, - "i16" => quote! { std::sync::atomic::AtomicI16 }, - "i32" => quote! { std::sync::atomic::AtomicI32 }, - "i64" => quote! { std::sync::atomic::AtomicI64 }, + "u8" => quote! { core::sync::atomic::AtomicU8 }, + "u16" => quote! { core::sync::atomic::AtomicU16 }, + "u32" => quote! { core::sync::atomic::AtomicU32 }, + "u64" => quote! { core::sync::atomic::AtomicU64 }, + "i8" => quote! { core::sync::atomic::AtomicI8 }, + "i16" => quote! { core::sync::atomic::AtomicI16 }, + "i32" => quote! { core::sync::atomic::AtomicI32 }, + "i64" => quote! { core::sync::atomic::AtomicI64 }, // The accepted set is `worktable_dsl::AUTOINCREMENT_TYPES`, and the // arms above must stay equal to it. `check` uses that list to // answer "would the macro accept this", so a second copy drifting diff --git a/codegen/src/generators/read_only/queries/select.rs b/codegen/src/generators/read_only/queries/select.rs index 0adcbe70..8c46e611 100644 --- a/codegen/src/generators/read_only/queries/select.rs +++ b/codegen/src/generators/read_only/queries/select.rs @@ -32,7 +32,7 @@ impl ReadOnlyGenerator { // Acquire the grace-period guard only when iteration starts. // Merely constructing and retaining a query builder must not // stall retired-link reclamation. - let iter = std::iter::once_with(move || { + let iter = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .iter_values() @@ -55,7 +55,7 @@ impl ReadOnlyGenerator { return None; } current_link = replacement; - std::hint::spin_loop(); + core::hint::spin_loop(); } None }) diff --git a/codegen/src/generators/read_only/table/impls.rs b/codegen/src/generators/read_only/table/impls.rs index 6e4577b6..db0fd756 100644 --- a/codegen/src/generators/read_only/table/impls.rs +++ b/codegen/src/generators/read_only/table/impls.rs @@ -237,19 +237,19 @@ impl ReadOnlyGenerator { let pk_types_unsized = is_unsized_vec(pk_types); let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( ArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); } } else if self.columns.primary_index_backend == crate::common::model::IndexBackend::Congee { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( CongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default() )); } } else if pk_types_unsized { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name) )); } @@ -260,7 +260,7 @@ impl ReadOnlyGenerator { }; quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + inner.primary_index = worktable::prelude::Arc::new(PrimaryIndex::from_map( #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) )); } @@ -369,7 +369,7 @@ impl ReadOnlyGenerator { #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds + 'a, + R: core::ops::RangeBounds + 'a, Pk: Clone + 'a, { let converted_range = ( @@ -378,7 +378,7 @@ impl ReadOnlyGenerator { ); // Delay the grace-period guard until the returned iterator is // consumed so an idle query builder cannot pin reclamation. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.primary_index.pk_map .range_links(converted_range) @@ -460,7 +460,7 @@ impl ReadOnlyGenerator { quote! { pub async fn iter_with_async< F: Fn(#row_type) -> Fut, - Fut: std::future::Future> + Fut: core::future::Future> >(&self, f: F) -> core::result::Result<(), WorkTableError> { #inner } diff --git a/codegen/src/generators/read_only/table/index_fns.rs b/codegen/src/generators/read_only/table/index_fns.rs index 98a1d0f6..13ebb9c8 100644 --- a/codegen/src/generators/read_only/table/index_fns.rs +++ b/codegen/src/generators/read_only/table/index_fns.rs @@ -99,7 +99,7 @@ impl ReadOnlyGenerator { if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None }; @@ -140,7 +140,7 @@ impl ReadOnlyGenerator { #column_range_type, #row_fields_ident> { - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); self.0.indexes.#field_ident .get(#by) @@ -173,7 +173,7 @@ impl ReadOnlyGenerator { let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { ( predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), @@ -183,7 +183,7 @@ impl ReadOnlyGenerator { ) } else { ( - quote! { std::ops::RangeBounds<#type_> }, + quote! { core::ops::RangeBounds<#type_> }, quote! { predicate_range.clone() }, ) }; @@ -216,7 +216,7 @@ impl ReadOnlyGenerator { }; let predicate_filter = quote! { .filter(move |row| { - std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + core::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) }) }; @@ -231,7 +231,7 @@ impl ReadOnlyGenerator { #predicate_setup // Query construction is not an active read. Pin the grace // period on the first row lookup instead. - let rows = std::iter::once_with(move || { + let rows = core::iter::once_with(move || { let read_guard = self.0.data.read_guard(); #index_range .filter_map(#select_row) diff --git a/codegen/src/generators/read_only/table/select_executor.rs b/codegen/src/generators/read_only/table/select_executor.rs index bd8b3f7c..7ebe9806 100644 --- a/codegen/src/generators/read_only/table/select_executor.rs +++ b/codegen/src/generators/read_only/table/select_executor.rs @@ -44,7 +44,7 @@ impl ReadOnlyGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - #variant_ident(std::ops::#range_ident<#ty_ident>), + #variant_ident(core::ops::#range_ident<#ty_ident>), } }) .collect(); @@ -65,8 +65,8 @@ impl ReadOnlyGenerator { ); let range_ident = Ident::new(&format!("Range{variant}"), Span::call_site()); quote! { - impl From> for #column_range_type { - fn from(range: std::ops::#range_ident<#ty_ident>) -> Self { + impl From> for #column_range_type { + fn from(range: core::ops::#range_ident<#ty_ident>) -> Self { Self::#variant_ident(range) } } @@ -100,8 +100,8 @@ impl ReadOnlyGenerator { let col_ident = Ident::new(&column.to_string(), Span::call_site()); quote! { #row_fields_ident::#column_variant => { - let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(std::cmp::Ordering::Equal); - if cmp != std::cmp::Ordering::Equal { + let cmp = a.#col_ident.partial_cmp(&b.#col_ident).unwrap_or(core::cmp::Ordering::Equal); + if cmp != core::cmp::Ordering::Equal { return match order { Order::Asc => cmp, Order::Desc => cmp.reverse(), @@ -165,7 +165,7 @@ impl ReadOnlyGenerator { _ => continue, } } - std::cmp::Ordering::Equal + core::cmp::Ordering::Equal }); iter = Box::new(items.into_iter()); }; diff --git a/codegen/src/mem_stat/mod.rs b/codegen/src/mem_stat/mod.rs index ea8d6b49..b674be14 100644 --- a/codegen/src/mem_stat/mod.rs +++ b/codegen/src/mem_stat/mod.rs @@ -3,11 +3,11 @@ use quote::quote; use syn::{Data, DeriveInput, Fields, Result, Type}; fn gen_heap_size_body(data: &Data) -> Result { - gen_mem_fn_body(data, quote! { heap_size() }, quote! { std::mem::size_of::() }) + gen_mem_fn_body(data, quote! { heap_size() }, quote! { core::mem::size_of::() }) } fn gen_used_size_body(data: &Data) -> Result { - gen_mem_fn_body(data, quote! { used_size() }, quote! { std::mem::size_of::() }) + gen_mem_fn_body(data, quote! { used_size() }, quote! { core::mem::size_of::() }) } fn gen_mem_fn_body(data: &Data, method: TokenStream, default_for_copy: TokenStream) -> Result { diff --git a/codegen/src/persist_index/generator.rs b/codegen/src/persist_index/generator.rs index 49b78938..754780a6 100644 --- a/codegen/src/persist_index/generator.rs +++ b/codegen/src/persist_index/generator.rs @@ -193,6 +193,7 @@ impl Generator { fn gen_persist_fn(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let ident = name_generator.get_work_table_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); let version_const_name = name_generator.get_version_const_ident(); let index_extension = Literal::string(WT_INDEX_EXTENSION); @@ -216,15 +217,15 @@ impl Generator { }, _ => quote! { { - let mut file = tokio::fs::File::create(format!("{}/{}{}", path, #index_name_literal, #index_extension)).await?; + let mut file = worktable::prelude::fsx::create(format!("{}/{}{}", path, #index_name_literal, #index_extension)).await?; let mut info = #ident::space_info_default(); info.inner.page_count = self.#i.1.len() as u32 + self.#i.0.len() as u32; - persist_page(&mut info, &mut file).await?; + persist_page::<_, { #page_const_name as u32 }>(&mut info, &mut file).await?; for mut page in &mut self.#i.0 { - persist_page(&mut page, &mut file).await?; + persist_page::<_, { #page_const_name as u32 }>(&mut page, &mut file).await?; } for mut page in &mut self.#i.1 { - persist_page(&mut page, &mut file).await?; + persist_page::<_, { #page_const_name as u32 }>(&mut page, &mut file).await?; } } }, @@ -295,18 +296,18 @@ impl Generator { _ => quote! { let #i: #parsed_type = { let mut #i = vec![]; - let mut file = tokio::fs::File::open(format!("{}/{}{}", path, #literal, #index_extension)).await?; - let info = parse_page::, { #page_const_name as u32 }>(&mut file, 0).await?; - let file_length = file.metadata().await?.len(); + let mut file = worktable::prelude::fsx::open(format!("{}/{}{}", path, #literal, #index_extension)).await?; + let info = parse_page::, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut file, 0).await?; + let file_length = worktable::prelude::fsx::file_metadata(&mut file).await?; // Pages sit at a fixed #page_const_name stride // (header inside the slot): the next free page id // is ceil(len / stride). The previous divisor used // stride + header and an unconditional +1. let page_id = file_length.div_ceil(#page_const_name as u64); - let next_page_id = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(page_id as u32)); - let toc = IndexTableOfContents::<_, { #page_const_name as u32 }>::parse_from_file(&mut file, 0.into(), next_page_id.clone()).await?; + let next_page_id = worktable::prelude::Arc::new(core::sync::atomic::AtomicU32::new(page_id as u32)); + let toc = IndexTableOfContents::<_, { #inner_const_name as u32 }, { #page_const_name as u32 }>::parse_from_file(&mut file, 0.into(), next_page_id.clone()).await?; for page_id in toc.iter().map(|(_, page_id)| page_id) { - let index = parse_page::<_, { #page_const_name as u32 }>(&mut file, (*page_id).into()).await?; + let index = parse_page::<_, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut file, (*page_id).into()).await?; #i.push(index); } (toc.pages, #i) @@ -370,6 +371,7 @@ impl Generator { fn gen_get_persisted_index_fn(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let const_name = name_generator.get_page_inner_size_const_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); let idents = self .struct_def @@ -401,7 +403,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(UnsizedIndexPage::from_node(node.as_ref())); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend == Some(ArtBackend::ArcticMulti) { @@ -415,7 +417,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend == Some(ArtBackend::Arctic) && is_unsized(&ty.to_string()) { @@ -428,7 +430,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(UnsizedIndexPage::from_node(node.as_ref())); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend == Some(ArtBackend::Arctic) { @@ -442,7 +444,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else if layout.art_backend.is_some() { @@ -462,7 +464,7 @@ impl Generator { let page = UnsizedIndexPage::from_node(node.as_ref()); pages.push(page); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else { @@ -472,7 +474,7 @@ impl Generator { let page = UnsizedIndexPage::from_node(node.as_ref()); pages.push(page); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } @@ -491,7 +493,7 @@ impl Generator { .collect(); pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } else { @@ -502,7 +504,7 @@ impl Generator { let page = IndexPage::from_node(&node, size); pages.push(page); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); let #i = (toc.pages, pages); }) } diff --git a/codegen/src/persist_index/mod.rs b/codegen/src/persist_index/mod.rs index ef17c461..c284b9d4 100644 --- a/codegen/src/persist_index/mod.rs +++ b/codegen/src/persist_index/mod.rs @@ -46,7 +46,7 @@ mod tests { #[derive(Debug, Default, Clone)] pub struct TestIndex { test_idx: TreeIndex, - exchnage_idx: TreeIndex>> + exchnage_idx: TreeIndex>> } }; diff --git a/codegen/src/persist_index/parser.rs b/codegen/src/persist_index/parser.rs index f7023697..4cb43d36 100644 --- a/codegen/src/persist_index/parser.rs +++ b/codegen/src/persist_index/parser.rs @@ -46,7 +46,7 @@ mod tests { #[derive(Debug, Default, Clone)] pub struct TestIndex { test_idx: TreeIndex, - exchnage_idx: TreeIndex>> + exchnage_idx: TreeIndex>> } }; assert!(Parser::parse_struct(input).is_ok()) diff --git a/codegen/src/persist_index/space/events.rs b/codegen/src/persist_index/space/events.rs index 28b66819..720096ad 100644 --- a/codegen/src/persist_index/space/events.rs +++ b/codegen/src/persist_index/space/events.rs @@ -100,8 +100,8 @@ impl Generator { .collect(); quote! { - fn first_evs(&self) -> std::collections::HashMap<#avt_index_ident, Option> { - let mut map = std::collections::HashMap::new(); + fn first_evs(&self) -> worktable::prelude::HashMap<#avt_index_ident, Option> { + let mut map = worktable::prelude::HashMap::new(); #(#fields_first)* map } @@ -126,8 +126,8 @@ impl Generator { .collect(); quote! { - fn last_evs(&self) -> std::collections::HashMap<#avt_index_ident, Option> { - let mut map = std::collections::HashMap::new(); + fn last_evs(&self) -> worktable::prelude::HashMap<#avt_index_ident, Option> { + let mut map = worktable::prelude::HashMap::new(); #(#fields_last)* map } @@ -200,7 +200,7 @@ impl Generator { quote! { fn iter_event_ids(&self) -> impl Iterator { - > as Iterator>::flatten( + > as Iterator>::flatten( vec![ #(#fields_iter),* ] diff --git a/codegen/src/persist_index/space/index.rs b/codegen/src/persist_index/space/index.rs index 166b514e..de1cfab1 100644 --- a/codegen/src/persist_index/space/index.rs +++ b/codegen/src/persist_index/space/index.rs @@ -9,6 +9,7 @@ impl Generator { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let ident = name_generator.get_space_secondary_index_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); let fields: Vec<_> = self .struct_def @@ -20,31 +21,31 @@ impl Generator { let t = self.field_types.get(i).expect("field type was collected"); Ok(match layout.art_backend { Some(ArtBackend::Arctic) if is_unsized(&t.to_string()) => quote! { - #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::Arctic) => quote! { - #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::ArcticMulti) if is_unsized(&t.to_string()) => quote! { - #i: SpaceLogicalMultiIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalMultiIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::ArcticMulti) => quote! { - #i: SpaceLogicalMultiIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalMultiIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, Some(ArtBackend::Congee) => quote! { #i: SpaceCongeeIndex<#t, { #inner_const_name as u32}>, }, None if layout.logical_wti && is_unsized(&t.to_string()) => quote! { - #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, None if layout.logical_wti => quote! { - #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, None if is_unsized(&t.to_string()) => quote! { - #i: SpaceIndexUnsized<#t, { #inner_const_name as u32}>, + #i: SpaceIndexUnsized<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, None => quote! { - #i: SpaceIndex<#t, { #inner_const_name as u32}>, + #i: SpaceIndex<#t, { #inner_const_name as u32}, { #page_const_name as u32 }>, }, }) }) diff --git a/codegen/src/persist_table/generator/space.rs b/codegen/src/persist_table/generator/space.rs index 35a9c929..a93e59a3 100644 --- a/codegen/src/persist_table/generator/space.rs +++ b/codegen/src/persist_table/generator/space.rs @@ -32,33 +32,35 @@ impl Generator { let ident = name_generator.get_persistence_engine_ident(); let primary_key_type = name_generator.get_primary_key_type_ident(); let inner_const_name = name_generator.get_page_inner_size_const_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); let const_name = name_generator.get_page_size_const_ident(); let space_primary_index = name_generator.get_space_primary_index_ident(); let space_secondary_indexes = name_generator.get_space_secondary_index_ident(); let space_secondary_indexes_events = name_generator.get_space_secondary_index_events_ident(); let avt_index_ident = name_generator.get_available_indexes_ident(); - let space_index_type = - if self.attributes.pk_arctic_string || (self.attributes.pk_unsized && self.attributes.pk_wti_logical) { - quote! { - SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }> - } - } else if self.attributes.pk_unsized { - quote! { - SpaceIndexUnsized<#primary_key_type, { #inner_const_name as u32 }> - } - } else if self.attributes.pk_wti_logical || self.attributes.pk_arctic { - quote! { - SpaceLogicalIndex<#primary_key_type, { #inner_const_name as u32 }> - } - } else if self.attributes.pk_congee { - quote! { - SpaceCongeeIndex<#primary_key_type, { #inner_const_name as u32 }> - } - } else { - quote! { - SpaceIndex<#primary_key_type, { #inner_const_name as u32 }> - } - }; + let space_index_type = if self.attributes.pk_arctic_string + || (self.attributes.pk_unsized && self.attributes.pk_wti_logical) + { + quote! { + SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }, { #page_const_name as u32 }> + } + } else if self.attributes.pk_unsized { + quote! { + SpaceIndexUnsized<#primary_key_type, { #inner_const_name as u32 }, { #page_const_name as u32 }> + } + } else if self.attributes.pk_wti_logical || self.attributes.pk_arctic { + quote! { + SpaceLogicalIndex<#primary_key_type, { #inner_const_name as u32 }, { #page_const_name as u32 }> + } + } else if self.attributes.pk_congee { + quote! { + SpaceCongeeIndex<#primary_key_type, { #inner_const_name as u32 }> + } + } else { + quote! { + SpaceIndex<#primary_key_type, { #inner_const_name as u32 }, { #page_const_name as u32 }> + } + }; quote! { pub type #space_primary_index = #space_index_type; diff --git a/codegen/src/persist_table/generator/space_file/mod.rs b/codegen/src/persist_table/generator/space_file/mod.rs index 93626f25..c883e42e 100644 --- a/codegen/src/persist_table/generator/space_file/mod.rs +++ b/codegen/src/persist_table/generator/space_file/mod.rs @@ -241,7 +241,7 @@ impl Generator { data.set_page_id(page_id.into()); page_id += 1; - std::sync::Arc::new(data) + worktable::prelude::Arc::new(data) }) .collect(); let data = DataPages::from_data(data) @@ -251,13 +251,13 @@ impl Generator { #primary_index_init let table = WorkTable { - data: std::sync::Arc::new(data), - primary_index: std::sync::Arc::new(primary_index), - indexes: std::sync::Arc::new(indexes), + data: worktable::prelude::Arc::new(data), + primary_index: worktable::prelude::Arc::new(primary_index), + indexes: worktable::prelude::Arc::new(indexes), pk_gen: PrimaryKeyGeneratorState::from_state(self.data_info.inner.pk_gen_state), - lock_manager: std::sync::Arc::new(LockMap::<#lock_type, #pk_type>::default()), + lock_manager: worktable::prelude::Arc::new(LockMap::<#lock_type, #pk_type>::default()), table_name: #table_name, - pk_phantom: std::marker::PhantomData, + pk_phantom: core::marker::PhantomData, }; table.validate_persisted_state(path)?; @@ -310,7 +310,7 @@ impl Generator { data.set_page_id(page_id.into()); page_id += 1; - std::sync::Arc::new(data) + worktable::prelude::Arc::new(data) }) .collect(); let data = DataPages::from_data(data) @@ -320,13 +320,13 @@ impl Generator { #primary_index_init let table = WorkTable { - data: std::sync::Arc::new(data), - primary_index: std::sync::Arc::new(primary_index), - indexes: std::sync::Arc::new(indexes), + data: worktable::prelude::Arc::new(data), + primary_index: worktable::prelude::Arc::new(primary_index), + indexes: worktable::prelude::Arc::new(indexes), pk_gen: PrimaryKeyGeneratorState::from_state(self.data_info.inner.pk_gen_state), - lock_manager: std::sync::Arc::new(LockMap::<#lock_type, #pk_type>::default()), + lock_manager: worktable::prelude::Arc::new(LockMap::<#lock_type, #pk_type>::default()), table_name: #table_name, - pk_phantom: std::marker::PhantomData, + pk_phantom: core::marker::PhantomData, }; table.validate_persisted_state(path)?; @@ -353,11 +353,11 @@ impl Generator { let parse_pk_page = if self.attributes.pk_unsized { quote! { - let index = parse_page::, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; + let index = parse_page::, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; } } else { quote! { - let index = parse_page::, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; + let index = parse_page::, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; } }; @@ -372,17 +372,17 @@ impl Generator { quote! { { let mut primary_index = vec![]; - let mut primary_file = tokio::fs::File::open(format!("{}/primary{}", path, #index_extension)).await?; - let info = parse_page::, { #page_const_name as u32 }>(&mut primary_file, 0).await?; - let file_length = primary_file.metadata().await?.len(); + let mut primary_file = worktable::prelude::fsx::open(format!("{}/primary{}", path, #index_extension)).await?; + let info = parse_page::, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut primary_file, 0).await?; + let file_length = worktable::prelude::fsx::file_metadata(&mut primary_file).await?; // Pages sit at a fixed #page_const_name stride with the // general header inside the slot, so the next free page id // is ceil(len / stride). The previous divisor added the // header on top of the full stride and lagged one page // behind roughly every 512 pages. let count = file_length.div_ceil(#page_const_name as u64); - let next_page_id = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(count as u32)); - let toc = IndexTableOfContents::<_, { #page_const_name as u32 }>::parse_from_file(&mut primary_file, 0.into(), next_page_id.clone()).await?; + let next_page_id = worktable::prelude::Arc::new(core::sync::atomic::AtomicU32::new(count as u32)); + let toc = IndexTableOfContents::<_, { #inner_const_name as u32 }, { #page_const_name as u32 }>::parse_from_file(&mut primary_file, 0.into(), next_page_id.clone()).await?; for page_id in toc.iter().map(|(_, page_id)| page_id) { #parse_pk_page primary_index.push(index); @@ -399,9 +399,9 @@ impl Generator { let indexes = #persisted_index_name::parse_from_file(path).await?; let (data, data_info) = { let mut data = vec![]; - let mut data_file = tokio::fs::File::open(format!("{}/{}", path, #data_extension)).await?; - let info = parse_page::::Generator as PrimaryKeyGeneratorState>::State>, { #page_const_name as u32 }>(&mut data_file, 0).await?; - let file_length = data_file.metadata().await?.len(); + let mut data_file = worktable::prelude::fsx::open(format!("{}/{}", path, #data_extension)).await?; + let info = parse_page::::Generator as PrimaryKeyGeneratorState>::State>, { #page_const_name as u32 }, { #page_const_name as u32 }>(&mut data_file, 0).await?; + let file_length = worktable::prelude::fsx::file_metadata(&mut data_file).await?; // ceil(len / stride) counts every occupied page slot, // including the info page at id 0, whether or not the last // page fills its slot. The previous floor + inclusive @@ -410,7 +410,7 @@ impl Generator { // exactly fills its slot), failing the whole load. let count = file_length.div_ceil(#page_const_name as u64); for page_id in 1..count { - let index = parse_data_page::<{ #page_const_name as u32}, { #inner_const_name as usize }>(&mut data_file, page_id as u32).await?; + let index = parse_data_page::<{ #page_const_name as u32}, { #inner_const_name as usize }, { #page_const_name as u32 }>(&mut data_file, page_id as u32).await?; data.push(index); } (data, info) diff --git a/codegen/src/persist_table/generator/space_file/worktable_impls.rs b/codegen/src/persist_table/generator/space_file/worktable_impls.rs index c3d19f7d..e67694f0 100644 --- a/codegen/src/persist_table/generator/space_file/worktable_impls.rs +++ b/codegen/src/persist_table/generator/space_file/worktable_impls.rs @@ -33,13 +33,13 @@ impl Generator { /// Retires an Arc-owned table generation after the caller's /// quiesce barrier has stopped new leases and drained old ones. pub async fn unload_gracefully( - self: std::sync::Arc, - timeout: std::time::Duration, + self: worktable::prelude::Arc, + timeout: core::time::Duration, quiesce: F, ) -> Result> where F: FnOnce() -> Fut, - Fut: std::future::Future, + Fut: core::future::Future, { // Attribute the generation at the retirement request. The // quiesce callback can give background maintenance time to @@ -54,10 +54,10 @@ impl Generator { )); } - let owned = match std::sync::Arc::try_unwrap(self) { + let owned = match worktable::prelude::Arc::try_unwrap(self) { Ok(owned) => owned, Err(arc) => { - let outstanding = std::sync::Arc::strong_count(&arc).saturating_sub(1); + let outstanding = worktable::prelude::Arc::strong_count(&arc).saturating_sub(1); return Err(UnloadFailure::retained( arc, eyre::eyre!("cannot unload generation: {outstanding} Arc lease(s) remain"), @@ -80,7 +80,7 @@ impl Generator { /// Returns the physical size of this table's `.wt.data` file. /// Persisted vacuum makes freed pages reusable across reloads, /// but does not truncate this file. - pub async fn persisted_data_file_size_bytes(&self) -> std::io::Result { + pub async fn persisted_data_file_size_bytes(&self) -> Result { self.1.persisted_data_file_size_bytes().await } } @@ -184,6 +184,7 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let pk_type = name_generator.get_primary_key_type_ident(); let const_name = name_generator.get_page_inner_size_const_ident(); + let page_const_name = name_generator.get_page_size_const_ident(); if self.attributes.pk_congee { // Congee durability is maintained by its native checkpoint/WAL. quote! {} @@ -198,7 +199,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(UnsizedIndexPage::from_node(node.as_ref())); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } @@ -214,7 +215,7 @@ impl Generator { for node in shadow.snapshot_nodes() { pages.push(IndexPage::from_node(&node, size)); } - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } @@ -226,7 +227,7 @@ impl Generator { let page = UnsizedIndexPage::from_node(node.as_ref()); pages.push(page); } - let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } @@ -257,7 +258,7 @@ impl Generator { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); let mut pages = vec![]; #collect_pages - let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }, { #page_const_name as u32 }>(pages); (toc.pages, pages) } } diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index b7820ef8..fae07538 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -481,9 +481,14 @@ mod tests { ); } + /// This used to assert the opposite. A persisted table was refused any page + /// size but 16384, because the seeks computed every offset from a hardcoded + /// constant while the generated table threaded the configured one, so the + /// two disagreed and the file was silently corrupt. Both take the stride as + /// a parameter now. #[test] - fn persisted_tables_reject_non_default_page_size() { - let error = expand(quote! { + fn persisted_tables_accept_a_non_default_page_size() { + expand(quote! { name: PersistedSmallPages, persist: true, columns: { @@ -493,10 +498,27 @@ mod tests { page_size: 8192, } }) + .expect("a persisted table may choose its page size"); + } + + /// What is left of the rule: a page on disk carries a 28-byte header, so + /// one this small is mostly header. + #[test] + fn persisted_tables_reject_a_page_smaller_than_the_floor() { + let error = expand(quote! { + name: PersistedTinyPages, + persist: true, + columns: { + id: u64 primary_key, + }, + config: { + page_size: 64, + } + }) .unwrap_err(); assert!( - error.to_string().contains("cannot be combined with `persist: true`"), + error.to_string().contains("below the 512-byte"), "unexpected error: {error}" ); } diff --git a/docs/TODO.md b/docs/TODO.md index 15824525..6e8d0a85 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -3,7 +3,9 @@ What is known to be unfinished, and enough context to act on it without the conversation it came from. Ordered by whether it blocks a release. -Last reviewed 2026-09-04, against `master` after beta.17 publication. +Last reviewed 2026-09-07. Sections below still describe the repository as of +beta.17; `master` is now at 1.0.0-beta.19 and this file has not been swept for +what those two releases closed. ## Closed, and how @@ -111,46 +113,115 @@ yank beta.16 once beta.17 supersedes it. ## Not blocking, but wrong today -### `congee-wt` still pulls `crossbeam-epoch` - -beta.16 removes crossbeam from WorkTable's own reclamation, not from the build. -`congee-wt` depends on it directly and re-exports its `Guard`, which -`src/index/congee.rs:101` names in a signature; `crossbeam-skiplist` also -arrives under `WorkTablesIndex` and `indexset`. - -congee's use is shallow: 34 references, none of them `Atomic<>`, `Owned::` or -`Shared<>`, and most in tests. It only ever calls `pin()` and passes `Guard` -around as an opaque token, so porting it to `ps-reclaim` is mechanical. The -catch is that `Guard` is in congee-wt's public API, so it is a breaking change -there plus the call sites here. - -`arctic-wt` should **not** be ported. It reclaims through `seize`, and is right -to: a trie with short reads reaches quiescence constantly, which is the exact -property that makes `seize` wrong for this crate, where `select` holds a read -guard. - -### Persistence stalls on a primary index event gap, rarely - -One run of `cargo test --workspace --all-targets --all-features` failed with - - persistence stalled on primary index event gap: last applied Id(1439), - next available Id(1455) (attempt 9) - -in `tests/persistence/loaded_index_growth.rs`. Not a flaky timeout: the guard -at `src/persistence/operation/batch.rs:346` is deliberate, added in `c0c06ba`, -and its comment says a gap that persists past eight deferrals means an event id -was consumed without its event being queued, which only non-CDC index mutations -do. The gap is 16 ids wide. - -1 failure in 6 full runs on this branch, 0 in 3 on master, 0 in 15 -persistence-only runs, so it needs whole-suite load and is not a beta.16 -regression. Do not start with a repro hunt: instrument `IndexChangeEventId` -assignment against event queueing so the next occurrence names its own cause. -Evidence at `~/code/wt-event-gap-2026-09-01.txt`. - +### `congee-wt` no longer pulls `crossbeam-epoch` + +Corrected 2026-09-07. This section said the port was outstanding and mechanical. +Both halves were wrong. + +`congee-wt` dropped `crossbeam-epoch` in 0.4.4. `Cargo.toml:22` now reads +`ps-reclaim = { version = "0.1.4", default-features = false, features = +["libc", "spin"] }`, no `crossbeam_epoch` reference survives in its sources or +tests, and this repository's `Cargo.lock` already resolves `congee-wt 0.4.4`. +The two remaining `crossbeam` strings in that crate are attribution comments on +a seqlock and a backoff loop. + +The call site this section worried about needed no change. It has moved to +`src/index/congee.rs:120` and still reads +`fn retire_old(pointer: usize, guard: &congee::epoch::Guard) -> Arc`. The +`congee::epoch::Guard` path was deliberately preserved across the port, and the +lifetime the new guard carries elides in reference position. + +The port was not the mechanical rename described here. A literal swap would have +kept one global epoch; what shipped gives each tree its own `Domain`, adds a +bounded pending-retire batch, checks guard provenance so a guard from another +tree panics rather than corrupting, and drains the tree's own domain on `Drop`. +Worth knowing because the new `Guard` is `!Send` and tree-scoped, so a guard may +not be created outside the thread and tree that uses it. Nothing in either +repository does; every threaded test builds its guard inside the spawned +closure. + +### Persistence event gap: three leak sites found, instrumented, and fixed + +Updated 2026-09-07. This section previously said the cause was unknown and that +the next step was instrumentation rather than a repro hunt. The instrumentation +was built, and reading for it found the leaks. + +`IndexChangeEventId` is `indexset::cdc::change::Id`, allocated by +`event_id.fetch_add` in the same statement that stamps the event, so indexset +never consumes an id without emitting its event. Every leak is on our side: an +event handed back and then dropped. Three sites, all in generated persisted +query code, all on secondary streams, all confirmed by reading: + +- `codegen/src/generators/persist/queries/update.rs:569`. The + `IndexError::NotFound => Err(WorkTableError::NotFound)` arm returns with no + acknowledge, while its sibling `AlreadyExists` arm immediately above builds an + `Acknowledge` carrying `merged_events` and applies it. The events from + `process_difference_insert_cdc` are dropped on the `NotFound` path. This + asymmetry between two adjacent arms is the clearest of the three. +- `codegen/src/generators/persist/queries/update.rs:593`, + `gen_process_diffs_remove_on_index`: `let (secondary_keys_events_remove, res) + = ...; res?;`. On `Err` the `?` returns before + `op.extend_secondary_key_events`, dropping the events bound on the line above. +- `codegen/src/generators/persist/queries/delete.rs:101`: the same `res?;` shape + after `delete_row_cdc`. + +Checked and NOT leaks: the rollback arms of `insert_cdc`, `insert_many_cdc` and +`reinsert_cdc` in `src/table/mod.rs` all merge forward and rollback events into +an `Acknowledge`, as does the data-delete-failure restore path. Vacuum's +`update_index_after_move` takes the non-CDC branch only when `persistence` is +`None`, so the one case commit `c0c06ba` named is closed for persisted tables. + +The only structurally possible primary-stream leak is a refused +`apply_operation` after the index mutation already consumed ids, and more +generally any `?` between a CDC index mutation and its `apply_operation`. + +**The failure text quoted in earlier versions of this section is stale.** It +said "attempt 9"; `GIVE_UP_AFTER_ATTEMPTS` is now 120, and +`COLLECT_WHOLE_QUEUE_AFTER_ATTEMPTS` plus its regression test +`collection_recovers_when_event_order_and_operation_order_disagree` were added +since, for a symptom that reads identically but is a collection failure rather +than a leak. Telling those two apart is exactly what the new ledger does. + +`src/persistence/event_ledger.rs` records queued, collected, requeued, trimmed +and applied per stream in a bounded 8192-id window, and the guard's message now +ends in a verdict: either ASSIGNED BUT NEVER QUEUED with the id range and the +producer sites either side of the gap, or QUEUED BUT NOT APPLIED with the per-id +stage history. It says so plainly when part of the gap fell outside the retained +window, so it never claims "never queued" about an id it cannot answer for. +Always compiled, gated at run time on `debug_assertions` or `WT_EVENT_LEDGER`, +which puts it on exactly where the bug appears, since the stall needs a full +debug `--all-features` run. + +**Fixed 2026-09-08.** All three sites now do what the rollback arms already did: +build an `Acknowledge` carrying the orphaned events and apply it before +propagating the error. The two `res?` sites became `if let Err(e) = res` so the +events are moved into the acknowledge and the error is returned explicitly, and +the `NotFound` arm acknowledges the events its sibling arm was already +acknowledging. + +The events are **moved** into the acknowledge rather than cloned, and that is +load-bearing rather than tidy. Cloning them fails to compile: the events type is +still an inference variable at that point in the generated code, pinned only by +the `op.extend_secondary_key_events` call further down, and method resolution for +`.clone()` needs the type resolved where the call is written. The result is an +`E0282` reported against the `worktable!` invocation with no inner span, which is +an expensive thing to diagnose twice. + +Covered by emitted-token assertions in both generators +(`indexed_update_write_failure_unwinds_and_acknowledges` and +`delete_data_failure_restores_indexes_and_acknowledges`), which assert the +acknowledge is emitted **before** the return or the extend rather than merely +present somewhere in the output. The write failure itself is not forcible through +the public API, so the wiring is pinned on the tokens, which is the same approach +those tests already took. + +The in-memory generator has the same two `NotFound` arms +(`codegen/src/generators/in_memory/queries/update.rs:528` and `552`) and they are +correctly untouched: there is no persistence stream behind them to gap. ## Housekeeping -- `CHANGELOG.md` stops at 0.4.1, long before the 1.0.0-beta line. +- `CHANGELOG.md` was backfilled to 0.3.10 on 2026-09-08. It previously stopped at + beta.18. - `.github/workflows/rust.yml` has no `cargo fmt --check` job, so formatting drift accumulates unnoticed; `scripts/ci-local.sh` does check it, which makes the script stricter than CI rather than equal to it. diff --git a/docs/crate.md b/docs/crate.md index 993dc840..12155086 100644 --- a/docs/crate.md +++ b/docs/crate.md @@ -8,7 +8,7 @@ does not provide multi-table transactions or multi-process access. ## In-memory quick start ```rust -# fn main() { futures::executor::block_on(async { +# fn main() { nagoya::block_on(async { use worktable::prelude::*; use worktable::worktable; @@ -38,7 +38,7 @@ String and tuple primary keys accept borrowed forms, so callers do not need to write an explicit clone merely to perform a lookup or delete. ```rust -# fn main() { futures::executor::block_on(async { +# fn main() { nagoya::block_on(async { use worktable::prelude::*; use worktable::worktable; diff --git a/docs/page-size.md b/docs/page-size.md new file mode 100644 index 00000000..f33eee9b --- /dev/null +++ b/docs/page-size.md @@ -0,0 +1,105 @@ +# Page size: every place it is decided + +A WorkTable page has two sizes and they are not interchangeable. + +* **stride** is what one page occupies on disk, header included. It is what + every file offset is computed from, and getting it wrong puts a read in the + middle of a neighbouring page. In `data_bucket` it is the const generic + spelled `STRIDE`, a `u32`. +* **inner size** is the stride less the 28-byte `GeneralHeader`: how many bytes + of row or index content a page can hold. It is spelled `INNER_PAGE_SIZE`, + `DATA_LENGTH`, or `DATA_INNER_LENGTH` depending on where you are. + +A generated table emits both as constants named after itself, so `SomeTable` +gets `SOME_TABLE_PAGE_SIZE` and `SOME_TABLE_INNER_PAGE_SIZE`. Every location +below either defines one of those, threads one through, or consumes it. + +## What it costs to get this wrong + +A persisted table used to be refused any page size but 16384. The seeks computed +every offset from a hardcoded constant while the generated table threaded the +configured one, so the two disagreed and the file was silently corrupt. Both now +take the stride as a parameter and `page_size` works for a persisted table; +`tests/persistence/custom_page_size.rs` writes one at 8192, checks the file spans +several pages of that size, and reloads every row. + +**Three separate places had to be found before that passed, and each was silent.** +They are worth knowing because the next such parameter will hide in the same +kind of place: + +* `check_value_write_bounds` measured a slot write against the crate default + rather than the page. At a smaller page it let the write run past the page end + into its neighbour and returned success. +* one of two `WorkTable<...>` emissions in the in-memory generator hardcoded + `INNER_PAGE_SIZE` where the other used the table constant, so an in-memory page + and a persisted one disagreed whenever the two differed. +* `SpaceLogicalIndex` and its three siblings wrapped `SpaceIndex` without passing + a stride, and a `= DEFAULT_PAGE_STRIDE` default on that parameter made it + compile. The index file was then written at 16384 and read at 8192. + +That last one is the lesson: **the defaults were removed.** Every instantiation +now names its stride, so a wrapper that forgets to thread it fails to compile +rather than quietly picking 16384. + +## data_bucket + +| Location | What it decides | +|---|---| +| `src/page/mod.rs` | `PAGE_SIZE` (the 16384 default), `INNER_PAGE_SIZE`, and `DEFAULT_PAGE_STRIDE`, which is `PAGE_SIZE` as a `u32` so callers wanting the default need no cast in generic position | +| `src/page/util.rs` `page_start_offset` | The only multiplication of a page index by a stride. Everything else goes through it | +| `src/page/util.rs` `seek_to_page_start`, `seek_to_page_start_relatively`, `seek_by_link` | The three seeks | +| `src/page/util.rs` `persist_page`, `persist_page_in_place`, `persist_pages_batch` | The writes. `persist_page_in_place` also checks the payload against `STRIDE - GENERAL_HEADER_SIZE` rather than the crate default | +| `src/page/util.rs` `update_at` | In-place row rewrite. Two parameters because the bound it checks and the offset it seeks to are different quantities | +| `src/page/util.rs` `parse_page`, `parse_pages_batch`, `parse_general_header_by_index`, `parse_data_page`, `parse_data_pages_batch` | The reads. `parse_data_page` and `parse_data_pages_batch` already took a `const PAGE_SIZE: u32` that nothing used; the stride is a separate parameter and that one is still the payload length | +| `src/page/index/mod.rs` `IndexPageUtility` | `parse_index_page_utility` and `persist_index_page_utility`. The default body's overflow check uses the page's own capacity | +| `src/page/index/page.rs` | `read_value_with_index`, `persist_value`, `remove_value` | +| `src/page/index/page_for_unsized.rs` | `persist_value`, `read_value_with_offset` | +| `src/page/iterators.rs` | Reads at `DEFAULT_PAGE_STRIDE`. A standalone file reader with no table to ask | + +## worktable + +| Location | What it decides | +|---|---| +| `src/table/mod.rs` | `WorkTable<..., const DATA_LENGTH: usize = INNER_PAGE_SIZE, ...>`: the in-memory data page size | +| `src/in_memory/data.rs` | `DATA_INNER_LENGTH`, the row area of an in-memory page, and the one place still fixed to the crate default | +| `src/persistence/space/data.rs` | `SpaceData`. Its `PAGE_SIZE` is the stride and is passed to every data-file call | +| `src/persistence/space/index/mod.rs` | `SpaceIndex` | +| `src/persistence/space/index/unsized_.rs` | `SpaceIndexUnsized` | +| `src/persistence/space/index/util.rs` | `map_index_pages_to_toc_and_general` and its unsized form build a table of contents, so they carry the stride it will be read at | +| `src/persistence/space/logical_index.rs` | `SpaceLogicalIndex`, `SpaceLogicalIndexUnsized`, `SpaceLogicalMultiIndex` and `SpaceLogicalMultiIndexUnsized` each wrap one of the above and pass the stride through | +| `src/persistence/space/index/table_of_contents.rs` | `IndexTableOfContents`. It lives in the index file, so it takes that file's stride | +| `src/migration/mod.rs` | Reads at `DEFAULT_PAGE_STRIDE`. It opens files whose table it has not loaded | + +There are no defaults on these parameters. A default is what let +`SpaceLogicalIndex` wrap `SpaceIndex` without a stride and take 16384 in +silence, so every instantiation names it and the compiler finds the ones that +do not. + +## worktable_codegen + +| Location | What it decides | +|---|---| +| `generators/in_memory/table/mod.rs` `gen_page_size_consts` | Emits `_PAGE_SIZE` and `
_INNER_PAGE_SIZE` from `config.page_size`, or from the crate default when it is absent. **This is the hook.** The same function exists in `generators/persist/table/mod.rs` and `generators/read_only/table/mod.rs` | +| `generators/in_memory/table/mod.rs` | The emitted `WorkTable<...>` takes the inner constant as its `DATA_LENGTH`. There are two such emissions in this file and only one of them used to be parameterised | +| `persist_index/space/index.rs`, `persist_table/generator/space.rs` | Instantiate `SpaceIndex` and `SpaceIndexUnsized` with the inner constant and the page constant | +| `persist_index/generator.rs`, `persist_table/generator/space_file/mod.rs` | Every emitted `parse_page`, `persist_page` and `parse_data_page` call passes the table's page constant as the stride | + +## worktable_dsl + +| Location | What it decides | +|---|---| +| `dsl/src/parser/config.rs` | Parses `page_size` out of the `config` block | +| `dsl/src/model/config.rs` | `Config::page_size` and the span kept for diagnostics | +| `dsl/src/validate.rs` `validate_page_size` | Refuses a non-default size on a persisted table, and explains which half of the problem is still open | +| `dsl/src/validate.rs` `validate_arctic_page_size` | Refuses a size above 65535 for an Arctic-backed table: Arctic packs a link into one `u64` with 16-bit offset and length fields | + +## If you add another location + +1. Do not give the stride a default. Every one of the three bugs above was a + place that compiled because something else supplied 16384. +2. Assert file lengths as well as a round trip. A round trip alone proves + nothing: a table that silently fell back to the default still reads its own + writes. +3. `DataPage` is an inline `[u8; N]`, so a 32768-byte page overflows the + stack of a debug-build test before it reaches any of this. Measure large + pages in release. diff --git a/docs/paper-2-plan.md b/docs/paper-2-plan.md new file mode 100644 index 00000000..474414be --- /dev/null +++ b/docs/paper-2-plan.md @@ -0,0 +1,115 @@ +# Paper two: plan and evidence map + +Written 2026-09-06 against master `d5656e6` (1.0.0-beta.19). Companion to the CIDR 2027 +submission (beta.6, submitted for the 2026-08-04 deadline; notification 2026-10-06). + +## Thesis candidates + +The CIDR paper argued *compile-time engine specialization*. It deferred (§5, §7): the lock +discipline scaling comparison, crash consistency, formal checking of the protocols, the cost +of monomorphization, and external baselines beyond redb/LMDB. Paper two should be the paper +those deferrals point at, not a re-statement of the thesis. + +**A. Lifecycle paper (recommended).** "Row lifecycle in a specialized engine: exact-cell +locking, epoch reclamation, and reactive vacuum without a transaction manager." Everything +in it has landed since beta.6 and has numbers. Contributions: + +1. Exact-cell synchronization replacing hashed stripes and the table-global barrier + (`src/in_memory/data.rs` `CellLocks`; `docs/versioned-row-publication.md`). +2. Quiescent-state reclamation via a one-word `!Send` guard (`ps-reclaim`), replacing a + global reader counter; the `Send`-guard use-after-free found on the way is a good + cautionary section (`docs/TODO.md` "ps-reclaim 0.1.1"). +3. Reactive vacuum planned from the free-range registry, gated by a mutation lease and a + quiet epoch; root-cause note that all four reclaim bugs came from indexes storing + physical addresses (`docs/vacuum-design-directions.md`, `src/table/vacuum/`). +4. The 1-32 thread grid across three index backends: the lock-discipline scaling result the + CIDR paper promised (`docs/beta18-validation.md`). +5. `partition_by` with the loom model of the slot protocol (`src/partition/loom_tests.rs`), + the first model-checked component; HFT review that produced `partition_ref` + (`wt-review.md`). + +**B. Persistence paper.** "Index persistence by replaying the tree's own CDC stream, and what +it costs." Needs the durability work first: structural CDC is ~28% of a 126 ns insert +(`docs/wti-dirty-generation-persistence-plan.md`); ART logical WAL exists +(`src/persistence/space/art_index.rs`) but data pages and WTI still end at `flush()`; the +watermark/fsync design is proposal only (`docs/durability-visibility-proposal.md`). Not +ready before a Q1 2027 deadline unless the journal lands. + +**C. Schema-as-IR paper (PL venue).** `worktable_dsl`: schema read as data, diff with a +per-change cost model, declarations baked into generated code, `wt-dsl` CLI, migration +engine with on-disk version detection (`dsl/`, `docs/migration.md`). Fits PEPM / OOPSLA +better than a DB venue; overlaps with the PEPM plan in the other session. + +## Evidence already in hand (all M4 Max, local trees; needs a pinned Linux rerun) + +| Claim | Number | Source | +|---|---|---| +| Hot-page writer, exact-cell vs hashed | 322 ns vs 1,536 ns | beta18-validation | +| Per-page vs table-global barrier | +25% @4, +65% @8 disjoint writers | versioned-row-publication | +| Generation retire, beta.18 vs 15 | 33.6x (WTI), 16.1x (Arctic) | beta18-validation | +| Read scaling best/1T at 16 threads | 3.6x / 3.3x / 2.6x per backend | beta18-validation | +| 10%-write mix ceiling | peaks at 4 threads | beta18-validation (also beta.13/15) | +| Reactive vacuum foreground penalty | -1.1..-5.1% vs 16-41% unpaced | beta17/18-validation | +| Vacuum reclamation | 18/18 cells, 196 pages, 100% | beta17-validation | +| Point lookup vs stripped index | 15.65 ns vs 9.25 ns bare Arctic vs 33.5 Vec+BTreeMap | beta18-validation | +| Memory overhead | 0.14 B/row over control | beta18-validation | +| Partition route | 0.73 ns Vec vs 9.5 ns string hash; `partition_ref` 3.35 ns | TODO.md, partition docs | +| Persisted insert, beta.15 to 18 | -33..-42% (Arctic 6,130 to 3,753 ns/row) | beta18-validation | +| CDC share of insert | ~35 ns of 126 ns | wti-dirty-generation plan | + +Not in hand: `paper-bench/results/` (never committed), `compile_cost.sh` never run, no +sled/SQLite/DashMap baselines, no beta.19 rerun of Table 2. + +## Consumer workloads (surveyed 2026-09-06, all under ~/code) + +| Repo | Shape | What it gives the paper | Open? | +|---|---|---|---| +| `agentcode` | 11 tables, 8 persisted, Arctic on nearly every index, u128 keys. Stress = 8 concurrent 800-file `update_latency` processes. | The concurrency bug that motivates the paper: `docs/known-defects.md:70-137`, torn/corrupt page header in secondary-index batch apply at beta.11, 6/8 runs failing, 28/28 after moving to Arctic. Map it to the beta.18 fix ("torn reads and premature physical-link reuse") and show the same harness clean on beta.19. Also: Arctic vs WTI at 20k rows, insert 2.18M/s vs 0.89M/s, lookup 17.0M/s vs 5.0M/s (`docs/benchmarks/index-backends.md`); state 42.4 to 22.2 MB after u128 keys (`state-growth.md`); the request for a non-unique fixed-width index that became `ArcticMultiIndex`. | Proprietary | +| `agencyzero` | 19 persisted tables, String PKs, migration engine, `LoadMode::Recovery`, single-writer flock, QA fixture of 248 projects (~30 MB store). | The reclamation case study: `docs/store-recovery.md` records four production corruptions, including a variable-width index page that forgot fragmentation across restart (174 live entries, 2,664 B dead, 64 B tail overlap) and beta.5 whole-row rebuilds churning `pr_project_idx` (38 disagreeing rows) fixed by in-place updates. Production migrations via schema fingerprint. This is the "why indexes must not store physical addresses" story with real data. | Private (GitHub) | +| `karen` | 2 persisted tables, tiny (~700 rows). | Row-level, queryable, durable learned session state instead of one blob: the per-turn Confirm write-through gives 15-20 points top-1. One paragraph of motivation, not evaluation. Uses `unload_gracefully`. | Closed | +| `ekopathrs` | No `worktable!` at all. Uses `worktable-vec::AtomicKeyTable` for two in-memory profiling tables. | The honest negative: `docs/STORAGE-REVIEW.md` rejects full WorkTable (318-package resolve, no `no_std`) for 399 entries. Cite as the boundary of the design space; `worktable-vec` is the lock-free, `no_std` sibling. | Private | + +Use agentcode as the headline stress workload in §4 alongside the shadow-state harness; use +agencyzero as the recovery/fragmentation case study; mention karen and ekopathrs in one +paragraph each in the experience section. Get written OK before naming private repos. + +## Gaps to close for option A + +- Rerun the beta.18 grid and `paper-bench` on a quiet pinned x86 box; commit `results/`. +- Lock-discipline ablation as a proper figure: field vs row vs table lock, 1-32 threads, + skewed keys (`paper-bench/src/bin/contention`). +- Semi-formal statement of the cell-lock + reclamation invariants; ideally extend loom + beyond partitions to the cell/retire path (the CIDR reviewers will ask). +- Wart sweep: 9 `todo!()` sites remain (`codegen/.../queries/in_place.rs`, `update.rs`, + `src/features/s3_support.rs`), `Avaiable` typo in 2 files. +- Merge or explicitly exclude `feat/columnar-fields-indexes` (branch, Aug 6, unmerged). + +## Target: EDBT 2027, 3rd cycle (verified 2026-09-06) + +- Submission **2026-10-07, 5pm PST** (31 days out). Author feedback 11-19, notification + Acc/Rej/Revise 12-05, revised paper 2027-01-04, final 01-27, camera-ready 02-10. + Conference Lille, April 6-9, 2027. +- Paper types: Research long (12p) or short (6p, title prefixed "[Short Paper]"), + Experiments & Analysis, Vision (6p). Topics list includes "Concurrency control, recovery, + and transaction management", "Storage, indexing, and physical database design", + "Data management on modern hardware", "Benchmarking and performance evaluation". +- The revise cycle matters: a paper that gets "revise" on Dec 5 has until Jan 4 to add + the pinned-Linux rerun, so the Oct 7 draft can ship on the M4 grid with the caveat stated. +- CIDR notification is Oct 6, one day before: paper two cannot depend on the outcome and + must not overlap the CIDR text (still under review until then). Option A is disjoint by + construction; cite the CIDR paper as "under submission". + +Alternatives if A slips: ICDE 2027 R2 (2026-11-11), PVLDB rolling (monthly to 2027-03-01), +SIGMOD R4 (2026-10-17). DaMoN 2027 CFP not posted. + +## 31-day schedule for option A (long paper) + +| Week | Dates | Deliverable | +|---|---|---| +| 1 | Sep 7-13 | Freeze the claim list. Run `paper-bench` contention (field/row/table/inplace, 1-32 tasks) and beta.18 grid on the pinned Linux box if available, else M4 with three rotated passes; commit `results/`. Decide long vs short by Sep 13 based on whether the scaling figure holds. | +| 2 | Sep 14-20 | Draft §2 protocols (cell lock, retire, vacuum lease) with invariants stated; §3 partition + loom. Wart sweep PR (`todo!()`, `Avaiable`). | +| 3 | Sep 21-27 | Draft §4 evaluation from `results/`; figures; related work (Hekaton, epoch/QSBR: Fraser, Hart et al., Bw-tree, OLC, DaMoN vacuum/compaction lineage). | +| 4 | Sep 28-Oct 4 | Full read-through, internal review, page trim to 12. | +| 5 | Oct 5-7 | Buffer. Submit by Oct 6 evening local time (Oct 7 5pm PST is 07:00 Oct 8 in Bangkok, but do not use it). | + +Short-paper fallback (6p): contributions 1-3 only, one scaling figure, one vacuum figure. diff --git a/docs/wt-user-guide.pdf b/docs/wt-user-guide.pdf new file mode 100644 index 00000000..38928a84 Binary files /dev/null and b/docs/wt-user-guide.pdf differ diff --git a/docs/wt-user-guide.typ b/docs/wt-user-guide.typ new file mode 100644 index 00000000..64ad738a --- /dev/null +++ b/docs/wt-user-guide.typ @@ -0,0 +1,276 @@ +#set document(title: "WorkTable User Guide", author: "PathScale") +#set page(paper: "a4", margin: (x: 2.2cm, y: 2.4cm), numbering: "1") +#set text(font: ("Helvetica", "Arial"), size: 10pt) +#set par(justify: true, leading: 0.62em) +#show heading: set block(above: 1.4em, below: 0.7em) +#show heading.where(level: 1): set text(size: 17pt, weight: "bold") +#show heading.where(level: 2): set text(size: 12.5pt, weight: "bold") +#show heading.where(level: 3): set text(size: 10.5pt, weight: "bold") +#show raw.where(block: true): it => block( + fill: rgb("#f4f4f2"), inset: 9pt, radius: 3pt, width: 100%, breakable: false, text(size: 8.5pt, it), +) +#show raw.where(block: false): it => box(fill: rgb("#f0f0ee"), inset: (x: 2.5pt, y: 0pt), outset: (y: 2.5pt), radius: 2pt, text(size: 9pt, it)) +#show link: set text(fill: rgb("#1a4f8a")) +#let note(title, body) = block( + fill: rgb("#fbf6e8"), stroke: (left: 2.5pt + rgb("#c8a13a")), inset: 9pt, radius: 2pt, width: 100%, + [*#title.* #body], +) + +#align(center)[ + #text(size: 26pt, weight: "bold")[WorkTable] + #v(-0.4em) + #text(size: 11pt, style: "italic")[Absolutely not a database.] + #v(0.6em) + #text(size: 9.5pt)[A user's guide to the `worktable!` macro, its queries, its indexes and its + persistence tier. Written against 1.0.0-beta.19.] +] +#v(1.2em) + += What this is + +Embedded table storage for Rust. You declare a table with a macro and get a typed +struct back: a primary key, secondary indexes, and generated queries. Rows live in +memory as paged, zero-copy records. Persisting them to local disk or to S3 is opt-in. + +If you have used .NET's `DataTable` this will feel familiar. The differences are that +the type is generated for you, and that persistence is one feature away. + +#note("What it is not")[There is no transaction journal and no fsync on every batch. +A mutation returning means the change was accepted and queued, not that it is on +stable storage. Section 6 says exactly what each boundary guarantees.] + += Getting started + +```sh +cargo add worktable +``` + +A table is one macro invocation. The name is the only required key beyond the columns. + +```rust +use worktable::prelude::*; +use worktable::worktable; + +worktable! ( + name: Order, + columns: { + id: u64 primary_key autoincrement, + symbol: String, + quantity: u64, + }, + indexes: { + symbol_idx: symbol, + } +); +``` + +That generates `OrderWorkTable`, `OrderRow`, `OrderPrimaryKey`, and a `select_by_symbol` +method from the index. Nothing is written by hand per table. + +```rust +let table = OrderWorkTable::default(); +table + .insert(OrderRow { id: table.get_next_pk().into(), symbol: "ETH".into(), quantity: 3 }) + .await?; +let found = table.select_by_symbol("ETH".into()).execute()?; +``` + +Mutations are `async`; reads are not. This declaration and these three calls are +compiled and run by `examples/guide_check.rs`, so the guide cannot drift from the API +without the build noticing. + += Declaring a table + +The grammar is positional at the top and block-structured below it. The order is +*name, version, persist, partition_by*, and then the blocks `columns`, `indexes`, +`queries`, `config`, in any order. Putting `persist` after a block is an error that +names the required order rather than failing as an unexpected token. + +== Columns + +Each column is `name: Type` followed by any inline attributes. `primary_key` is +required on exactly one column, or on several to form a tuple key. `autoincrement` +asks the table to generate the key. `optional` makes the column an `Option`. + +== Indexes + +Each entry is `name: column`, optionally `unique`, optionally `using `. +A non-unique index maps one key to many rows. Every index adds a `select_by_` +method. + +== Queries + +Beyond the generated `select`, `insert`, `insert_many`, `upsert`, `update`, `delete` +and `select_all`, the `queries` block declares your own update and delete shapes. + +#note("in_place queries")[An `in_place` query hands you a mutable reference to the +archived column bytes and skips index maintenance entirely, so a column covered by any +index cannot be mutated that way. The macro refuses it rather than letting an index go +stale.] + += Index backends + +An index can name its physical structure with `using`. Four are available and they +differ in what they can express, not only in speed. + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Backend*], [*When it fits*], + [`worktables_index`], [The general one, and the default for persisted indexes. Takes an ordered key of any type.], + [`indexset`], [Vanilla IndexSet, selectable explicitly while keeping the same disk representation.], + [`arctic`], [Fixed-width keys only, and the fast one. Packs a row link into a single `u64`.], + [`congee`], [Fixed-width integer keys. Refuses `String` and other variable-width types.], +) + +Arctic and Congee must state `persist: true` or `persist: false` explicitly, because +their persistence uses native checkpoint and WAL adapters rather than the shared page +format. + +#note("Arctic and page size")[Arctic packs a link into 64 bits with 16-bit offset and +length fields, so it cannot address a page larger than 65535 bytes. The macro checks +this and refuses the combination.] + += Page size + +A page has two sizes and they are not interchangeable. The *stride* is what one page +occupies on disk, header included, and every file offset is computed from it. The +*inner size* is the stride less the 28-byte header: what a page can actually hold. + +Set it in the `config` block: + +```rust +worktable! ( + name: Small, + columns: { id: u64 primary_key, v: u64 }, + config: { page_size: 4096 } +); +``` + +#note("Persisted tables have a floor, not a fixed size")[`page_size` works for a +persisted table, and the only rule is a 512-byte minimum: a page on disk carries a +28-byte header, so anything much smaller is mostly header. An Arctic-backed table is +also capped at 65535. In-memory tables have neither limit. + +It was refused outright until recently, because the seeks computed offsets from a +hardcoded constant while the table threaded the configured one. Every location that +decides a page size, and the three silent bugs found while making them agree, are in +`docs/page-size.md`.] + += Persistence + +Persistence is implemented, not planned. Add `persist: true` and load the table through +an engine. + +```rust +let config = DiskConfig::new_with_table_name(dir, OrderWorkTable::name_snake_case(), OrderWorkTable::version()); +let engine = OrderPersistenceEngine::new(config).await?; +let table = OrderWorkTable::load(engine).await?; +``` + +S3 layers on top of the disk engine rather than replacing it: +`S3SyncDiskPersistenceEngine` wraps a `DiskPersistenceEngine` and syncs it. Enable the +`s3-support` feature. + +== The durability contract + +This is the part to read before relying on it. + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Boundary*], [*What you actually get*], + [A mutation returns], [The in-memory change was accepted and its persistence operation was queued.], + [`wait_for_ops()` returns], [The engine completed the queued operations. No fsync, no stable-storage guarantee.], + [`close()` returns], [Intake stopped, the queue drained, the engine task joined. Still no fsync guarantee.], + [Process crash or `SIGKILL`], [Acknowledged rows may be lost and the file may be torn.], + [Power loss], [No atomic-batch or stable-storage guarantee.], +) + +Call `close()` during orderly shutdown. `wait_for_ops()` is not a shutdown boundary on +its own: it does not stop another task from queueing later work, so it needs +application-level writer quiescence to mean anything. + +A persistence failure is terminal. An unrecoverable event gap, queue-analysis error, +batch-apply error or engine-task failure moves the table into a failed state, and the +original error is returned to waiters, to `close()`, and to later mutations. + +== Loading a torn store + +A normal load audits archived rows and both primary and secondary index consistency +before exposing the table, and refuses torn state with `PersistenceLoadError` rather +than opening plausible-but-invented rows. + +`LoadMode::Recovery` exists for offline tools only. It copies individually validated +rows through a surviving index into a clean table, which must then pass a normal strict +load before anyone reads it. It is not an in-place repair and must never serve live +traffic. + +== Vacuum + +Persisted vacuum compacts the in-memory layout and keeps disk indexes consistent with +moved rows. It does not truncate `.wt.data`. Watch physical growth with +`persisted_data_file_size_bytes().await` and decide when to snapshot and rebuild. + += The filesystem + +WorkTable reaches the filesystem through one module, `worktable::prelude::fsx`, and +names no async runtime. The file type is `std::fs::File` behind `AllowStdIo`, which +carries the `futures-io` traits the storage layer asks for while keeping blocking +semantics. + +That is a deliberate choice and it was measured: `tokio::fs` ran scattered updates at +12,316 rows per second against 74,728 for the same code on `std::fs`, a factor of 6.1, +with bulk insert within noise and the in-memory control matching. A scattered update is +many small IOs and `tokio::fs` pays a thread-pool round trip for each one. + +#note("Where to put the work")[Because the calls block, a persistence engine should own +a thread rather than share a runtime's worker pool. The calls were never waiting on the +disk through a runtime anyway: the persistence path measured 89 voluntary context +switches across 25,000 inserts.] + += Concurrency + +Indexes are lock-free with change-data-capture, and a row-level `LockMap` gives ordered +access when you need it. Generated reads always use immutable row-version publication, +including in `default-features = false` builds: turning off a Cargo feature must never +expose a safe API that races deserialization against page-byte mutation. + +Point lookups use a strict backend-specific visibility contract by default. +WorkTablesIndex pins the structural mapping until its selected node is locked, so both +hits and misses are definitive. + += Feature flags worth knowing + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Feature*], [*Effect*], + [`std`], [On by default. Off, the crate links no `std` and the whole persistence half is gone with it.], + [`s3-support`], [The S3 sync engine, and the HTTP stack under it.], + [`logical-index-persistence`], [Moves unique structural CDC work off the mutation path into the background worker. The page format is unchanged either way.], + [`wti-predictable-search`], [On by default. The branch-based node search, which avoids a measured regression on sequential numeric keys.], +) + +The three alternative search policies (`wti-hybrid-search`, `wti-std-search`, +`wti-superslice-search`) are compile-time gates. Enable one, and only one, for an +unambiguous build. If feature unification turns on several, WorkTablesIndex applies a +documented precedence rather than refusing the graph. + += Where to look next + +#table( + columns: (auto, 1fr), + stroke: 0.4pt + rgb("#cccccc"), + inset: 6pt, + [*Document*], [*Covers*], + [`docs/persistence-durability.md`], [The durability contract in full, and the snapshot-restore procedure.], + [`docs/index-backend-dsl-proposal.md`], [The `using` syntax and the capability matrix per backend.], + [`docs/page-size.md`], [Every location across the four crates that decides a page size.], + [`docs/queries.md`], [The generated query surface and the custom query grammar.], + [`docs/migration.md`], [Moving a store between formats.], + [`docs/known-issues.md`], [What is known to be wrong right now.], +) diff --git a/dsl/src/validate.rs b/dsl/src/validate.rs index 36415291..174c32ce 100644 --- a/dsl/src/validate.rs +++ b/dsl/src/validate.rs @@ -19,31 +19,42 @@ use crate::model::{Columns, IndexBackend, Persistence}; -/// data_bucket's on-disk layer seeks with its own hardcoded `PAGE_SIZE` of -/// 16384 bytes (`seek_to_page_start`, `seek_by_link`, `persist_page`), while -/// the generated table threads the user's `page_size` through its page-id and -/// length arithmetic. Any other value therefore reads and writes the wrong -/// file offsets as soon as the table persists, silently corrupting it. -/// In-memory tables never seek a file: for them `page_size` only sizes index -/// nodes and stays configurable. -const DATA_BUCKET_PAGE_SIZE: u32 = 16384; +/// Bytes of `GeneralHeader` at the front of every persisted page. A page has +/// to be larger than this or there is no room left for a row. +const GENERAL_HEADER_SIZE: u32 = 28; +/// The smallest persisted page worth allowing. Below this the header is most +/// of the page and the table spends its time on page transitions; the number +/// is a floor against obvious mistakes, not a tuned value. +/// +/// It applies to persisted tables only. An in-memory table writes no header, +/// so its `page_size` only sizes index nodes and a small one is a legitimate +/// choice rather than a mistake. +const MINIMUM_PERSISTED_PAGE_SIZE: u32 = 512; + +/// A persisted table used to be refused any page size but 16384, because +/// `data_bucket` computed every offset from a hardcoded `PAGE_SIZE` in +/// `seek_to_page_start`, `seek_by_link` and `persist_page` while the generated +/// table threaded the configured size through its page-id arithmetic. The two +/// disagreed and the file was silently corrupt. +/// +/// Those seeks take the stride as a parameter now, and the generated table +/// passes its own constant to every one of them, in the data file and the index +/// file alike. The restriction is gone. What is left is the arithmetic that +/// still has to hold. pub fn validate_page_size(config: Option<&crate::model::Config>, persistence: Persistence) -> syn::Result<()> { let Some(config) = config else { return Ok(()) }; let Some(page_size) = config.page_size else { return Ok(()); }; - if persistence.is_persisted() && page_size != DATA_BUCKET_PAGE_SIZE { - let span = config.page_size_span.unwrap_or_else(proc_macro2::Span::call_site); + let span = config.page_size_span.unwrap_or_else(proc_macro2::Span::call_site); + if persistence.is_persisted() && page_size < MINIMUM_PERSISTED_PAGE_SIZE { return Err(syn::Error::new( span, format!( - "`page_size: {page_size}` cannot be combined with `persist: true`: the on-disk \ - layer (data_bucket) hardcodes {DATA_BUCKET_PAGE_SIZE}-byte pages in every file \ - seek, so a persisted table with any other page size reads and writes the wrong \ - pages and corrupts its files. Remove `page_size` (or set it to \ - {DATA_BUCKET_PAGE_SIZE}); custom page sizes remain available for in-memory \ - tables, where they only size index nodes" + "`page_size: {page_size}` is below the {MINIMUM_PERSISTED_PAGE_SIZE}-byte \ + minimum for a persisted table. Each page on disk carries a \ + {GENERAL_HEADER_SIZE}-byte header, so a page this small is mostly header" ), )); } diff --git a/dsl/tests/check.rs b/dsl/tests/check.rs index f10dc2a0..2e882e94 100644 --- a/dsl/tests/check.rs +++ b/dsl/tests/check.rs @@ -59,7 +59,7 @@ fn every_broken_rule_is_reported() { persist: true, columns: { id: u64 primary_key, label: String }, indexes: { label_idx: label unique using congee }, - config: { page_size: 4096 }", + config: { page_size: 64 }", ); assert!(checked.schema.is_some()); diff --git a/dsl/tests/cli.rs b/dsl/tests/cli.rs index bfcfe6b6..2358caa0 100644 --- a/dsl/tests/cli.rs +++ b/dsl/tests/cli.rs @@ -71,10 +71,13 @@ fn scan_finds_declarations_in_a_function_body() { /// declaration". /// /// The difference is not academic and this test exists because the first -/// version of the binary got it wrong. `page_size: 4096` beside -/// `persist: true` parses perfectly: it is a well-formed declaration. The -/// macro refuses it, because the on-disk layer hardcodes 16384-byte pages and -/// any other value reads and writes the wrong file offsets. +/// version of the binary got it wrong. `page_size: 64` parses perfectly: it is +/// a well-formed declaration. The macro refuses it, because a persisted page +/// that small is mostly its own 28-byte header. +/// +/// The fixture used to be `page_size: 4096` beside `persist: true`, refused +/// while the on-disk layer hardcoded its stride. It takes the stride as a +/// parameter now, so that combination is accepted and tests nothing. /// /// A round trip built on `Schema::parse` therefore reports success for output /// that does not compile, which is precisely the mistake a second @@ -82,7 +85,7 @@ fn scan_finds_declarations_in_a_function_body() { /// to stop. #[test] fn a_declaration_the_macro_refuses_is_not_a_successful_round_trip() { - let refused = "name: T, persist: true, columns: { id: u64 primary_key }, config: { page_size: 4096 },"; + let refused = "name: T, persist: true, columns: { id: u64 primary_key }, config: { page_size: 64 },"; // It parses. That is the trap. assert!( diff --git a/examples/guide_check.rs b/examples/guide_check.rs new file mode 100644 index 00000000..e61653bd --- /dev/null +++ b/examples/guide_check.rs @@ -0,0 +1,31 @@ +//! The declaration and the three calls printed in `docs/wt-user-guide.typ`. +//! If the guide drifts from the API, this stops compiling. +use worktable::prelude::*; +use worktable::worktable; + +worktable! ( + name: Order, + columns: { + id: u64 primary_key autoincrement, + symbol: String, + quantity: u64, + }, + indexes: { + symbol_idx: symbol, + } +); + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let table = OrderWorkTable::default(); + table + .insert(OrderRow { + id: table.get_next_pk().into(), + symbol: "ETH".into(), + quantity: 3, + }) + .await?; + let found = table.select_by_symbol("ETH".into()).execute()?; + assert_eq!(found.len(), 1); + Ok(()) +} diff --git a/examples/system_info_render.rs b/examples/system_info_render.rs new file mode 100644 index 00000000..a6e23aa2 --- /dev/null +++ b/examples/system_info_render.rs @@ -0,0 +1,24 @@ +use worktable::prelude::*; +use worktable::worktable; + +worktable! ( + name: Shown, + columns: { id: u64 primary_key autoincrement, symbol: String, qty: u64 }, + indexes: { symbol_idx: symbol, qty_idx: qty } +); + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let table = ShownWorkTable::default(); + for i in 0..5_000u64 { + table + .insert(ShownRow { + id: table.get_next_pk().into(), + symbol: format!("SYM{}", i % 97), + qty: i, + }) + .await?; + } + print!("{}", table.system_info()); + Ok(()) +} diff --git a/src/features/s3_support.rs b/src/features/s3_support.rs index 6a93ced7..4ec19810 100644 --- a/src/features/s3_support.rs +++ b/src/features/s3_support.rs @@ -1,8 +1,9 @@ -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::{string::String, string::ToString}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; +use core::time::Duration; use std::path::Path; -use std::time::Duration; use reqwest::Client; use rusty_s3::{Bucket, Credentials, S3Action, UrlStyle}; @@ -135,7 +136,7 @@ where tracing::debug!(local_path = %local_path.display(), s3_key = %s3_key, "Uploading file to S3"); - let content = tokio::fs::read(local_path).await?; + let content = crate::fsx::read(local_path).await?; let action = self.bucket.put_object(Some(&self.credentials), &s3_key); let url = action.sign(Duration::from_secs(3600)); @@ -191,7 +192,7 @@ where return Ok(()); } - tokio::fs::create_dir_all(table_path).await?; + crate::fsx::create_dir_all(table_path).await?; for obj in parsed.contents { let s3_key = &obj.key; @@ -213,7 +214,7 @@ where let response = client.get(url).send().await?.error_for_status()?; let content = response.bytes().await?; - tokio::fs::write(&local_path, content).await?; + crate::fsx::write(&local_path, content).await?; } tracing::info!(table_name = %table_name, "S3 download sync complete"); diff --git a/src/fsx.rs b/src/fsx.rs new file mode 100644 index 00000000..cccab9f0 --- /dev/null +++ b/src/fsx.rs @@ -0,0 +1,57 @@ +//! The filesystem, without a runtime attached to it. +//! +//! A thin shim over [`nagoya::io`], which is where these operations live now. +//! They were here first, privately, and `data_bucket` could not name them at +//! all, so the two halves of one storage engine disagreed about what a file +//! was. The whole crate still goes through this module, so swapping backends +//! is still swapping one file. +//! +//! # Why the calls block +//! +//! Neither `tokio::fs` nor `async-fs` performs asynchronous file I/O: both hand +//! a blocking `std::fs` call to a thread pool, and what that buys is not +//! occupying a runtime worker rather than any actual overlap. It is not free. +//! Measured on this crate's scattered-update path, `tokio::fs` ran 12,316 rows +//! per second against 74,728 for the same code on blocking `std::fs`, and cold +//! reopen is 2.8x faster on an Arctic index without it. +//! +//! Because the calls block, the persistence engine should own a thread rather +//! than share a runtime's worker pool. It was never waiting on the disk through +//! a runtime anyway: the path measures 89 voluntary context switches across +//! 25,000 inserts. + +/// A file this crate reads and writes. +pub type File = nagoya::io::HostFile; + +pub use nagoya::io::{ + Error, SeekFrom, append, create, create_dir_all, open, open_or_create, read, remove_dir_all, remove_file, rename, + write, +}; + +/// How long the file at `path` is, without opening it. +/// +/// Named for what it does. `nagoya::io::metadata` returns the length rather +/// than a metadata handle, because a length is all anything here ever wanted. +pub use nagoya::io::metadata; + +/// The operations no I/O trait carries, as free functions. +/// +/// They are methods on [`nagoya::io::File`]; these wrappers exist so call sites +/// read the same as they did when this module owned the implementation, and so +/// that a backend swap stays a change to one file. +pub async fn sync_all(file: &mut File) -> Result<(), Error> { + nagoya::io::File::sync_all(file).await +} + +pub async fn sync_data(file: &mut File) -> Result<(), Error> { + nagoya::io::File::sync_data(file).await +} + +pub async fn set_len(file: &mut File, length: u64) -> Result<(), Error> { + nagoya::io::File::set_length(file, length).await +} + +/// How long an already-open file is. +pub async fn file_metadata(file: &mut File) -> Result { + nagoya::io::File::length(file).await +} diff --git a/src/in_memory/data.rs b/src/in_memory/data.rs index 7b15d55e..94c4c590 100644 --- a/src/in_memory/data.rs +++ b/src/in_memory/data.rs @@ -1,8 +1,9 @@ -use std::cell::UnsafeCell; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::ops::{Deref, DerefMut}; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use alloc::vec::Vec; +use core::cell::UnsafeCell; +use core::fmt::Debug; +use core::marker::PhantomData; +use core::ops::{Deref, DerefMut}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use data_bucket::page::INNER_PAGE_SIZE; use data_bucket::page::PageId; @@ -37,7 +38,7 @@ struct CellLocks { impl Default for CellLocks { fn default() -> Self { Self { - slots: std::array::from_fn(|_| AtomicU64::new(0)), + slots: core::array::from_fn(|_| AtomicU64::new(0)), } } } @@ -59,10 +60,10 @@ impl CellLocks { #[inline] fn wait(spins: &mut u32) { if *spins < 64 { - std::hint::spin_loop(); + core::hint::spin_loop(); *spins += 1; } else { - std::thread::yield_now(); + crate::util::yield_now(); } } @@ -497,7 +498,7 @@ impl Data { // Use ptr::copy for overlapping memory regions (safe for shifting left) // When moving left (dst_offset < src_offset), this works correctly unsafe { - std::ptr::copy( + core::ptr::copy( inner_data.as_ptr().add(src_offset), inner_data.as_mut_ptr().add(dst_offset), length, @@ -567,10 +568,12 @@ impl Data { .map_err(|_| ExecutionError::LiveCellCountUnderflow) } + #[cfg(feature = "std")] pub(crate) fn has_live_cells(&self) -> bool { self.live_cells.load(Ordering::Acquire) != 0 } + #[cfg(feature = "std")] pub(crate) fn live_cell_count(&self) -> u32 { self.live_cells.load(Ordering::Acquire) } @@ -605,8 +608,9 @@ pub enum ExecutionError { #[cfg(test)] mod tests { - use std::sync::atomic::Ordering; - use std::sync::{Arc, mpsc}; + use alloc::sync::Arc; + use core::sync::atomic::Ordering; + use std::sync::mpsc; use std::thread; use rkyv::{Archive, Deserialize, Serialize}; diff --git a/src/in_memory/empty_link_registry.rs b/src/in_memory/empty_link_registry.rs index 825c7ccc..1bbab632 100644 --- a/src/in_memory/empty_link_registry.rs +++ b/src/in_memory/empty_link_registry.rs @@ -1,5 +1,6 @@ -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use data_bucket::Link; use data_bucket::page::PageId; @@ -65,13 +66,13 @@ impl IndexOrdLink { } impl PartialOrd for IndexOrdLink { - fn partial_cmp(&self, other: &Self) -> Option { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for IndexOrdLink { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { self.absolute_index().cmp(&other.absolute_index()) } } @@ -138,7 +139,7 @@ pub struct EmptyLinkRegistry { /// reclamation, and the most valuable. A scattered single-row delete says /// nothing about where to look, so [`Self::push`] does not record /// anything and only [`Self::push_many`] does. - targeted_pages: FairMutex>, + targeted_pages: FairMutex>, } /// A [`Link`] popped from the registry, together with the read guard that @@ -418,11 +419,11 @@ impl EmptyLinkRegistry { /// ranged delete emptied part of, and it is where a sweep should look /// first. fn note_coalesced_pages(&self, links: &[Link], runs: &[IndexOrdLink]) { - let mut links_per_page: std::collections::BTreeMap = Default::default(); + let mut links_per_page: alloc::collections::BTreeMap = Default::default(); for link in links { *links_per_page.entry(link.page_id).or_default() += 1; } - let mut runs_per_page: std::collections::BTreeMap = Default::default(); + let mut runs_per_page: alloc::collections::BTreeMap = Default::default(); for run in runs { *runs_per_page.entry(run.0.page_id).or_default() += 1; } @@ -443,8 +444,8 @@ impl EmptyLinkRegistry { /// Draining rather than reading: a sweep that has taken them is /// responsible for them, and leaving them would make every later sweep /// re-prioritise pages that are already compact. - pub fn take_targeted_pages(&self) -> std::collections::BTreeSet { - std::mem::take(&mut *self.targeted_pages.lock()) + pub fn take_targeted_pages(&self) -> alloc::collections::BTreeSet { + core::mem::take(&mut *self.targeted_pages.lock()) } /// Wakes a parked vacuum when freeing crossed the configured threshold. diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index f118dff5..851f6770 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -1,6 +1,14 @@ +use alloc::collections::VecDeque; +use alloc::sync::Arc; +use alloc::{boxed::Box, vec::Vec}; use arc_swap::ArcSwap; +use core::fmt::Debug; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; +use core::sync::atomic::{AtomicU64, Ordering}; use data_bucket::page::PageId; use derive_more::{Display, Error, From}; +use hashbrown::HashSet; use parking_lot::Mutex; use parking_lot::RwLock; #[cfg(feature = "perf_measurements")] @@ -12,14 +20,6 @@ use rkyv::{ ser::{Serializer, allocator::ArenaHandle, sharing::Share}, util::AlignedVec, }; -use std::collections::{HashSet, VecDeque}; -use std::marker::PhantomData; -use std::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; -use std::{ - fmt::Debug, - sync::Arc, - sync::atomic::{AtomicU64, Ordering}, -}; use crate::in_memory::empty_link_registry::EmptyLinkRegistry; use crate::prelude::ArchivedRowWrapper; @@ -37,7 +37,18 @@ fn page_id_mapper(page_id: usize) -> usize { } const PAGE_DIRECTORY_CHUNK_SIZE: usize = 64; -const PAGE_DIRECTORY_ROOTS: usize = 64; +/// Roots in the page directory, so its reach is `ROOTS * CHUNK_SIZE` pages. +/// +/// **This was 64, which reached 4,096 pages: 64 MiB at the default page size.** +/// Past that, `publish` returns early and every page access falls back to an +/// `ArcSwap` snapshot of the owning list. A table of 4 KiB rows, which fit +/// three to a page, crosses it after twelve thousand rows. +/// +/// Raising it did not measurably change insert cost on that fixture - the +/// copy-on-write page list dominated, and still does at these sizes - so this +/// is a ceiling being moved rather than a cost being removed. 1,024 roots +/// reach 65,536 pages, or 1 GiB, for an 8 KiB array of pointers. +const PAGE_DIRECTORY_ROOTS: usize = 1024; const GHOSTED: u8 = 1 << 0; const DELETED: u8 = 1 << 1; const VACUUMED: u8 = 1 << 2; @@ -67,6 +78,103 @@ const RECLAIM_BATCH_LIMIT: usize = 256; /// gain. const RECLAIM_BACKLOG_TRIGGER: usize = RECLAIM_BATCH_LIMIT; +/// Pages per chunk of [`PageList`]. +/// +/// The list is copy-on-write, so an append copies whatever the writer has to +/// replace. Chunking bounds that at one chunk plus the (much shorter) spine, +/// instead of the whole list. +const PAGE_LIST_CHUNK: usize = 256; + +/// Owns every page, and appends one without copying the ones already there. +/// +/// **This was a `Vec` behind an `ArcSwap`, and appending cloned all of it.** +/// Every existing page is an `Arc`, so the clone was one atomic increment per +/// page, each touching a separately allocated page header: a cache miss apiece. +/// Appending page N cost O(N) and filling a table cost O(N^2). It only showed +/// up with large rows, because those are what make pages plentiful - at 4 KiB a +/// row, three rows to a page, per-row insert cost grew tenfold over twenty +/// thousand rows while a 256-byte row stayed flat. +/// +/// Readers still take an `ArcSwap` snapshot and never block. +#[derive(Debug)] +struct PageList { + chunks: ArcSwap>>>>, +} + +impl PageList { + fn from_pages(pages: Vec>) -> Self { + let chunks = pages + .chunks(PAGE_LIST_CHUNK) + .map(|chunk| Arc::new(chunk.to_vec())) + .collect::>(); + Self { + chunks: ArcSwap::from_pointee(chunks), + } + } + + /// Append a page. Copies the last chunk, or starts a new one, plus the + /// spine of chunk pointers. + fn push(&self, page: Arc) { + let chunks = self.chunks.load_full(); + let mut next = (*chunks).clone(); + match next.last() { + // Every chunk but the last is full, so only the last can take one. + Some(last) if last.len() < PAGE_LIST_CHUNK => { + let mut grown = (**last).clone(); + grown.push(page); + *next.last_mut().expect("the branch matched on it") = Arc::new(grown); + } + _ => { + let mut chunk = Vec::with_capacity(PAGE_LIST_CHUNK); + chunk.push(page); + next.push(Arc::new(chunk)); + } + } + self.chunks.store(Arc::new(next)); + } + + fn len(&self) -> usize { + let chunks = self.chunks.load(); + match chunks.last() { + None => 0, + // Full but for the last, so its length is the only remainder. + Some(last) => (chunks.len() - 1) * PAGE_LIST_CHUNK + last.len(), + } + } + + fn get(&self, index: usize) -> Option> { + let chunks = self.chunks.load(); + chunks + .get(index / PAGE_LIST_CHUNK)? + .get(index % PAGE_LIST_CHUNK) + .cloned() + } + + /// Run `visit` against the page at `index`, borrowing it rather than + /// handing back an owned `Arc`. + /// + /// **`get` costs an atomic increment and the matching decrement on drop.** + /// A read that only needs the page for the length of one call pays both for + /// nothing, and it is measurable: routing the link-based read through `get` + /// moved a delete from 665 to 751 ns, while a select by primary key - which + /// goes through the page directory and never touches this - did not move at + /// all. + fn with_page(&self, index: usize, visit: impl FnOnce(&T) -> R) -> Option { + let chunks = self.chunks.load(); + let page = chunks.get(index / PAGE_LIST_CHUNK)?.get(index % PAGE_LIST_CHUNK)?; + Some(visit(page)) + } + + fn for_each(&self, mut visit: impl FnMut(&Arc)) { + let chunks = self.chunks.load(); + for chunk in chunks.iter() { + for page in chunk.iter() { + visit(page); + } + } + } +} + #[derive(Debug)] struct PageDirectoryChunk { pages: [AtomicPtr; PAGE_DIRECTORY_CHUNK_SIZE], @@ -75,7 +183,7 @@ struct PageDirectoryChunk { impl PageDirectoryChunk { fn new() -> Self { Self { - pages: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + pages: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), } } } @@ -95,7 +203,7 @@ struct PageDirectory { impl PageDirectory { fn new(pages: &[Arc]) -> Self { let directory = Self { - roots: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + roots: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), chunks: Mutex::new(Vec::new()), }; for (index, page) in pages.iter().enumerate() { @@ -115,7 +223,7 @@ impl PageDirectory { chunk = root.load(Ordering::Acquire); if chunk.is_null() { chunks.push(Box::new(PageDirectoryChunk::new())); - chunk = std::ptr::from_ref::>( + chunk = core::ptr::from_ref::>( chunks.last().expect("the chunk was just appended").as_ref(), ) .cast_mut(); @@ -261,7 +369,7 @@ where /// Immutable page-directory snapshots. Reads load one snapshot without a /// shared read-modify-write; rare growth copies and swaps the short vector. - pages: ArcSwap::WrappedRow, DATA_LENGTH>>>>, + pages: PageList::WrappedRow, DATA_LENGTH>>, /// Stable pointers for point access without ArcSwap's shared snapshot /// accounting. The corresponding `Arc`s remain owned by `pages`. page_directory: PageDirectory::WrappedRow, DATA_LENGTH>>, @@ -303,11 +411,11 @@ where return Ok(page); } - let page = { - let pages = self.pages.load(); - pages.get(index).map(Arc::as_ptr) - } - .ok_or(ExecutionError::PageNotFound(page_id))?; + let page = self + .pages + .get(index) + .map(|page| Arc::as_ptr(&page)) + .ok_or(ExecutionError::PageNotFound(page_id))?; // SAFETY: as above, the current directory retains this allocation and // all future directory snapshots clone its Arc. @@ -358,7 +466,7 @@ where /// thread can later collect it; it executes only after every reader /// pinned right now has unpinned. fn retire(&self, item: Retired) { - self.retire_many(std::iter::once(item)); + self.retire_many(core::iter::once(item)); } /// Queue several retired items behind one grace marker. @@ -578,7 +686,7 @@ where queued_page_retirements: AtomicUsize::new(0), // We are starting ID's from `1` because `0`'s page in file is info page. page_directory: PageDirectory::new(&pages), - pages: ArcSwap::from_pointee(pages), + pages: PageList::from_pages(pages), pages_write: Mutex::new(()), empty_links: EmptyLinkRegistry::::default(), empty_pages: Default::default(), @@ -602,7 +710,7 @@ where pending_retirements: AtomicUsize::new(0), queued_page_retirements: AtomicUsize::new(0), page_directory, - pages: ArcSwap::from_pointee(vec), + pages: PageList::from_pages(vec), pages_write: Mutex::new(()), empty_links: EmptyLinkRegistry::default(), empty_pages: Default::default(), @@ -743,18 +851,8 @@ where let _write = self.pages_write.lock(); if tried_page == page_id_mapper(self.current_page_id.load(Ordering::Acquire) as usize) { let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; - let pages = self.pages.load_full(); - let mut next = (*pages).clone(); let page = Arc::new(Data::new(index.into())); - next.push(page.clone()); - debug_assert_eq!(next.len(), pages.len() + 1); - debug_assert!( - next[..pages.len()] - .iter() - .zip(pages.iter()) - .all(|(new, old)| Arc::ptr_eq(new, old)) - ); - self.pages.store(Arc::new(next)); + self.pages.push(page.clone()); self.publish_page(&page); self.current_page_id.store(index, Ordering::Release); } @@ -771,9 +869,11 @@ where }; if let Some(page_id) = page_id { - let pages = self.pages.load(); let index = page_id_mapper(page_id.into()); - let page = pages[index].clone(); + let page = self + .pages + .get(index) + .expect("an empty page id names a page that was allocated"); { let _page_guard = page.access.write(); page.reset(); @@ -785,17 +885,7 @@ where let _write = self.pages_write.lock(); let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; let page = Arc::new(Data::new(index.into())); - let pages = self.pages.load_full(); - let mut next = (*pages).clone(); - next.push(page.clone()); - debug_assert_eq!(next.len(), pages.len() + 1); - debug_assert!( - next[..pages.len()] - .iter() - .zip(pages.iter()) - .all(|(new, old)| Arc::ptr_eq(new, old)) - ); - self.pages.store(Arc::new(next)); + self.pages.push(page.clone()); self.publish_page(&page); page @@ -842,22 +932,24 @@ where + Deserialize<::WrappedRow, HighDeserializer> + for<'a> rkyv::bytecheck::CheckBytes>, { - let pages = self.pages.load(); let page_id: usize = link.page_id.into(); let page_index = page_id .checked_sub(1) .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let page = pages - .get(page_index) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let wrapped = page.get_row_checked(link).map_err(ExecutionError::DataPageError)?; - if wrapped.is_ghosted() { - return Err(ExecutionError::Ghosted); - } - if wrapped.is_deleted() { - return Err(ExecutionError::Deleted); - } - Ok(wrapped.get_inner()) + // Borrowed rather than cloned: this is a read path and an `Arc` bump + // here showed up as a 13% slower delete. + self.pages + .with_page(page_index, |page| { + let wrapped = page.get_row_checked(link).map_err(ExecutionError::DataPageError)?; + if wrapped.is_ghosted() { + return Err(ExecutionError::Ghosted); + } + if wrapped.is_deleted() { + return Err(ExecutionError::Deleted); + } + Ok(wrapped.get_inner()) + }) + .ok_or(ExecutionError::PageNotFound(link.page_id))? } pub fn select_non_vacuumed(&self, link: Link) -> Result @@ -1119,9 +1211,7 @@ where } pub fn get_page(&self, page_id: PageId) -> Option::WrappedRow, DATA_LENGTH>>> { - let pages = self.pages.load(); - let page = pages.get(page_id_mapper(page_id.into()))?; - Some(page.clone()) + self.pages.get(page_id_mapper(page_id.into())) } /// Registers an already-indexed cell while rebuilding runtime metadata @@ -1138,16 +1228,19 @@ where Ok(()) } + #[cfg(feature = "std")] pub(crate) fn page_has_cells(&self, page_id: PageId) -> Result { let page = self.page_ref(page_id)?; Ok(page.has_live_cells()) } + #[cfg(feature = "std")] pub(crate) fn page_live_cell_count(&self, page_id: PageId) -> Result { let page = self.page_ref(page_id)?; Ok(page.live_cell_count()) } + #[cfg(feature = "std")] pub(crate) fn set_loaded_row_count(&self, count: usize) -> Result<(), ExecutionError> { let count = u64::try_from(count).map_err(|_| ExecutionError::RowCountOverflow)?; self.row_count.store(count, Ordering::Release); @@ -1156,6 +1249,7 @@ where /// Completes the vacuum's source-side accounting after every index has /// been swung to the destination link. + #[cfg(feature = "std")] pub(crate) fn remove_moved_cell(&self, link: Link) -> Result<(), ExecutionError> { self.remove_cell(link) } @@ -1170,11 +1264,10 @@ where /// Approximate under concurrency: a failing `save_row`'s transient /// reservation may be counted before its rollback. Metrics only. pub fn used_bytes(&self) -> u64 { - let pages = self.pages.load(); - pages - .iter() - .map(|p| u64::from(p.free_offset.load(Ordering::Relaxed))) - .sum() + let mut total = 0u64; + self.pages + .for_each(|page| total += u64::from(page.free_offset.load(Ordering::Relaxed))); + total } /// Copies a row to another page without exposing either mutable byte @@ -1188,6 +1281,7 @@ where /// concurrent low-level mutation may access either physical row while the /// move is in progress. After success, the caller must swing every index /// reference to the returned link before retiring `from_link`. + #[cfg(feature = "std")] pub(crate) unsafe fn move_row_for_vacuum( &self, from_link: Link, @@ -1230,7 +1324,7 @@ where } pub fn get_page_count(&self) -> usize { - self.pages.load().len() + self.pages.len() } pub fn get_empty_links(&self) -> Vec { @@ -1253,12 +1347,12 @@ where /// figure without it cannot be checked, because a sweep that never runs /// looks exactly like a sweep that is free. pub fn allocated_pages(&self) -> usize { - self.pages.load().len() + self.pages.len() } /// Heap bytes reserved by the fixed-size data-page allocations. pub fn allocated_bytes(&self) -> usize { - self.pages.load().len() * std::mem::size_of::::WrappedRow, DATA_LENGTH>>() + self.pages.len() * core::mem::size_of::::WrappedRow, DATA_LENGTH>>() } /// Pages allocated but currently on the empty list, so reusable without @@ -1302,6 +1396,7 @@ where /// current page serves as the sweep's first destination. Concurrent /// inserts are safe: the insert path rechecks `current_page_id` under the /// page barrier before writing and retries if the target changed. + #[cfg(feature = "std")] pub(crate) fn rotate_current_for_vacuum(&self, page_id: PageId) { debug_assert!( self.get_page(page_id).is_some(), @@ -1348,12 +1443,12 @@ impl ExecutionError { #[cfg(test)] mod tests { - use std::collections::HashSet; - use std::sync::Arc; - use std::sync::atomic::Ordering; + use alloc::sync::Arc; + use core::sync::atomic::Ordering; + use core::time::Duration; + use hashbrown::HashSet; use std::sync::mpsc; use std::thread; - use std::time::Duration; use std::time::Instant; use parking_lot::RwLock; @@ -1627,7 +1722,7 @@ mod tests { impl Drop for RemoteReader { fn drop(&mut self) { let (disconnected, _rx) = mpsc::channel(); - let _ = std::mem::replace(&mut self.commands, disconnected); + let _ = core::mem::replace(&mut self.commands, disconnected); if let Some(thread) = self.thread.take() { thread.join().unwrap(); } diff --git a/src/in_memory/row.rs b/src/in_memory/row.rs index 5b38803b..38066fff 100644 --- a/src/in_memory/row.rs +++ b/src/in_memory/row.rs @@ -1,5 +1,5 @@ +use core::fmt::Debug; use rkyv::Archive; -use std::fmt::Debug; pub trait PublicationSafe: Send + Sync + 'static {} diff --git a/src/index/arctic.rs b/src/index/arctic.rs index 42336285..9ea8dcb3 100644 --- a/src/index/arctic.rs +++ b/src/index/arctic.rs @@ -1,10 +1,11 @@ //! Arctic adapter for memory-only unique WorkTable indexes. -use std::borrow::Borrow; -use std::fmt::{self, Debug}; -use std::marker::PhantomData; -use std::ops::{Bound, RangeBounds}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use alloc::{string::String, vec::Vec}; +use core::borrow::Borrow; +use core::fmt::{self, Debug}; +use core::marker::PhantomData; +use core::ops::{Bound, RangeBounds}; +use core::sync::atomic::{AtomicUsize, Ordering}; use arctic::{ConcurrentMap, Key, Order}; @@ -327,6 +328,7 @@ where self.inner.allocated_node_bytes() } + #[cfg(feature = "std")] pub(crate) fn export_topology( &mut self, mut encode: impl FnMut(&V) -> T, @@ -337,6 +339,7 @@ where self.inner.export_topology(|value| encode(&V::from_arctic(*value))) } + #[cfg(feature = "std")] pub(crate) fn from_topology( topology: arctic::topology::Topology, mut decode: impl FnMut(T) -> V, @@ -481,8 +484,9 @@ where #[cfg(test)] mod tests { - use std::ops::Bound; - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use core::ops::Bound; + use std::sync::Barrier; use super::{ArcticIndex, UniqueIndex}; diff --git a/src/index/arctic_multi.rs b/src/index/arctic_multi.rs index 744eba81..f9fc346c 100644 --- a/src/index/arctic_multi.rs +++ b/src/index/arctic_multi.rs @@ -45,10 +45,11 @@ //! the dead entry and retries with a fresh slot, and the SMR guard it holds //! keeps the memory valid throughout. -use std::borrow::Borrow; -use std::fmt::{self, Debug}; -use std::ops::{Bound, ControlFlow, RangeBounds}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use alloc::{boxed::Box, vec::Vec}; +use core::borrow::Borrow; +use core::fmt::{self, Debug}; +use core::ops::{Bound, ControlFlow, RangeBounds}; +use core::sync::atomic::{AtomicUsize, Ordering}; use arctic::{ConcurrentMap, Key as ArcticNativeKey, Order}; use parking_lot::RwLock; @@ -190,7 +191,7 @@ where /// Returns every `(key, value)` pair stored under `key`, in insertion /// order, as a stable snapshot. An unknown key yields an empty iterator. - pub fn get(&self, key: &K) -> std::vec::IntoIter<(K, V)> { + pub fn get(&self, key: &K) -> alloc::vec::IntoIter<(K, V)> { let raw = key.to_arctic(); let Some(slot) = self.inner.get(raw.borrow()) else { return Vec::new().into_iter(); @@ -222,7 +223,7 @@ where let mut slots = 0; while let Some((_, slot)) = entries.lend() { let links = slot.read(); - slots += std::mem::size_of::>>() + links.links.capacity() * std::mem::size_of::(); + slots += core::mem::size_of::>>() + links.links.capacity() * core::mem::size_of::(); } self.inner.allocated_node_bytes() + slots } @@ -287,8 +288,9 @@ where #[cfg(test)] mod tests { - use std::ops::Bound; - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use core::ops::Bound; + use std::sync::Barrier; use super::ArcticMultiIndex; diff --git a/src/index/available_index.rs b/src/index/available_index.rs index 32b18991..a9b3e4ac 100644 --- a/src/index/available_index.rs +++ b/src/index/available_index.rs @@ -1,3 +1,4 @@ +use alloc::{string::String, string::ToString}; pub trait AvailableIndex { fn to_string_value(&self) -> String; } diff --git a/src/index/congee.rs b/src/index/congee.rs index e564c754..42ba8a6d 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -1,9 +1,10 @@ //! Congee adapter for memory-only unique WorkTable indexes. -use std::fmt::{self, Debug}; -use std::ops::{Bound, RangeBounds}; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::{self, Debug}; +use core::ops::{Bound, RangeBounds}; +use core::sync::atomic::{AtomicUsize, Ordering}; use congee::{CongeeRaw, DefaultAllocator}; use parking_lot::Mutex; @@ -53,7 +54,7 @@ pub struct CongeeIndex { // serialize mutations until the backend offers the required visibility. mutation: Mutex<()>, len: AtomicUsize, - marker: std::marker::PhantomData<(K, V)>, + marker: core::marker::PhantomData<(K, V)>, } impl Debug for CongeeIndex { @@ -79,7 +80,7 @@ where inner: CongeeRaw::new_with_drainer(DefaultAllocator {}, drainer), mutation: Mutex::new(()), len: AtomicUsize::new(0), - marker: std::marker::PhantomData, + marker: core::marker::PhantomData, } } } @@ -113,7 +114,7 @@ where // SAFETY: callers guarantee that `pointer` was produced by // `Arc::into_raw(...).expose_provenance()` for the same `V` and still // owns one strong reference. - unsafe { Arc::from_raw(std::ptr::with_exposed_provenance(pointer)) } + unsafe { Arc::from_raw(core::ptr::with_exposed_provenance(pointer)) } } #[inline] @@ -180,12 +181,13 @@ where .map(|(key, pointer)| { // SAFETY: the pinned epoch keeps every returned tree-owned // pointer alive until its value has been cloned. - let value = unsafe { &*std::ptr::with_exposed_provenance::(pointer) }; + let value = unsafe { &*core::ptr::with_exposed_provenance::(pointer) }; (K::from_congee(key), value.clone()) }) .collect() } + #[cfg(feature = "std")] pub(crate) fn export_topology( &mut self, mut encode: impl FnMut(&V) -> T, @@ -193,10 +195,11 @@ where self.inner.export_topology(|pointer| { // SAFETY: every raw payload is a live tree-owned `Arc` pointer, // and the exclusive borrow prevents removal while it is cloned. - unsafe { encode(&*std::ptr::with_exposed_provenance::(pointer)) } + unsafe { encode(&*core::ptr::with_exposed_provenance::(pointer)) } }) } + #[cfg(feature = "std")] pub(crate) fn from_topology( topology: congee::topology::Topology, mut decode: impl FnMut(T) -> V, @@ -217,7 +220,7 @@ where inner, mutation: Mutex::new(()), len: AtomicUsize::new(len), - marker: std::marker::PhantomData, + marker: core::marker::PhantomData, }) } } @@ -238,7 +241,7 @@ where let pointer = self.inner.get(&key.into_congee(), &guard)?; // SAFETY: the epoch guard keeps the tree-owned `Arc` alive for the // duration of `read`, and the pointer originated from `Arc::into_raw`. - let value = unsafe { &*std::ptr::with_exposed_provenance::(pointer) }; + let value = unsafe { &*core::ptr::with_exposed_provenance::(pointer) }; Some(read(value)) } @@ -334,8 +337,9 @@ where #[cfg(test)] mod tests { - use std::ops::Bound; - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use core::ops::Bound; + use std::sync::Barrier; use super::{CongeeIndex, UniqueIndex}; diff --git a/src/index/mod.rs b/src/index/mod.rs index c36970cd..22456318 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -23,13 +23,15 @@ pub use persistent_art::{ }; pub use persistent_wti::PersistentWtiIndex; pub use primary_index::PrimaryIndex; -pub use table_index::{ - TableIndex, TableIndexCdc, convert_change_events, convert_multi_change_events, convert_upstream_change_events, -}; +#[cfg(feature = "vanilla-index")] +pub use table_index::convert_upstream_change_events; +pub use table_index::{TableIndex, TableIndexCdc, convert_change_events, convert_multi_change_events}; pub use table_secondary_index::{ IndexError, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, }; -pub use unique::{UniqueIndex, UpstreamIndexMap, UpstreamIndexPair}; +pub use unique::UniqueIndex; +#[cfg(feature = "vanilla-index")] +pub use unique::{UpstreamIndexMap, UpstreamIndexPair}; pub use unsized_node::UnsizedNode; #[derive(Clone, Debug)] diff --git a/src/index/persistent_art.rs b/src/index/persistent_art.rs index eda5a690..2f73d88b 100644 --- a/src/index/persistent_art.rs +++ b/src/index/persistent_art.rs @@ -6,12 +6,13 @@ //! different stripes remain concurrent because their Set/Remove records //! commute during recovery. -use std::array; -use std::collections::hash_map::DefaultHasher; -use std::fmt::{self, Debug}; -use std::hash::{Hash, Hasher}; -use std::ops::RangeBounds; -use std::sync::atomic::{AtomicU64, Ordering}; +use alloc::vec::Vec; +use core::array; +use core::fmt::{self, Debug}; +use core::hash::{Hash, Hasher}; +use core::ops::RangeBounds; +use core::sync::atomic::{AtomicU64, Ordering}; +use rustc_hash::FxHasher as DefaultHasher; use data_bucket::Link; use indexset::cdc::change::{ChangeEvent, Id}; @@ -49,7 +50,7 @@ where K: ArcticKey, V: Clone + Debug + PartialEq + Send + Sync + 'static, { - pub fn get(&self, key: &K) -> std::vec::IntoIter<(K, V)> { + pub fn get(&self, key: &K) -> alloc::vec::IntoIter<(K, V)> { self.inner.get(key) } @@ -128,7 +129,7 @@ impl PersistentArtIndex { } fn mutation_stripe(&self, key: &K) -> &Mutex<()> { - let mut hasher = DefaultHasher::new(); + let mut hasher = DefaultHasher::default(); key.hash(&mut hasher); &self.mutation_stripes[hasher.finish() as usize % MUTATION_STRIPES] } @@ -318,7 +319,8 @@ where #[cfg(test)] mod tests { - use std::sync::{Arc, Barrier}; + use alloc::sync::Arc; + use std::sync::Barrier; use super::*; diff --git a/src/index/persistent_wti.rs b/src/index/persistent_wti.rs index 8c05edde..b8e629d8 100644 --- a/src/index/persistent_wti.rs +++ b/src/index/persistent_wti.rs @@ -9,11 +9,12 @@ //! not claims about the live WTI node position or maximum; the shadow validates //! that marker and derives the real structural metadata itself. -use std::array; -use std::fmt::{self, Debug}; -use std::hash::{Hash, Hasher}; -use std::ops::RangeBounds; -use std::sync::atomic::{AtomicU64, Ordering}; +use alloc::vec::Vec; +use core::array; +use core::fmt::{self, Debug}; +use core::hash::{Hash, Hasher}; +use core::ops::RangeBounds; +use core::sync::atomic::{AtomicU64, Ordering}; use data_bucket::Link; use indexset::cdc::change::{ChangeEvent, Id}; @@ -285,7 +286,7 @@ where #[cfg(test)] mod tests { - use std::collections::HashSet; + use hashbrown::HashSet; use data_bucket::page::PageId; diff --git a/src/index/primary_index.rs b/src/index/primary_index.rs index d8883498..49969579 100644 --- a/src/index/primary_index.rs +++ b/src/index/primary_index.rs @@ -1,8 +1,9 @@ //! Primary-key to row-location index. -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; diff --git a/src/index/table_index/cdc.rs b/src/index/table_index/cdc.rs index 0cb4a588..e8658a06 100644 --- a/src/index/table_index/cdc.rs +++ b/src/index/table_index/cdc.rs @@ -1,18 +1,26 @@ -use std::fmt::Debug; -use std::hash::Hash; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::concurrent::map::BTreeMap as VanillaIndexMap; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; -use crate::index::table_index::util::{convert_change_events, convert_upstream_change_events}; +#[cfg(feature = "vanilla-index")] +use crate::UpstreamIndexMap; +use crate::index::table_index::util::convert_change_events; +#[cfg(feature = "vanilla-index")] +use crate::index::table_index::util::convert_upstream_change_events; use crate::util::OffsetEqLink; -use crate::{ArcticIndex, ArcticKey, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, UniqueIndex, UpstreamIndexMap}; +use crate::{ArcticIndex, ArcticKey, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, UniqueIndex}; pub trait TableIndexCdc { fn insert_cdc(&self, value: T, link: Link) -> (Option, Vec>>); @@ -74,6 +82,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl TableIndexCdc for UpstreamIndexMap, Node> where T: Debug + Eq + Hash + Clone + Send + Ord, diff --git a/src/index/table_index/mod.rs b/src/index/table_index/mod.rs index a405b5d3..8a40d6ed 100644 --- a/src/index/table_index/mod.rs +++ b/src/index/table_index/mod.rs @@ -1,24 +1,32 @@ -use std::fmt::Debug; -use std::hash::Hash; +use core::fmt::Debug; +use core::hash::Hash; use data_bucket::Link; use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; +#[cfg(feature = "vanilla-index")] +use crate::UpstreamIndexMap; use crate::util::OffsetEqLink; +#[allow(unused_imports)] +use crate::{}; use crate::{ ArcticIndex, ArcticKey, ArcticMultiIndex, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, PersistentArcticIndex, - PersistentArcticMultiIndex, PersistentCongeeIndex, PersistentWtiIndex, UniqueIndex, UpstreamIndexMap, + PersistentArcticMultiIndex, PersistentCongeeIndex, PersistentWtiIndex, UniqueIndex, }; mod cdc; pub mod util; pub use cdc::TableIndexCdc; -pub use util::{convert_change_events, convert_multi_change_events, convert_upstream_change_events}; +#[cfg(feature = "vanilla-index")] +pub use util::convert_upstream_change_events; +pub use util::{convert_change_events, convert_multi_change_events}; pub trait TableIndex { fn insert(&self, value: T, link: Link) -> Option; @@ -114,6 +122,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl TableIndex for UpstreamIndexMap where T: Debug + Eq + Hash + Clone + Send + Ord, diff --git a/src/index/table_index/util.rs b/src/index/table_index/util.rs index 4ffc57cb..e056db39 100644 --- a/src/index/table_index/util.rs +++ b/src/index/table_index/util.rs @@ -1,7 +1,10 @@ +use alloc::vec::Vec; use indexset::cdc::change::ChangeEvent; use indexset::core::multipair::MultiPair; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::cdc::change::ChangeEvent as VanillaChangeEvent; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; pub fn convert_change_event(ev: ChangeEvent>) -> ChangeEvent> @@ -142,6 +145,7 @@ where /// Normalizes upstream IndexSet CDC events into WorkTablesIndex's event type, /// which remains the stable persistence boundary used by DataBucket. +#[cfg(feature = "vanilla-index")] pub fn convert_upstream_change_events( evs: Vec>>, ) -> Vec>> @@ -193,6 +197,7 @@ where .collect() } +#[cfg(feature = "vanilla-index")] fn upstream_pair(pair: VanillaPair) -> Pair where L1: Into, diff --git a/src/index/table_secondary_index/cdc.rs b/src/index/table_secondary_index/cdc.rs index 804014d8..0b7914da 100644 --- a/src/index/table_secondary_index/cdc.rs +++ b/src/index/table_secondary_index/cdc.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use alloc::vec::Vec; +use hashbrown::HashMap; use data_bucket::Link; diff --git a/src/index/table_secondary_index/index_events.rs b/src/index/table_secondary_index/index_events.rs index 9183da9b..acdf0a9f 100644 --- a/src/index/table_secondary_index/index_events.rs +++ b/src/index/table_secondary_index/index_events.rs @@ -1,6 +1,6 @@ use crate::prelude::IndexChangeEventId; +use hashbrown::HashMap; use indexset::cdc::change; -use std::collections::HashMap; pub trait TableSecondaryIndexEventsOps { fn extend(&mut self, another: Self) diff --git a/src/index/table_secondary_index/info.rs b/src/index/table_secondary_index/info.rs index 569a4953..6da1dfdc 100644 --- a/src/index/table_secondary_index/info.rs +++ b/src/index/table_secondary_index/info.rs @@ -1,4 +1,5 @@ use crate::prelude::IndexInfo; +use alloc::vec::Vec; pub trait TableSecondaryIndexInfo { fn index_info(&self) -> Vec; diff --git a/src/index/table_secondary_index/mod.rs b/src/index/table_secondary_index/mod.rs index 69434a82..415e4c61 100644 --- a/src/index/table_secondary_index/mod.rs +++ b/src/index/table_secondary_index/mod.rs @@ -1,9 +1,10 @@ +use alloc::vec::Vec; mod cdc; mod index_events; mod info; use data_bucket::Link; -use std::collections::HashMap; +use hashbrown::HashMap; use crate::WorkTableError; use crate::{AvailableIndex, Difference}; diff --git a/src/index/unique.rs b/src/index/unique.rs index 55e559f3..a6f63724 100644 --- a/src/index/unique.rs +++ b/src/index/unique.rs @@ -4,15 +4,23 @@ //! backend's guard type. That keeps generated code independent from the //! concurrency and reclamation strategy used by each index implementation. -use std::fmt::Debug; -use std::hash::Hash; -use std::ops::RangeBounds; +// Only `UpstreamIndexMap`'s default node type and the tests name `Vec`, and the +// first of those is behind `vanilla-index`. Ungated, this warns on every +// `--no-default-features` build, which is the build that has to stay quiet. +#[cfg(any(feature = "vanilla-index", test))] +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::Hash; +use core::ops::RangeBounds; use crate::IndexMap; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::concurrent::map::BTreeMap as VanillaIndexMap; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; /// Point, mutation, and ordered-scan operations used by generated unique @@ -131,6 +139,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl UniqueIndex for VanillaIndexMap where K: Debug + Eq + Hash + Clone + Send + Ord + 'static, @@ -198,15 +207,16 @@ where self.range(range).map(|(_, value)| value.clone()) } } - /// Vanilla upstream IndexSet map, kept distinct from WorkTable's default /// WorkTablesIndex alias so both implementations may coexist in one binary. +#[cfg(feature = "vanilla-index")] pub type UpstreamIndexMap>> = VanillaIndexMap; +#[cfg(feature = "vanilla-index")] pub type UpstreamIndexPair = VanillaPair; #[cfg(test)] mod tests { - use std::sync::Arc; + use alloc::sync::Arc; use super::{UniqueIndex, UpstreamIndexMap}; use crate::{ArcticIndex, CongeeIndex, IndexMap}; @@ -260,7 +270,7 @@ mod tests { let iterated = index.iter_values().find(|(candidate, _)| *candidate == key); panic!( "backend={}, key={key}, point={value:?}, iterated={iterated:?}", - std::any::type_name::(), + core::any::type_name::(), ); } } @@ -292,7 +302,7 @@ mod tests { threads.push(std::thread::spawn(move || { for sequence in 0..1_000_u64 { let key = worker * 1_000 + sequence; - let backend = std::any::type_name::(); + let backend = core::any::type_name::(); assert_eq!( index.insert_value_checked(key, key + 1), Some(()), diff --git a/src/index/unsized_node.rs b/src/index/unsized_node.rs index eefd0c8c..3214cada 100644 --- a/src/index/unsized_node.rs +++ b/src/index/unsized_node.rs @@ -1,11 +1,12 @@ +use alloc::vec::Vec; use data_bucket::{SizeMeasurable, UnsizedIndexPageUtility, VariableSizeMeasurable}; use indexset::core::node::NodeLike; -use std::borrow::Borrow; -use std::collections::Bound; -use std::fmt::Debug; -use std::ops::Deref; -use std::slice::Iter; +use core::borrow::Borrow; +use core::fmt::Debug; +use core::ops::Bound; +use core::ops::Deref; +use core::slice::Iter; pub const UNSIZED_HEADER_LENGTH: u32 = 64; @@ -238,7 +239,7 @@ where fn replace(&mut self, idx: usize, value: T) -> Option { let value_size = value.aligned_size(); if let Some(old) = self.inner.get_mut(idx) { - let old = std::mem::replace(old, value); + let old = core::mem::replace(old, value); self.length += value_size; self.removed_length += old.aligned_size(); if idx + 1 == self.inner.len() { diff --git a/src/lib.rs b/src/lib.rs index 173b172f..972e1d62 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,20 @@ +#![cfg_attr(not(feature = "std"), no_std)] #![doc = include_str!("../docs/crate.md")] +#[macro_use] +extern crate alloc; + +/// Generated code names `worktable::` paths, which must also resolve inside +/// this crate, where `worktable!` is invoked for the persistence queue. +extern crate self as worktable; + +#[cfg(feature = "std")] +pub mod fsx; pub mod in_memory; mod index; pub mod lock; mod mem_stat; +#[cfg(feature = "std")] pub mod migration; pub mod partition; pub mod persistence; @@ -16,6 +27,7 @@ mod util; pub mod features; pub use index::*; +#[cfg(feature = "std")] pub use persistence::{ LoadMode, PersistedWorkTable, PersistenceConfig, PersistenceLoadError, UnloadFailure, UnloadReport, }; @@ -34,38 +46,56 @@ pub use worktable_dsl; pub use worktable_codegen::s3_sync_persistence; pub mod prelude { + /// The filesystem this crate goes through. Generated code opens files by + /// this path, so a consumer of `worktable!` gets the same backend the + /// crate itself uses without naming it. + #[cfg(feature = "std")] + pub use crate::fsx; + pub use alloc::collections::BTreeMap; + pub use alloc::sync::Arc; + pub use alloc::vec::IntoIter; + pub use hashbrown::{HashMap, HashSet}; + pub use crate::in_memory::{ArchivedRowWrapper, Data, DataPages, Query, RowWrapper, StorableRow}; pub use crate::lock::FullRowLock; pub use crate::lock::{Lock, RowLock}; pub use crate::lock::{LockAcquirer, LockGuard, LockMap, PendingLock}; pub use crate::mem_stat::MemStat; pub use crate::partition::{MAX_PARTITIONS, PartRef, PartitionError, PartitionSet}; + pub use crate::persistence::{AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, OperationId}; + #[cfg(feature = "std")] pub use crate::persistence::{ - AcknowledgeOperation, ArtPersistenceKey, DeleteOperation, DiskConfig, DiskPersistenceEngine, - IndexTableOfContents, InsertOperation, LoadMode, Operation, OperationId, PersistedWorkTable, PersistenceConfig, - PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, PersistenceMonitor, - PersistenceResult, PersistenceState, PersistenceTask, ReadOnlyPersistenceEngine, SpaceArcticIndex, - SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, - SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, SpaceLogicalMultiIndex, - SpaceLogicalMultiIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, UnloadFailure, UnloadReport, - UpdateOperation, load_persisted_state, map_index_pages_to_toc_and_general, - map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, validate_events, + ArtPersistenceKey, DiskConfig, DiskPersistenceEngine, IndexTableOfContents, LoadMode, PersistedWorkTable, + PersistenceConfig, PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, + PersistenceMonitor, PersistenceResult, PersistenceState, PersistenceTask, ReadOnlyPersistenceEngine, + SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, + SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, + SpaceLogicalMultiIndex, SpaceLogicalMultiIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, + UnloadFailure, UnloadReport, load_persisted_state, map_index_pages_to_toc_and_general, + map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, }; + pub use crate::persistence::{OperationType, UpdateOperation, validate_events}; pub use crate::primary_key::{ PrimaryKeyGenerator, PrimaryKeyGeneratorRange, PrimaryKeyGeneratorState, TablePrimaryKey, }; pub use crate::table::select::{Order, QueryParams, SelectQueryBuilder, SelectQueryExecutor}; pub use crate::table::system_info::{IndexInfo, IndexKind, SystemInfo}; pub use crate::util::{OffsetEqLink, OrderedF32Def, OrderedF64Def}; + #[allow(unused_imports)] + pub use crate::{}; pub use crate::{ ArcticEntry, ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticStringKey, AvailableIndex, BatchDeleteError, BatchInsertError, CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, PersistentArcticIndex, PersistentArcticMultiIndex, PersistentArtIndex, PersistentCongeeIndex, PersistentWtiIndex, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, - UpstreamIndexMap, UpstreamIndexPair, WorkTable, WorkTableError, vacuum::EmptyDataVacuum, - vacuum::VacuumPersistence, vacuum::WorkTableVacuum, validate_arctic_link, + WorkTable, WorkTableError, validate_arctic_link, }; + /// The upstream IndexSet backend, when the `vanilla-index` feature selects it. + #[cfg(feature = "vanilla-index")] + pub use crate::{UpstreamIndexMap, UpstreamIndexPair}; + #[cfg(feature = "std")] + pub use crate::{vacuum::EmptyDataVacuum, vacuum::VacuumPersistence, vacuum::WorkTableVacuum}; pub use data_bucket::{ DATA_VERSION, DataPage, GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, INNER_PAGE_SIZE, IndexPage, Interval, Link, PAGE_SIZE, PageType, Persistable, PersistableIndex, SizeMeasurable, SizeMeasure, SpaceInfoPage, diff --git a/src/lock/map.rs b/src/lock/map.rs index ea79742e..22961737 100644 --- a/src/lock/map.rs +++ b/src/lock/map.rs @@ -1,10 +1,11 @@ -use std::collections::HashMap; -use std::collections::hash_map::DefaultHasher; -use std::fmt::Debug; -use std::hash::{Hash, Hasher}; -use std::ops::Deref; -use std::sync::Arc; -use std::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::{Hash, Hasher}; +use core::ops::Deref; +use core::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering}; +use hashbrown::HashMap; +use rustc_hash::FxHasher as DefaultHasher; use parking_lot::RwLock; @@ -138,7 +139,7 @@ impl Default for LockMap { Self { map: RwLock::new(HashMap::new()), next_id: AtomicU16::default(), - mutation_stripes: Arc::new(std::array::from_fn(|_| MutationStripe::default())), + mutation_stripes: Arc::new(core::array::from_fn(|_| MutationStripe::default())), bulk_mutations: Arc::default(), } } @@ -283,7 +284,7 @@ where } fn stripe_of(key: &PrimaryKey) -> usize { - let mut hasher = DefaultHasher::new(); + let mut hasher = DefaultHasher::default(); key.hash(&mut hasher); (hasher.finish() as usize) % MUTATION_STRIPE_COUNT } @@ -343,9 +344,9 @@ where while gate.serving.load(Ordering::Acquire) != ticket { if spins < 16 { spins += 1; - std::hint::spin_loop(); + core::hint::spin_loop(); } else { - std::thread::yield_now(); + crate::util::yield_now(); } } diff --git a/src/lock/mod.rs b/src/lock/mod.rs index db352d32..6c465e18 100644 --- a/src/lock/mod.rs +++ b/src/lock/mod.rs @@ -1,15 +1,16 @@ +use alloc::vec::Vec; mod map; mod row_lock; -use std::cell::Cell; -use std::fmt::Debug; -use std::future::Future; -use std::hash::{Hash, Hasher}; -use std::marker::PhantomData; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::task::{Context, Poll}; +use alloc::sync::Arc; +use core::cell::Cell; +use core::fmt::Debug; +use core::future::Future; +use core::hash::{Hash, Hasher}; +use core::marker::PhantomData; +use core::pin::Pin; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::{Context, Poll}; use futures::task::AtomicWaker; use parking_lot::Mutex; @@ -261,7 +262,7 @@ impl Future for LockWait { // Spin phase: try up to MAX_SPINS before going async for _ in 0..MAX_SPINS { - std::hint::spin_loop(); + core::hint::spin_loop(); if !self.locked.load(Ordering::Acquire) { return Poll::Ready(()); } diff --git a/src/lock/row_lock.rs b/src/lock/row_lock.rs index 9e8aff49..e9022790 100644 --- a/src/lock/row_lock.rs +++ b/src/lock/row_lock.rs @@ -1,7 +1,7 @@ -use std::collections::HashSet; -use std::fmt::Debug; -use std::hash::Hash; -use std::sync::Arc; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::hash::Hash; +use hashbrown::HashSet; use crate::lock::{Lock, LockGuard, LockMap, LockWait}; diff --git a/src/mem_stat/mod.rs b/src/mem_stat/mod.rs index 8b45b3db..345d9239 100644 --- a/src/mem_stat/mod.rs +++ b/src/mem_stat/mod.rs @@ -1,9 +1,10 @@ +use alloc::{boxed::Box, string::String, vec::Vec}; mod primitives; -use std::collections::HashMap; -use std::fmt::Debug; -use std::rc::Rc; -use std::sync::Arc; +use alloc::rc::Rc; +use alloc::sync::Arc; +use core::fmt::Debug; +use hashbrown::HashMap; use data_bucket::Link; use data_bucket::page::PageId; @@ -14,16 +15,22 @@ use ordered_float::OrderedFloat; use psc_nanoid::PackedNanoid; use psc_nanoid::packed::AlphabetPackExt; use uuid::Uuid; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +#[cfg(feature = "vanilla-index")] use vanilla_indexset::core::pair::Pair as VanillaPair; +#[cfg(feature = "vanilla-index")] +use crate::UpstreamIndexMap; use crate::in_memory::{RowWrapper, StorableRow}; use crate::persistence::OperationType; use crate::prelude::OperationId; use crate::util::OffsetEqLink; +#[allow(unused_imports)] +use crate::{}; use crate::{ ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticValue, CongeeIndex, CongeeKey, IndexMultiMap, PersistentArtIndex, - PersistentWtiIndex, UniqueIndex, UpstreamIndexMap, WorkTable, + PersistentWtiIndex, UniqueIndex, WorkTable, }; use crate::{IndexMap, impl_memstat_zero}; @@ -55,7 +62,7 @@ impl< PkMap, > where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, PkMap: UniqueIndex> + MemStat, @@ -81,10 +88,10 @@ impl MemStat for Option { impl MemStat for Vec { fn heap_size(&self) -> usize { - self.capacity() * std::mem::size_of::() + self.iter().map(|v| v.heap_size()).sum::() + self.capacity() * core::mem::size_of::() + self.iter().map(|v| v.heap_size()).sum::() } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::() + self.iter().map(|v| v.used_size()).sum::() + self.len() * core::mem::size_of::() + self.iter().map(|v| v.used_size()).sum::() } } @@ -104,7 +111,7 @@ where Node: NodeLike> + Send + 'static, { fn heap_size(&self) -> usize { - let slot_size = std::mem::size_of::>(); + let slot_size = core::mem::size_of::>(); let base_heap = self.capacity() * slot_size; let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); @@ -113,7 +120,7 @@ where } fn used_size(&self) -> usize { - let pair_size = std::mem::size_of::>(); + let pair_size = core::mem::size_of::>(); let base = self.len() * pair_size; let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); @@ -122,6 +129,7 @@ where } } +#[cfg(feature = "vanilla-index")] impl MemStat for UpstreamIndexMap where K: Debug + Ord + Clone + 'static + MemStat + Send, @@ -129,14 +137,14 @@ where Node: VanillaNodeLike> + Send + 'static, { fn heap_size(&self) -> usize { - let slot_size = std::mem::size_of::>(); + let slot_size = core::mem::size_of::>(); let base_heap = self.capacity() * slot_size; let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); base_heap + kv_heap } fn used_size(&self) -> usize { - let pair_size = std::mem::size_of::>(); + let pair_size = core::mem::size_of::>(); let base = self.len() * pair_size; let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); base + used @@ -151,7 +159,7 @@ where fn heap_size(&self) -> usize { let values = self.iter_values().map(|(_, value)| value.heap_size()).sum::(); self.allocated_node_bytes() - + self.len() * (std::mem::size_of::() + 2 * std::mem::size_of::()) + + self.len() * (core::mem::size_of::() + 2 * core::mem::size_of::()) + values } @@ -170,7 +178,7 @@ where } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + self.len() * core::mem::size_of::<(K, V)>() } } @@ -184,7 +192,7 @@ where } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + self.len() * core::mem::size_of::<(K, V)>() } } @@ -220,7 +228,7 @@ where Node: NodeLike> + Send + 'static, { fn heap_size(&self) -> usize { - let slot_size = std::mem::size_of::>(); + let slot_size = core::mem::size_of::>(); let base_heap = self.capacity() * slot_size; let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); @@ -229,7 +237,7 @@ where } fn used_size(&self) -> usize { - let pair_size = std::mem::size_of::>(); + let pair_size = core::mem::size_of::>(); let base = self.len() * pair_size; let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); @@ -240,32 +248,32 @@ where impl MemStat for Box { fn heap_size(&self) -> usize { - std::mem::size_of::() + (**self).heap_size() + core::mem::size_of::() + (**self).heap_size() } fn used_size(&self) -> usize { - std::mem::size_of::() + (**self).used_size() + core::mem::size_of::() + (**self).used_size() } } impl MemStat for Arc { fn heap_size(&self) -> usize { - std::mem::size_of::() + (**self).heap_size() + core::mem::size_of::() + (**self).heap_size() } fn used_size(&self) -> usize { - std::mem::size_of::() + (**self).used_size() + core::mem::size_of::() + (**self).used_size() } } impl MemStat for Rc { fn heap_size(&self) -> usize { - std::mem::size_of::() + (**self).heap_size() + core::mem::size_of::() + (**self).heap_size() } fn used_size(&self) -> usize { - std::mem::size_of::() + (**self).used_size() + core::mem::size_of::() + (**self).used_size() } } -impl MemStat for HashMap { +impl MemStat for HashMap { fn heap_size(&self) -> usize { let bucket_size = size_of::<(K, V)>(); let base_heap = self.capacity() * bucket_size; diff --git a/src/mem_stat/primitives.rs b/src/mem_stat/primitives.rs index ee159f96..348b2484 100644 --- a/src/mem_stat/primitives.rs +++ b/src/mem_stat/primitives.rs @@ -29,23 +29,24 @@ impl_memstat_zero!( char, u128, i128, - std::num::NonZeroU8, - std::num::NonZeroU16, - std::num::NonZeroU32, - std::num::NonZeroU64, - std::num::NonZeroU128, - std::num::NonZeroUsize, - std::num::NonZeroI8, - std::num::NonZeroI16, - std::num::NonZeroI32, - std::num::NonZeroI64, - std::num::NonZeroI128, - std::num::NonZeroIsize, - std::time::Duration, - std::time::SystemTime, - std::time::Instant + core::num::NonZeroU8, + core::num::NonZeroU16, + core::num::NonZeroU32, + core::num::NonZeroU64, + core::num::NonZeroU128, + core::num::NonZeroUsize, + core::num::NonZeroI8, + core::num::NonZeroI16, + core::num::NonZeroI32, + core::num::NonZeroI64, + core::num::NonZeroI128, + core::num::NonZeroIsize, + core::time::Duration ); +#[cfg(feature = "std")] +impl_memstat_zero!(std::time::SystemTime, std::time::Instant); + impl_memstat_zero!( [u8], [i8], diff --git a/src/migration/mod.rs b/src/migration/mod.rs index 5074ed81..ca4e34dc 100644 --- a/src/migration/mod.rs +++ b/src/migration/mod.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use rkyv::api::high::HighDeserializer; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -5,7 +6,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize}; -use tokio::fs::File; use crate::prelude::{GeneralPage, Persistable, SpaceInfoPage, WT_DATA_EXTENSION, parse_page}; @@ -24,7 +24,7 @@ where SpaceInfoPage: Persistable, { let data_file_path = format!("{}/{}", table_path, WT_DATA_EXTENSION); - let mut file = File::open(&data_file_path).await?; - let info: GeneralPage> = parse_page::<_, 4096>(&mut file, 0).await?; + let mut file = crate::fsx::open(&data_file_path).await?; + let info: GeneralPage> = parse_page::<_, 4096, DEFAULT_PAGE_STRIDE>(&mut file, 0).await?; Ok(info.inner.version) } diff --git a/src/partition/mod.rs b/src/partition/mod.rs index fdc2779b..bef57a9a 100644 --- a/src/partition/mod.rs +++ b/src/partition/mod.rs @@ -55,22 +55,23 @@ //! instance measures 110 KB and 6.1 ms to construct, of which 95 percent is //! inside `PersistenceEngine::new`. +use alloc::{boxed::Box, vec::Vec}; // Under `--cfg wt_loom` the atomics and the mutex come from loom, which explores // every interleaving of them rather than whichever one this machine happened // to produce. `Arc` stays `std`: loom's has no `into_raw` or // `increment_strong_count`, and std's own atomics are already model-checked // upstream. What loom is being asked about here is the slot protocol and the // double-checked lock in `get_or_create`, not reference counting. +use alloc::collections::VecDeque; +use alloc::sync::Arc; +#[cfg(not(wt_loom))] +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; #[cfg(wt_loom)] use loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; #[cfg(wt_loom)] use loom::sync::{Mutex, MutexGuard}; #[cfg(not(wt_loom))] use parking_lot::{Mutex, MutexGuard}; -use std::collections::VecDeque; -use std::sync::Arc; -#[cfg(not(wt_loom))] -use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; use crate::mem_stat::MemStat; #[cfg(not(wt_loom))] @@ -109,7 +110,7 @@ struct Chunk { impl Chunk { fn empty() -> Box { Box::new(Chunk { - slots: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + slots: core::array::from_fn(|_| AtomicPtr::new(core::ptr::null_mut())), }) } } @@ -144,7 +145,7 @@ pub struct PartRef<'a, T> { value: &'a T, } -impl std::ops::Deref for PartRef<'_, T> { +impl core::ops::Deref for PartRef<'_, T> { type Target = T; fn deref(&self) -> &T { @@ -180,7 +181,7 @@ impl Default for PartitionSet { impl PartitionSet { pub fn new() -> Self { Self { - spine: (0..MAX_CHUNKS).map(|_| AtomicPtr::new(std::ptr::null_mut())).collect(), + spine: (0..MAX_CHUNKS).map(|_| AtomicPtr::new(core::ptr::null_mut())).collect(), live: AtomicUsize::new(0), grow: Mutex::new(VecDeque::new()), #[cfg(not(wt_loom))] @@ -293,7 +294,7 @@ impl PartitionSet { /// So a tick loop should not pin per lookup. Pin once, read many: /// /// ``` - /// # fn main() { futures::executor::block_on(async { + /// # fn main() { nagoya::block_on(async { /// use worktable::prelude::*; /// use worktable::worktable; /// @@ -488,7 +489,7 @@ impl PartitionSet { let table = { let mut retired = self.lock(); let chunk = self.chunk(idx)?; - let p = chunk.slots[idx % CHUNK].swap(std::ptr::null_mut(), Ordering::AcqRel); + let p = chunk.slots[idx % CHUNK].swap(core::ptr::null_mut(), Ordering::AcqRel); if p.is_null() { return None; } @@ -594,13 +595,13 @@ impl PartitionSet { } } -impl std::fmt::Debug for PartitionSet { +impl core::fmt::Debug for PartitionSet { /// Deliberately shallow: a partition set can hold thousands of tables and /// printing them would be useless as well as slow. - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("PartitionSet") .field("live", &self.len()) - .field("table", &std::any::type_name::()) + .field("table", &core::any::type_name::()) .finish() } } @@ -608,7 +609,7 @@ impl std::fmt::Debug for PartitionSet { impl Drop for PartitionSet { fn drop(&mut self) { for cell in &self.spine { - let p = cell.swap(std::ptr::null_mut(), Ordering::AcqRel); + let p = cell.swap(core::ptr::null_mut(), Ordering::AcqRel); if p.is_null() { continue; } @@ -616,7 +617,7 @@ impl Drop for PartitionSet { // published exactly once, and nothing else frees it. let chunk = unsafe { Box::from_raw(p) }; for slot in chunk.slots.iter() { - let sp = slot.swap(std::ptr::null_mut(), Ordering::AcqRel); + let sp = slot.swap(core::ptr::null_mut(), Ordering::AcqRel); if !sp.is_null() { // Safety: a live slot owns one strong reference. drop(unsafe { Arc::from_raw(sp as *const T) }); @@ -652,8 +653,8 @@ pub enum PartitionError { OutOfRange { key: u64 }, } -impl std::fmt::Display for PartitionError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for PartitionError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { PartitionError::OutOfRange { key } => { write!(f, "partition key {key} exceeds the maximum of {MAX_PARTITIONS}") @@ -662,7 +663,7 @@ impl std::fmt::Display for PartitionError { } } -impl std::error::Error for PartitionError {} +impl core::error::Error for PartitionError {} #[cfg(all(test, not(wt_loom)))] mod tests; diff --git a/src/partition/tests.rs b/src/partition/tests.rs index a18f9e68..1a920c2d 100644 --- a/src/partition/tests.rs +++ b/src/partition/tests.rs @@ -1,5 +1,6 @@ use super::*; -use std::sync::atomic::AtomicU32; +use alloc::{string::ToString, vec::Vec}; +use core::sync::atomic::AtomicU32; #[derive(Debug, PartialEq)] struct Counted(u64); @@ -119,7 +120,7 @@ fn concurrent_creation_of_one_key_makes_one_table() { #[test] fn concurrent_readers_see_a_partition_created_under_them() { let set: Arc> = Arc::new(PartitionSet::new()); - let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop = Arc::new(core::sync::atomic::AtomicBool::new(false)); let reader = { let set = set.clone(); let stop = stop.clone(); @@ -389,7 +390,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { let drops = Arc::new(AtomicU32::new(0)); let set: Arc> = Arc::new(PartitionSet::new()); - let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop = Arc::new(core::sync::atomic::AtomicBool::new(false)); let seen = Arc::new(AtomicU32::new(0)); let readers: Vec<_> = (0..if cfg!(miri) { 2 } else { 3 }) @@ -434,7 +435,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { for k in 0..KEYS { set.get_or_create(k, || make(k, &drops)).unwrap(); } - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let deadline = std::time::Instant::now() + core::time::Duration::from_secs(30); while seen.load(Ordering::Relaxed) == 0 && std::time::Instant::now() < deadline { std::thread::yield_now(); } @@ -447,7 +448,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { // read to finish, never a zero-reader instant. The pre-epoch retire list // could only assert the opposite here (everything retired, nothing // freed, unbounded growth through the shared router). - let reclaim_deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let reclaim_deadline = std::time::Instant::now() + core::time::Duration::from_secs(30); while drops.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < reclaim_deadline { set.collect(); std::thread::yield_now(); @@ -473,7 +474,7 @@ fn a_reader_racing_a_remove_never_touches_freed_memory() { // Drain the remainder through the shared handle: no `&mut`, no // `Arc::try_unwrap` gymnastics needed for reclamation any more. - let drain_deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let drain_deadline = std::time::Instant::now() + core::time::Duration::from_secs(30); while set.retired_len() > 0 && std::time::Instant::now() < drain_deadline { set.collect(); } @@ -669,7 +670,7 @@ fn mem_stat_reports_each_partition_and_their_sum() { // `size_of::()` on top of the payload, because that is what the // allocation actually holds. Derived rather than hard coded so the // expectation is the rule, not one machine's numbers. - let overhead = std::mem::size_of::(); + let overhead = core::mem::size_of::(); for (k, size) in [(3u64, 17usize), (1, 5), (2048, 300)] { set.get_or_create(k, || Sized_(size)).unwrap(); } @@ -698,6 +699,6 @@ fn partition_error_says_which_key_and_which_bound() { ); // The `Error` impl is what a caller using `?` and `eyre` will format. - let as_error: &dyn std::error::Error = &out_of_range; + let as_error: &dyn core::error::Error = &out_of_range; assert_eq!(as_error.to_string(), text); } diff --git a/src/persistence/engine.rs b/src/persistence/engine.rs index 37254f6f..bcf9487d 100644 --- a/src/persistence/engine.rs +++ b/src/persistence/engine.rs @@ -1,8 +1,9 @@ -use std::fmt::Debug; +use alloc::{string::String, vec::Vec}; +use core::fmt::Debug; +use core::future::Future; +use core::hash::Hash; +use core::marker::PhantomData; use std::fs; -use std::future::Future; -use std::hash::Hash; -use std::marker::PhantomData; use std::panic::{AssertUnwindSafe, resume_unwind}; use std::path::Path; diff --git a/src/persistence/error.rs b/src/persistence/error.rs index fbfac49e..1acb39a2 100644 --- a/src/persistence/error.rs +++ b/src/persistence/error.rs @@ -1,9 +1,10 @@ -use std::error::Error; -use std::fmt::{Display, Formatter}; -use std::future::Future; +use alloc::sync::Arc; +use alloc::{string::String, string::ToString}; +use core::error::Error; +use core::fmt::{Display, Formatter}; +use core::future::Future; use std::panic::AssertUnwindSafe; use std::path::{Path, PathBuf}; -use std::sync::Arc; use futures::FutureExt; @@ -39,7 +40,7 @@ impl PersistenceLoadError { } impl Display for PersistenceLoadError { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { write!( formatter, "torn or corrupt persisted table at {}: {}", @@ -111,7 +112,7 @@ impl PersistenceIndexCorruption { } impl Display for PersistenceIndexCorruption { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { write!( formatter, "persisted index at {} was quarantined: {}", @@ -137,7 +138,7 @@ pub enum PersistenceError { } impl Display for PersistenceError { - fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result { match self { Self::Closing => formatter.write_str("persistence task is closing"), Self::Closed => formatter.write_str("persistence task is closed"), diff --git a/src/persistence/event_ledger.rs b/src/persistence/event_ledger.rs new file mode 100644 index 00000000..00ef9f4d --- /dev/null +++ b/src/persistence/event_ledger.rs @@ -0,0 +1,774 @@ +//! Pairs index change event id *assignment* with event *queueing*, so that a +//! persistence stall on an event gap names its own cause instead of only its +//! symptom. +//! +//! # The defect this exists for +//! +//! `BatchOperation::validate` refuses to apply an event stream with a hole in +//! it (see the guard there and commit `c0c06ba`). A hole is normally +//! transient: the operation carrying the missing id has been produced but not +//! yet batched. A hole that survives the whole deferral budget means something +//! else, and the old message could not tell the two apart. It reported the +//! range and nothing more: +//! +//! ```text +//! persistence stalled on primary index event gap: last applied Id(1439), +//! next available Id(1455) (attempt 9) +//! ``` +//! +//! Two very different bugs produce that line: +//! +//! 1. **A leak.** An id was assigned by the index and its event was then +//! dropped instead of being pushed onto the persistence queue. Nothing will +//! ever deliver it and the stream is permanently gapped. +//! 2. **A collection failure.** The operation carrying the id *was* queued and +//! is still sitting in the analyzer, but batch collection keeps assembling +//! batches that exclude it. +//! +//! Distinguishing those needs a record of what was queued, which is what this +//! ledger keeps. It is deliberately not a fix for either: it only observes. +//! +//! # Where the two sides are observed +//! +//! Assignment happens inside the index (`indexset` bumps an `AtomicU64` and +//! stamps the event in the same commit), which this crate cannot hook. What it +//! *can* hook is the other end: every event that reaches persistence passes +//! through [`crate::persistence::task::Queue`], so an id present in the stream +//! but absent from this ledger was assigned and never queued. That is the leak +//! signature, and it is what [`EventLedger::gap_report`] reports. +//! +//! The producer is named by [`std::panic::Location`], captured with +//! `#[track_caller]` at the queue push, so a leak points at the call site that +//! produced its neighbours: the generated query, `insert_many`, an +//! acknowledge path, or vacuum's `apply_move`. A full backtrace is captured +//! too, but only when `RUST_BACKTRACE` is set; `Backtrace::capture` is +//! essentially free otherwise. +//! +//! # Gating: `debug_assertions`, not a cargo feature +//! +//! Recording is compiled in unconditionally and gated at run time by +//! [`enabled`], which is true when `debug_assertions` is on or when +//! `WT_EVENT_LEDGER` is set in the environment. +//! +//! `debug_assertions` was chosen over a new cargo feature for one reason: the +//! stall is only ever observed under a full `cargo test --workspace +//! --all-targets --all-features` run, and that run is a debug build, so the +//! instrumentation is on exactly where the bug appears. A feature would also +//! have been enabled by `--all-features`, but it would have to be declared in +//! `Cargo.toml`, and a diagnostic that lives behind a flag nobody sets in the +//! failing configuration is worthless. Release builds fold `enabled()` down +//! to the environment check and every recording call returns immediately, so +//! they pay a predictable-branch and nothing else. +//! +//! `WT_EVENT_LEDGER=1` exists so a release build can be told to record without +//! being rebuilt, for the day the stall shows up outside a test run. +//! +//! Memory is bounded by [`WINDOW`] ids per stream. The gap sits at the head of +//! the stream by construction, so a recent window always covers it; the report +//! states the window it holds so a reader can see that for themselves. + +// The ledger itself is arithmetic and bookkeeping, so it takes `core` and +// `alloc`. Two of its capabilities genuinely need an operating system, and only +// those are gated: `Backtrace`, and reading `WT_EVENT_LEDGER` from the +// environment. Without `std` the ledger is simply never enabled, which is the +// right answer for a diagnostic that an environment variable turns on. +use alloc::borrow::ToOwned as _; +// Only the gated backtrace field boxes anything. +#[cfg(feature = "std")] +use alloc::boxed::Box; +use alloc::collections::BTreeMap; +use alloc::string::{String, ToString as _}; +use alloc::vec::Vec; +use core::fmt::Write as _; +use core::panic::Location; +use hashbrown::HashMap; +#[cfg(feature = "std")] +use std::backtrace::Backtrace; +#[cfg(feature = "std")] +use std::sync::LazyLock; + +use data_bucket::Link; +use indexset::cdc::change::ChangeEvent; +use indexset::core::pair::Pair; +use parking_lot::Mutex; + +use crate::persistence::{OperationId, OperationType}; + +/// Ids retained per stream. The gap the guard reports is at the head of the +/// stream, so a window this size covers it many times over: the one observed +/// stall was 16 ids wide. +const WINDOW: usize = 8192; + +/// Gap ids listed individually in a report before it summarises the rest. +const MAX_LISTED_GAP_IDS: usize = 64; + +#[cfg(feature = "std")] +static ENABLED: LazyLock = LazyLock::new(|| { + if cfg!(debug_assertions) { + return true; + } + match std::env::var("WT_EVENT_LEDGER") { + Ok(value) => !value.is_empty() && value != "0" && !value.eq_ignore_ascii_case("false"), + Err(_) => false, + } +}); + +/// Without `std` there is no environment to read the switch from, so the +/// ledger stays off. Everything below still compiles and can be driven +/// directly by a caller that wants it; what is missing is the ambient way to +/// turn it on. +#[cfg(not(feature = "std"))] +static ENABLED: bool = false; + +/// Whether event bookkeeping is recording in this process. +/// +/// See the module comment for why this is `debug_assertions` plus an +/// environment override rather than a cargo feature. +#[inline] +pub fn enabled() -> bool { + #[cfg(feature = "std")] + { + *ENABLED + } + #[cfg(not(feature = "std"))] + { + ENABLED + } +} + +/// Which index's event id sequence a record belongs to. +/// +/// Every index keeps its own counter, so ids only mean anything relative to a +/// stream. `Primary` allocates nothing, which keeps the hot path free of +/// allocation; secondary streams are labelled by the `Debug` rendering of the +/// table's `AvailableIndexes` value, which is the only name available to +/// non-generic code here. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum EventStream { + Primary, + Secondary(String), +} + +impl core::fmt::Display for EventStream { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + EventStream::Primary => f.write_str("primary"), + EventStream::Secondary(index) => write!(f, "secondary {index}"), + } + } +} + +/// What has been observed happening to one event id. +/// +/// Flags, not a state machine: an id is queued, then collected into a batch, +/// then possibly trimmed back out and requeued, possibly several times over. +/// The report reads the whole set. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Stages(u8); + +impl Stages { + /// Pushed onto the persistence queue by a producer. + pub const QUEUED: Stages = Stages(1 << 0); + /// Pulled into a `BatchOperation` by the analyzer. + pub const COLLECTED: Stages = Stages(1 << 1); + /// Handed back to the analyzer's queue after a deferral or a trim. + pub const REQUEUED: Stages = Stages(1 << 2); + /// Removed from a batch by `remove_operations_from_events`. + pub const TRIMMED: Stages = Stages(1 << 3); + /// Covered by the applied watermark reported after a batch. + pub const APPLIED: Stages = Stages(1 << 4); + + fn insert(&mut self, other: Stages) { + self.0 |= other.0; + } + + /// Both flag sets at once. + pub const fn union(self, other: Stages) -> Stages { + Stages(self.0 | other.0) + } + + fn contains(self, other: Stages) -> bool { + self.0 & other.0 == other.0 + } +} + +impl core::fmt::Display for Stages { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut first = true; + for (flag, name) in [ + (Stages::QUEUED, "queued"), + (Stages::COLLECTED, "collected"), + (Stages::REQUEUED, "requeued"), + (Stages::TRIMMED, "trimmed"), + (Stages::APPLIED, "applied"), + ] { + if self.contains(flag) { + if !first { + f.write_str("+")?; + } + f.write_str(name)?; + first = false; + } + } + if first { + f.write_str("none")?; + } + Ok(()) + } +} + +#[derive(Debug)] +struct IdRecord { + stages: Stages, + /// The operation that carried this id when it was first queued. + op_id: Option, + op_type: Option, + /// Producer call site, from `#[track_caller]` at the queue push. + site: Option<&'static Location<'static>>, + /// Captured only when `RUST_BACKTRACE` is set; otherwise `Disabled` and + /// free. Boxed so that the common empty case costs a pointer instead of an + /// inline `Backtrace` in each of the thousands of records held per stream. + // Gated with the capture below: without `std` there is no `Backtrace`, + // and the ledger keeps the call site from `#[track_caller]` regardless, + // which is the half that names the producer. + #[cfg(feature = "std")] + backtrace: Option>, + collected: u32, + requeued: u32, +} + +impl IdRecord { + fn new() -> Self { + Self { + stages: Stages::default(), + op_id: None, + op_type: None, + site: None, + #[cfg(feature = "std")] + backtrace: None, + collected: 0, + requeued: 0, + } + } +} + +#[derive(Debug, Default)] +struct StreamLedger { + ids: BTreeMap, + /// Ids below this were evicted by the window and cannot be answered for. + evicted_below: u64, + /// Highest id ever seen at any stage on this stream. + highest_seen: u64, + /// Highest id the analyzer reported as applied. + applied_upto: u64, + queued_total: u64, +} + +impl StreamLedger { + fn entry(&mut self, id: u64) -> &mut IdRecord { + if id > self.highest_seen { + self.highest_seen = id; + } + self.ids.entry(id).or_insert_with(IdRecord::new) + } + + fn trim(&mut self) { + while self.ids.len() > WINDOW { + // Ids are close to monotonic, so the lowest key is the oldest. + if let Some((id, _)) = self.ids.pop_first() { + self.evicted_below = self.evicted_below.max(id + 1); + } else { + break; + } + } + } + + /// Lowest id this ledger can still answer for. + fn window_start(&self) -> u64 { + self.ids.keys().next().copied().unwrap_or(self.evicted_below) + } +} + +/// Per-table record of which index change event ids reached the persistence +/// queue, and what happened to them afterwards. +/// +/// Shared by `Arc` between the queue (the producer side), the analyzer, and +/// the `BatchOperation` whose guard reads it. +#[derive(Debug)] +pub struct EventLedger { + label: String, + /// True when no producer writes to this ledger, so "never queued" here + /// means "never recorded", not "leaked". Reports say so rather than + /// accusing a producer that was never watched. + detached: bool, + streams: Mutex>, +} + +impl EventLedger { + pub fn new(label: impl Into) -> Self { + Self { + label: label.into(), + detached: false, + streams: Mutex::new(HashMap::new()), + } + } + + /// A ledger attached to nothing, for analyzers and `BatchOperation`s built + /// outside `run_engine` (unit tests, defensive callers). It records + /// normally; it is simply never shared with a producer, so its reports say + /// so instead of reading a missing record as a leak. + pub fn detached() -> Self { + Self { + label: "".to_owned(), + detached: true, + streams: Mutex::new(HashMap::new()), + } + } + + pub fn label(&self) -> &str { + &self.label + } + + /// Collects the event ids of `evs`, for a later [`EventLedger::record_queued`]. + /// + /// Split from the recording itself because the queue only knows a push was + /// accepted after the operation has been moved into it, and a refused push + /// must not be recorded as queued. + pub fn event_ids(evs: &[ChangeEvent>]) -> Vec { + if !enabled() { + return Vec::new(); + } + evs.iter().map(|ev| ev.id().inner()).collect() + } + + /// Records every id in `ids` as queued by `site`. + /// + /// Called from the persistence queue push, which is the single point every + /// operation passes through on its way to the engine. An id in the applied + /// stream that never appears here was assigned by the index and dropped + /// before it reached persistence. + pub fn record_queued( + &self, + stream: EventStream, + ids: &[u64], + op_id: OperationId, + op_type: OperationType, + site: &'static Location<'static>, + ) { + if !enabled() || ids.is_empty() { + return; + } + // Costs nothing unless RUST_BACKTRACE is set: `capture` returns + // `Disabled` without walking any frames. One capture per push, moved + // onto the first newly recorded id, because every id in this vector + // came from the same producer. + #[cfg(feature = "std")] + let backtrace = Backtrace::capture(); + #[cfg(feature = "std")] + let mut backtrace = + matches!(backtrace.status(), std::backtrace::BacktraceStatus::Captured).then_some(backtrace); + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + for id in ids.iter().copied() { + let record = ledger.entry(id); + let first_time = !record.stages.contains(Stages::QUEUED); + record.stages.insert(Stages::QUEUED); + if first_time { + record.op_id = Some(op_id); + record.op_type = Some(op_type); + record.site = Some(site); + // One backtrace per push, not per id: every id in this vector + // has the same producer, and keeping one each would multiply + // the cost of a `RUST_BACKTRACE` run for no extra signal. + #[cfg(feature = "std")] + if record.backtrace.is_none() { + record.backtrace = backtrace.take().map(Box::new); + } + } else { + record.stages.insert(Stages::REQUEUED); + record.requeued = record.requeued.saturating_add(1); + } + } + ledger.queued_total = ledger.queued_total.saturating_add(ids.len() as u64); + ledger.trim(); + } + + /// Records a stage transition for a single id already known to a stream. + pub fn record_stage(&self, stream: EventStream, id: u64, stage: Stages) { + if !enabled() { + return; + } + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + let record = ledger.entry(id); + record.stages.insert(stage); + if stage.contains(Stages::COLLECTED) { + record.collected = record.collected.saturating_add(1); + } + if stage.contains(Stages::REQUEUED) { + record.requeued = record.requeued.saturating_add(1); + } + ledger.trim(); + } + + /// Records a stage transition for every id in `evs`. + pub fn record_stage_for_events(&self, stream: EventStream, evs: &[ChangeEvent>], stage: Stages) { + if !enabled() || evs.is_empty() { + return; + } + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + for ev in evs { + let record = ledger.entry(ev.id().inner()); + record.stages.insert(stage); + if stage.contains(Stages::COLLECTED) { + record.collected = record.collected.saturating_add(1); + } + if stage.contains(Stages::REQUEUED) { + record.requeued = record.requeued.saturating_add(1); + } + } + ledger.trim(); + } + + /// Records the applied watermark reported after a batch was accepted. + pub fn record_applied_upto(&self, stream: EventStream, id: u64) { + if !enabled() || id == 0 { + return; + } + let mut streams = self.streams.lock(); + let ledger = streams.entry(stream).or_default(); + if id > ledger.applied_upto { + ledger.applied_upto = id; + } + if id > ledger.highest_seen { + ledger.highest_seen = id; + } + for (_, record) in ledger.ids.range_mut(..=id) { + record.stages.insert(Stages::APPLIED); + } + } + + /// Explains the gap between `last_applied` and `next_available`. + /// + /// This is the whole point of the ledger. For every id in the hole it says + /// whether the id ever reached the persistence queue, and if it did, what + /// happened to it afterwards, so the reader can tell a leaked event from a + /// batch collection that keeps missing one. + pub fn gap_report(&self, stream: &EventStream, last_applied: u64, next_available: u64) -> String { + let mut out = String::new(); + if !enabled() { + let _ = write!( + out, + " Event bookkeeping is off in this build, so the gap cannot be attributed. \ + Re-run with WT_EVENT_LEDGER=1 (and RUST_BACKTRACE=1 for producer backtraces) \ + to have the next occurrence name its own cause." + ); + return out; + } + + if self.detached { + let _ = write!( + out, + " This analyzer holds detached event bookkeeping: no producer ever wrote to it, \ + so the gap cannot be attributed. Only analyzers built outside `run_engine` are detached." + ); + return out; + } + + let streams = self.streams.lock(); + let Some(ledger) = streams.get(stream) else { + let _ = write!( + out, + " Event bookkeeping for table {label} holds no records at all for the {stream} stream, \ + which should be impossible while it is applying that stream's events.", + label = self.label, + ); + return out; + }; + + let window_start = ledger.window_start(); + let first_missing = last_applied.saturating_add(1); + if next_available <= first_missing { + return out; + } + + // Bounded on purpose: this runs while building a panic message, and a + // corrupt watermark could otherwise make it walk billions of ids. + let scan_end = next_available.min(first_missing.saturating_add(WINDOW as u64)); + let mut never_queued = Vec::new(); + let mut queued_not_applied = Vec::new(); + for id in first_missing..scan_end { + match ledger.ids.get(&id) { + Some(record) if record.stages.contains(Stages::QUEUED) => queued_not_applied.push((id, record)), + _ => never_queued.push(id), + } + } + + let _ = write!( + out, + " Bookkeeping for table {label}, {stream} stream: window covers ids {window_start}..={highest}, \ + applied watermark {applied}, {total} id(s) queued in all, \ + {gap_len} id(s) in the gap ({scanned} scanned), {never} never queued, {queued} queued.", + label = self.label, + window_start = window_start, + highest = ledger.highest_seen, + applied = ledger.applied_upto, + total = ledger.queued_total, + gap_len = next_available - first_missing, + scanned = scan_end - first_missing, + never = never_queued.len(), + queued = queued_not_applied.len(), + ); + + if first_missing < window_start { + let _ = write!( + out, + " CAUTION: part of the gap ({first_missing}..{window_start}) fell out of the retained window, \ + so those ids are unattributable rather than proven missing." + ); + } + + if !never_queued.is_empty() { + let _ = write!(out, " ASSIGNED BUT NEVER QUEUED: {}.", format_ids(&never_queued)); + let _ = write!( + out, + " Those ids were consumed by the index and their events never reached the persistence queue, \ + so nothing will ever deliver them: this is an event leak upstream of the analyzer, \ + not a batch collection problem." + ); + let _ = write!(out, "{}", bracketing_producers(ledger, last_applied, next_available)); + } + + if !queued_not_applied.is_empty() { + let _ = write!(out, " QUEUED BUT NOT APPLIED:"); + for (id, record) in queued_not_applied.iter().take(MAX_LISTED_GAP_IDS) { + let _ = write!( + out, + " [{id}: {stages}, collected {collected}x, requeued {requeued}x, {op_type} op {op_id} from {site}]", + stages = record.stages, + collected = record.collected, + requeued = record.requeued, + op_type = OptionDisplay(record.op_type.as_ref().map(|t| format!("{t:?}"))), + op_id = OptionDisplay(record.op_id.as_ref().map(|id| format!("{id:?}"))), + site = OptionDisplay(record.site.map(|site| site.to_string())), + ); + } + if queued_not_applied.len() > MAX_LISTED_GAP_IDS { + let _ = write!(out, " and {} more", queued_not_applied.len() - MAX_LISTED_GAP_IDS); + } + let _ = write!( + out, + ". Those events did reach the queue, so their operations are still somewhere in the analyzer \ + and batch collection is failing to assemble them: the bug is in collection, not in event production." + ); + } + + out + } +} + +/// Names the producers on either side of the hole. +/// +/// A leaked id has no record of its own, so the closest evidence about who +/// should have produced it is who produced its neighbours. One index allocates +/// its ids from one counter, so the neighbours are almost always the same call +/// path. +fn bracketing_producers(ledger: &StreamLedger, last_applied: u64, next_available: u64) -> String { + let mut out = String::new(); + let before = ledger.ids.range(..=last_applied).next_back(); + let after = ledger.ids.range(next_available..).next(); + for (side, entry) in [("before the gap", before), ("after the gap", after)] { + let Some((id, record)) = entry else { + let _ = write!(out, " No record {side}."); + continue; + }; + let _ = write!( + out, + " Producer {side} (id {id}): {op_type}, op {op_id}, pushed from {site}.", + op_type = OptionDisplay(record.op_type.as_ref().map(|t| format!("{t:?}"))), + op_id = OptionDisplay(record.op_id.as_ref().map(|id| format!("{id:?}"))), + site = OptionDisplay(record.site.map(|site| site.to_string())), + ); + #[cfg(feature = "std")] + if let Some(backtrace) = &record.backtrace { + let _ = write!(out, " Backtrace:\n{backtrace}\n"); + } + } + #[cfg(feature = "std")] + if !ledger.ids.values().any(|record| record.backtrace.is_some()) { + let _ = write!( + out, + " Re-run with RUST_BACKTRACE=1 for the full producer backtraces, which this run did not capture." + ); + } + out +} + +impl Default for EventLedger { + fn default() -> Self { + Self::detached() + } +} + +struct OptionDisplay(Option); + +impl core::fmt::Display for OptionDisplay { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match &self.0 { + Some(value) => f.write_str(value), + None => f.write_str(""), + } + } +} + +/// Renders a sorted id list compactly, collapsing runs. +fn format_ids(ids: &[u64]) -> String { + let mut out = String::new(); + let mut listed = 0usize; + let mut i = 0usize; + while i < ids.len() && listed < MAX_LISTED_GAP_IDS { + let start = ids[i]; + let mut end = start; + while i + 1 < ids.len() && ids[i + 1] == end + 1 { + i += 1; + end = ids[i]; + } + if !out.is_empty() { + out.push_str(", "); + } + if start == end { + let _ = write!(out, "{start}"); + } else { + let _ = write!(out, "{start}..={end}"); + } + listed += 1; + i += 1; + } + if i < ids.len() { + let _ = write!(out, ", and {} more", ids.len() - i); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn insert_at(id: u64) -> ChangeEvent> { + ChangeEvent::InsertAt { + event_id: id.into(), + max_value: Pair { + key: id, + value: Link::default(), + }, + value: Pair { + key: id, + value: Link::default(), + }, + index: 0, + } + } + + #[track_caller] + fn queue(ledger: &EventLedger, ids: &[u64], op_type: OperationType) { + let evs = ids.iter().copied().map(insert_at).collect::>(); + let ids = EventLedger::event_ids(&evs); + ledger.record_queued( + EventStream::Primary, + &ids, + OperationId::Single(uuid::Uuid::from_u128(1)), + op_type, + Location::caller(), + ); + } + + /// These tests assert on what the ledger recorded, so they are only + /// meaningful where it records. That is every normal test run + /// (`debug_assertions`); a `--release` test run skips them rather than + /// failing on a report that correctly says bookkeeping was off. + fn recording() -> bool { + enabled() + } + + #[test] + fn names_ids_that_were_never_queued() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3], OperationType::Insert); + // 4 and 5 are assigned by the index and leaked: nothing queues them. + queue(&ledger, &[6, 7], OperationType::Update); + + let report = ledger.gap_report(&EventStream::Primary, 3, 6); + + assert!(report.contains("ASSIGNED BUT NEVER QUEUED"), "{report}"); + assert!(report.contains("4..=5"), "{report}"); + assert!(report.contains("event leak upstream of the analyzer"), "{report}"); + } + + #[test] + fn distinguishes_a_queued_id_from_a_leaked_one() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3], OperationType::Insert); + queue(&ledger, &[4], OperationType::Update); + queue(&ledger, &[6], OperationType::Insert); + + let report = ledger.gap_report(&EventStream::Primary, 3, 6); + + assert!(report.contains("QUEUED BUT NOT APPLIED"), "{report}"); + assert!(report.contains("the bug is in collection"), "{report}"); + // Only id 5 is missing; 4 was queued. + assert!(report.contains("ASSIGNED BUT NEVER QUEUED: 5."), "{report}"); + } + + #[test] + fn reports_a_gapless_stream_as_nothing() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3], OperationType::Insert); + assert!(ledger.gap_report(&EventStream::Primary, 3, 4).is_empty()); + } + + #[test] + fn window_eviction_is_reported_rather_than_guessed() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + let ids = (1..=(WINDOW as u64 + 64)).collect::>(); + queue(&ledger, &ids, OperationType::Insert); + + // Ask about a gap far below the retained window. + let report = ledger.gap_report(&EventStream::Primary, 1, 20); + assert!(report.contains("fell out of the retained window"), "{report}"); + } + + #[test] + fn applied_watermark_marks_everything_behind_it() { + if !recording() { + return; + } + let ledger = EventLedger::new("test/table"); + queue(&ledger, &[1, 2, 3, 5], OperationType::Insert); + ledger.record_applied_upto(EventStream::Primary, 3); + + let report = ledger.gap_report(&EventStream::Primary, 3, 5); + assert!(report.contains("ASSIGNED BUT NEVER QUEUED: 4."), "{report}"); + } + + #[test] + fn stages_render_every_flag_set() { + let mut stages = Stages::default(); + assert_eq!(stages.to_string(), "none"); + stages.insert(Stages::QUEUED); + stages.insert(Stages::TRIMMED); + assert_eq!(stages.to_string(), "queued+trimmed"); + } +} diff --git a/src/persistence/mod.rs b/src/persistence/mod.rs index b5e03ce3..037823ef 100644 --- a/src/persistence/mod.rs +++ b/src/persistence/mod.rs @@ -1,20 +1,25 @@ -use std::future::Future; - -use data_bucket::page::PageId; +use core::future::Future; +#[cfg(feature = "std")] use crate::persistence::operation::BatchOperation; +#[cfg(feature = "std")] pub use engine::DiskConfig; +#[cfg(feature = "std")] pub use engine::DiskPersistenceEngine; +#[cfg(feature = "std")] pub use error::{ PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, PersistenceResult, PersistenceState, load_persisted_state, }; +pub use event_ledger::{EventLedger, EventStream, Stages}; pub use operation::{ AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, OperationId, OperationType, UpdateOperation, validate_events, }; +#[cfg(feature = "std")] pub use readonly_engine::ReadOnlyPersistenceEngine; +#[cfg(feature = "std")] pub use space::{ ArtPersistenceKey, IndexTableOfContents, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, @@ -22,6 +27,7 @@ pub use space::{ TocEntryOversizedError, map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, }; +#[cfg(feature = "std")] pub use task::{PersistenceMonitor, PersistenceTask}; /// Result of retiring one Arc-owned persisted table generation. @@ -39,13 +45,13 @@ pub struct UnloadReport { /// can keep serving it or retry. A failure returned by `close` has no retained /// generation because shutdown was already attempted and consumed it. pub struct UnloadFailure { - generation: Option>, + generation: Option>, error: eyre::Report, } impl UnloadFailure { #[doc(hidden)] - pub fn retained(generation: std::sync::Arc, error: eyre::Report) -> Self { + pub fn retained(generation: alloc::sync::Arc, error: eyre::Report) -> Self { Self { generation: Some(generation), error, @@ -61,7 +67,7 @@ impl UnloadFailure { } /// Returns the still-live generation when shutdown never began. - pub fn into_generation(self) -> Option> { + pub fn into_generation(self) -> Option> { self.generation } @@ -71,8 +77,8 @@ impl UnloadFailure { } } -impl std::fmt::Debug for UnloadFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Debug for UnloadFailure { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_struct("UnloadFailure") .field("generation_retained", &self.generation.is_some()) @@ -81,19 +87,28 @@ impl std::fmt::Debug for UnloadFailure { } } -impl std::fmt::Display for UnloadFailure { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for UnloadFailure { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { self.error.fmt(formatter) } } -impl std::error::Error for UnloadFailure {} +impl core::error::Error for UnloadFailure {} + +#[cfg(feature = "std")] +use data_bucket::page::PageId; +#[cfg(feature = "std")] mod engine; +#[cfg(feature = "std")] mod error; +pub mod event_ledger; pub mod operation; +#[cfg(feature = "std")] mod readonly_engine; +#[cfg(feature = "std")] mod space; +#[cfg(feature = "std")] mod task; // TODO: remove this @@ -142,6 +157,7 @@ where } } +#[cfg(feature = "std")] pub trait PersistenceEngine { type Config: PersistenceConfig; diff --git a/src/persistence/operation/batch.rs b/src/persistence/operation/batch.rs index 958bcfc0..57164192 100644 --- a/src/persistence/operation/batch.rs +++ b/src/persistence/operation/batch.rs @@ -1,7 +1,10 @@ -use std::collections::HashMap; -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::boxed::Box; +use alloc::sync::Arc; +use alloc::{string::ToString, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; +use hashbrown::HashMap; use data_bucket::page::PageId; use data_bucket::{Link, SizeMeasurable}; @@ -10,6 +13,7 @@ use indexset::core::pair::Pair; use worktable_codegen::{MemStat, worktable}; use crate::persistence::OperationType; +use crate::persistence::event_ledger::{self, EventLedger, EventStream, Stages}; use crate::persistence::space::{BatchChangeEvent, BatchData}; use crate::persistence::task::{LastEventIds, QueueInnerRow}; use crate::prelude::*; @@ -149,6 +153,10 @@ fn latest_data_writes( pub struct BatchOperation { ops: Vec>, info_wt: BatchInnerWorkTable, + /// Event bookkeeping shared with the queue that produced `ops`, read by + /// the event-gap guard in `validate` so a stall names its own cause. + /// Diagnostics only, and `None` for batches built outside the analyzer. + event_ledger: Option>, prepared_index_evs: Option>, phantom_data: PhantomData, } @@ -173,11 +181,23 @@ where Self { ops, info_wt, + event_ledger: None, prepared_index_evs: None, phantom_data: PhantomData, } } + /// Attaches the analyzer's event bookkeeping, so the gap guard below can + /// say which ids in a gap ever reached the persistence queue. + /// + /// A builder method rather than a `new` parameter, so every existing + /// caller of `new` keeps working unchanged and the batch stays usable + /// without any bookkeeping at all. + pub fn with_event_ledger(mut self, ledger: Arc) -> Self { + self.event_ledger = Some(ledger); + self + } + /// Remove metadata immediately after `self.ops.remove(removed_pos)`. /// /// At entry, `self.ops.len()` is already one shorter while `info_wt` still @@ -272,9 +292,54 @@ where prepared_evs.secondary_evs.remove(op_secondary); } + self.record_stage(&removed_ops, Stages::TRIMMED); + Ok(removed_ops) } + /// Records `stage` against every event id carried by `ops`. + /// + /// Diagnostics only. Skipped entirely when bookkeeping is off, which keeps + /// the `Debug` formatting of secondary index labels out of release builds. + fn record_stage(&self, ops: &[Operation], stage: Stages) { + let Some(ledger) = &self.event_ledger else { + return; + }; + if !event_ledger::enabled() { + return; + } + for op in ops { + if let Some(evs) = op.primary_key_events() { + ledger.record_stage_for_events(EventStream::Primary, evs, stage); + } + // See the matching note in `QueueAnalyzer::record_ops_stage`: the + // producer side records primary ids only, so an id observed on a + // secondary stream here is known to have been queued. + for (index, id) in op.secondary_key_events().iter_event_ids() { + ledger.record_stage( + EventStream::Secondary(format!("{index:?}")), + id.inner(), + stage.union(Stages::QUEUED), + ); + } + } + } + + /// The bookkeeping's account of a gap, or a note saying there is none. + fn gap_report( + &self, + stream: &EventStream, + last_applied: IndexChangeEventId, + next_available: IndexChangeEventId, + ) -> String { + match &self.event_ledger { + Some(ledger) => ledger.gap_report(stream, last_applied.inner(), next_available.inner()), + None => { + " This batch was built without event bookkeeping attached, so the gap cannot be attributed.".to_owned() + } + } + } + pub fn get_last_event_ids(&self) -> LastEventIds { let prepared_evs = self .prepared_index_evs @@ -358,8 +423,9 @@ where // that persists is a bug upstream of the analyzer; report it // loudly instead of force-applying and corrupting the file. if attempts > GIVE_UP_AFTER_ATTEMPTS { + let report = self.gap_report(&EventStream::Primary, last_ids.primary_id, id); return Err(eyre::eyre!( - "persistence stalled on primary index event gap: last applied {:?}, next available {:?} after {attempts} attempts, with {} operations queued. Every one of them was collected and the stream is still gapped, so the operation carrying the missing id never reached the queue: an event id was consumed without its event being pushed. The producer is upstream of the analyzer, not here.", + "persistence stalled on primary index event gap: last applied {:?}, next available {:?} after {attempts} attempts, with {} operations queued.{report}", last_ids.primary_id, id, self.ops.len() @@ -381,8 +447,9 @@ where // stream, defer until the missing event arrives, and report // a persistent gap as the bug it is. if attempts > GIVE_UP_AFTER_ATTEMPTS { + let report = self.gap_report(&EventStream::Secondary(format!("{index:?}")), *last, id); return Err(eyre::eyre!( - "persistence stalled on secondary index {index:?} event gap: last applied {last:?}, next available {id:?} after {attempts} attempts, with {} operations queued. All of them were collected and the stream is still gapped, so the operation carrying the missing id never reached the queue.", + "persistence stalled on secondary index {index:?} event gap: last applied {last:?}, next available {id:?} after {attempts} attempts, with {} operations queued.{report}", self.ops.len() )); } @@ -484,9 +551,10 @@ where #[cfg(test)] mod tests { - use std::collections::HashMap; + use hashbrown::HashMap; use data_bucket::Link; + use data_bucket::page::PageId; use indexset::core::pair::Pair; use uuid::Uuid; @@ -534,7 +602,7 @@ mod tests { // Deliberately reverse vector order: operation ids, not incidental // collection order, define which bytes are newest. let batch = latest_data_writes(&[insert(2, new_link, vec![2; 6]), insert(1, old_link, vec![1; 4])]); - let writes = batch.get(&1.into()).unwrap(); + let writes = batch.get(&PageId::from(1u32)).unwrap(); assert_eq!(writes, &vec![(new_link, vec![2; 6])]); } @@ -554,7 +622,7 @@ mod tests { for _ in 0..128 { let batch = latest_data_writes(&[insert(2, newer_link, vec![2; 8]), insert(1, older_link, vec![1; 8])]); - let writes = batch.get(&1.into()).unwrap(); + let writes = batch.get(&PageId::from(1u32)).unwrap(); assert_eq!(writes, &vec![(older_link, vec![1; 8]), (newer_link, vec![2; 8])]); } @@ -579,7 +647,7 @@ mod tests { ]); assert_eq!( - batch.get(&1.into()).unwrap(), + batch.get(&PageId::from(1u32)).unwrap(), &vec![(older_link, vec![1; 8]), (newer_link, vec![2; 8])] ); } @@ -602,7 +670,7 @@ mod tests { multi_insert(1, new_link, vec![2; 6]), ]); - assert_eq!(batch.get(&1.into()).unwrap(), &vec![(new_link, vec![2; 6])]); + assert_eq!(batch.get(&PageId::from(1u32)).unwrap(), &vec![(new_link, vec![2; 6])]); } #[tokio::test] @@ -634,7 +702,7 @@ mod tests { } fn iter_event_ids(&self) -> impl Iterator { - std::iter::empty() + core::iter::empty() } fn sort(&mut self) {} @@ -734,7 +802,7 @@ mod tests { let data = batch.get_batch_data_op().unwrap(); assert_eq!( - data.get(&1.into()).unwrap(), + data.get(&PageId::from(1u32)).unwrap(), &vec![(survivor_link, vec![7; 4])], "the surviving data-only write must stay in the applied batch" ); diff --git a/src/persistence/operation/mod.rs b/src/persistence/operation/mod.rs index 5f7954fc..3097abbe 100644 --- a/src/persistence/operation/mod.rs +++ b/src/persistence/operation/mod.rs @@ -1,11 +1,12 @@ +#[cfg(feature = "std")] mod batch; #[allow(clippy::module_inception)] mod operation; mod util; -use std::cmp::Ordering; -use std::fmt::Debug; -use std::hash::{Hash, Hasher}; +use core::cmp::Ordering; +use core::fmt::Debug; +use core::hash::{Hash, Hasher}; use data_bucket::SizeMeasurable; use derive_more::Display; @@ -14,6 +15,7 @@ use uuid::Uuid; use crate::prelude::From; +#[cfg(feature = "std")] pub use batch::{BatchInnerRow, BatchInnerWorkTable, BatchOperation}; pub use operation::{AcknowledgeOperation, DeleteOperation, InsertOperation, Operation, UpdateOperation}; pub use util::validate_events; diff --git a/src/persistence/operation/operation.rs b/src/persistence/operation/operation.rs index 36af8298..51a25c3c 100644 --- a/src/persistence/operation/operation.rs +++ b/src/persistence/operation/operation.rs @@ -1,5 +1,6 @@ -use std::fmt::Debug; -use std::hash::{Hash, Hasher}; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::hash::{Hash, Hasher}; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; diff --git a/src/persistence/operation/util.rs b/src/persistence/operation/util.rs index 405a9ce3..dd58bb20 100644 --- a/src/persistence/operation/util.rs +++ b/src/persistence/operation/util.rs @@ -1,7 +1,8 @@ +use alloc::vec::Vec; +use core::fmt::Debug; use data_bucket::Link; use indexset::cdc::change::{self, ChangeEvent}; use indexset::core::pair::Pair; -use std::fmt::Debug; pub fn validate_events(evs: &mut Vec>>) -> Vec>> where @@ -20,7 +21,7 @@ where } } - removed_events.sort_by_key(|ev2| std::cmp::Reverse(ev2.id())); + removed_events.sort_by_key(|ev2| core::cmp::Reverse(ev2.id())); removed_events } diff --git a/src/persistence/readonly_engine.rs b/src/persistence/readonly_engine.rs index 27a19d25..b35389bc 100644 --- a/src/persistence/readonly_engine.rs +++ b/src/persistence/readonly_engine.rs @@ -1,5 +1,5 @@ -use std::fmt::Debug; -use std::hash::Hash; +use core::fmt::Debug; +use core::hash::Hash; use crate::TableSecondaryIndexEventsOps; use crate::persistence::operation::{BatchOperation, Operation}; diff --git a/src/persistence/space/art_index.rs b/src/persistence/space/art_index.rs index d2db86c6..97f49a1a 100644 --- a/src/persistence/space/art_index.rs +++ b/src/persistence/space/art_index.rs @@ -6,17 +6,18 @@ //! applies the WAL, writes a new native checkpoint atomically, and drops the //! temporary tree; no duplicate ART is retained during normal operation. -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; +use alloc::{borrow::ToOwned, string::String, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; use std::path::{Path, PathBuf}; +use crate::fsx::File; use data_bucket::{Link, page::PageId}; use eyre::{Context, bail, eyre}; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; -use tokio::fs::{File, OpenOptions}; -use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use nagoya::io::{Read as _, Seek as _, Write as _}; use crate::index::{ ArcticIndex, ArcticKey, ArcticMultiIndex, CongeeIndex, CongeeKey, PersistentArcticIndex, @@ -76,14 +77,14 @@ macro_rules! impl_art_persistence_key { ($($type:ty),+ $(,)?) => { $( impl ArtPersistenceKey for $type { - const WIDTH: u8 = std::mem::size_of::() as u8; + const WIDTH: u8 = core::mem::size_of::() as u8; fn encode_art_key(&self, output: &mut Vec) { output.extend_from_slice(&self.to_be_bytes()); } fn decode_art_key(bytes: &[u8]) -> eyre::Result { - let bytes: [u8; std::mem::size_of::()] = bytes + let bytes: [u8; core::mem::size_of::()] = bytes .try_into() .map_err(|_| eyre!("invalid {}-byte ART key", Self::WIDTH))?; Ok(Self::from_be_bytes(bytes)) @@ -112,7 +113,7 @@ macro_rules! impl_art_persistence_key_signed { ($($type:ty => $raw:ty),+ $(,)?) => { $( impl ArtPersistenceKey for $type { - const WIDTH: u8 = std::mem::size_of::() as u8; + const WIDTH: u8 = core::mem::size_of::() as u8; fn encode_art_key(&self, output: &mut Vec) { let raw = (*self as $raw) ^ ((1 as $raw) << (<$raw>::BITS - 1)); @@ -120,7 +121,7 @@ macro_rules! impl_art_persistence_key_signed { } fn decode_art_key(bytes: &[u8]) -> eyre::Result { - let bytes: [u8; std::mem::size_of::()] = bytes + let bytes: [u8; core::mem::size_of::()] = bytes .try_into() .map_err(|_| eyre!("invalid {}-byte ART key", Self::WIDTH))?; let raw = <$raw>::from_be_bytes(bytes) ^ ((1 as $raw) << (<$raw>::BITS - 1)); @@ -200,17 +201,17 @@ impl ArtFile { // with live state or block a future rename. let stale_temporary = temporary_path(&path); if stale_temporary.exists() { - tokio::fs::remove_file(&stale_temporary).await?; + crate::fsx::remove_file(&stale_temporary).await?; } if !path.exists() { Self::write_new_file(&path, backend, table_version, &empty_snapshot).await?; } let image = Self::read_image(&path, backend, table_version).await?; - let mut file = OpenOptions::new().read(true).write(true).open(&path).await?; + let mut file = crate::fsx::open(&path).await?; // Remove an incomplete final frame before appending. Leaving it in // place would make every later valid frame unreachable on recovery. - file.set_len(image.durable_len).await?; - file.seek(std::io::SeekFrom::End(0)).await?; + crate::fsx::set_len(&mut file, image.durable_len).await?; + file.seek(nagoya::io::SeekFrom::End(0)).await?; Ok(Self { path, file, @@ -222,7 +223,7 @@ impl ArtFile { } async fn read_image(path: &Path, backend: Backend, table_version: u32) -> eyre::Result> { - let mut file = File::open(path) + let mut file = crate::fsx::open(path) .await .wrap_err_with(|| format!("open ART index {}", path.display()))?; let mut bytes = Vec::new(); @@ -339,8 +340,8 @@ impl ArtFile { async fn rewrite(&mut self, snapshot: &[u8]) -> eyre::Result<()> { Self::write_file_atomically(&self.path, self.backend, self.table_version, snapshot).await?; - self.file = OpenOptions::new().read(true).write(true).open(&self.path).await?; - self.file.seek(std::io::SeekFrom::End(0)).await?; + self.file = crate::fsx::open(&self.path).await?; + self.file.seek(nagoya::io::SeekFrom::End(0)).await?; self.wal_bytes = 0; Ok(()) } @@ -356,7 +357,7 @@ impl ArtFile { ) -> eyre::Result<()> { let temporary = temporary_path(path); Self::write_new_file(&temporary, backend, table_version, snapshot).await?; - tokio::fs::rename(&temporary, path).await?; + crate::fsx::rename(&temporary, path).await?; Ok(()) } @@ -372,11 +373,11 @@ impl ArtFile { header.extend_from_slice(&0u32.to_le_bytes()); debug_assert_eq!(header.len(), HEADER_LEN); - let mut file = File::create(path).await?; + let mut file = crate::fsx::create(path).await?; file.write_all(&header).await?; file.write_all(snapshot).await?; file.flush().await?; - file.sync_data().await?; + crate::fsx::sync_data(&mut file).await?; Ok(()) } } @@ -384,7 +385,7 @@ impl ArtFile { fn encode_wal_record(record: &WalRecord) -> Vec { let mut key = Vec::new(); record.key.encode_art_key(&mut key); - let variable_prefix = usize::from(K::WIDTH == 0) * std::mem::size_of::(); + let variable_prefix = usize::from(K::WIDTH == 0) * core::mem::size_of::(); let mut bytes = Vec::with_capacity(9 + variable_prefix + key.len() + 12); bytes.extend_from_slice(&record.event_id.to_le_bytes()); match record.op { @@ -717,7 +718,7 @@ where path, Backend::ArcticVariable, table_version, - encode_multi_pairs(std::iter::empty::<(K, Link)>()), + encode_multi_pairs(core::iter::empty::<(K, Link)>()), ) .await?, }) @@ -922,7 +923,7 @@ where K: ArtPersistenceKey + ArcticKey, { async fn new(path: PathBuf, table_version: u32) -> eyre::Result { - let snapshot = encode_multi_pairs(std::iter::empty::<(K, Link)>()); + let snapshot = encode_multi_pairs(core::iter::empty::<(K, Link)>()); Ok(Self { file: ArtFile::open(path, Backend::ArcticMulti, table_version, snapshot).await?, }) @@ -1356,15 +1357,15 @@ mod tests { space.process_change_event(set_event(0, 7, link(7))).await.unwrap(); drop(space); - let durable_len = tokio::fs::metadata(&path).await.unwrap().len(); - let mut file = OpenOptions::new().append(true).open(&path).await.unwrap(); + let durable_len = crate::fsx::metadata(&path).await.unwrap(); + let mut file = crate::fsx::append(&path).await.unwrap(); file.write_all(&WAL_MAGIC[..2]).await.unwrap(); file.flush().await.unwrap(); drop(file); - assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), durable_len + 2); + assert_eq!(crate::fsx::metadata(&path).await.unwrap(), durable_len + 2); let mut space = SpaceArcticIndex::::new(path.clone(), 1).await.unwrap(); - assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), durable_len); + assert_eq!(crate::fsx::metadata(&path).await.unwrap(), durable_len); space.process_change_event(set_event(1, 8, link(8))).await.unwrap(); drop(space); @@ -1373,7 +1374,7 @@ mod tests { .unwrap(); assert_eq!(index.get_value(&7).unwrap().0, link(7)); assert_eq!(index.get_value(&8).unwrap().0, link(8)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1401,7 +1402,7 @@ mod tests { .unwrap(); assert_eq!(index.len(), 128); assert_eq!(index.get_value(&91).unwrap().0, link(92)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } fn remove_event(id: u64, key: u64, value: Link) -> ChangeEvent> { @@ -1439,7 +1440,7 @@ mod tests { assert_eq!(decode_multi_pairs::(&bytes).unwrap(), pairs); assert!(decode_multi_pairs::(&bytes[..bytes.len() - 1]).is_err()); assert_eq!( - decode_multi_pairs::(&encode_multi_pairs(std::iter::empty::<(u64, Link)>())).unwrap(), + decode_multi_pairs::(&encode_multi_pairs(core::iter::empty::<(u64, Link)>())).unwrap(), vec![] ); } @@ -1500,7 +1501,7 @@ mod tests { // A unique reader must refuse the multi file outright. assert!(ArtFile::::read_image(&path, Backend::Arctic, 5).await.is_err()); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1522,7 +1523,7 @@ mod tests { .unwrap(); assert_eq!(reloaded.len(), 50); assert_eq!(reloaded.get(&(u128::MAX - 3)).len(), 10); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1551,7 +1552,7 @@ mod tests { .unwrap(); assert_eq!(reloaded.len(), 4); assert_eq!(reloaded.get_value(&"🦀".to_owned()).unwrap().0, link(4)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[test] @@ -1569,7 +1570,7 @@ mod tests { // A leftover temporary from a crashed checkpoint must be cleaned up // when the index opens. - tokio::fs::write(&temporary, b"crashed checkpoint leftovers") + crate::fsx::write(&temporary, b"crashed checkpoint leftovers") .await .unwrap(); let mut space = SpaceArcticIndex::::new(path.clone(), 1).await.unwrap(); @@ -1597,7 +1598,7 @@ mod tests { .unwrap(); assert_eq!(reloaded.get_value(&7).unwrap().0, link(7)); assert_eq!(reloaded.get_value(&9).unwrap().0, link(9)); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } #[tokio::test] @@ -1611,16 +1612,16 @@ mod tests { assert!(ArtFile::::read_image(&path, Backend::Congee, 7).await.is_err()); assert!(ArtFile::::read_image(&path, Backend::Arctic, 7).await.is_err()); - let mut file = OpenOptions::new().read(true).write(true).open(&path).await.unwrap(); - file.seek(std::io::SeekFrom::End(-1)).await.unwrap(); + let mut file = crate::fsx::open(&path).await.unwrap(); + file.seek(nagoya::io::SeekFrom::End(-1)).await.unwrap(); let mut last = [0u8; 1]; file.read_exact(&mut last).await.unwrap(); - file.seek(std::io::SeekFrom::End(-1)).await.unwrap(); + file.seek(nagoya::io::SeekFrom::End(-1)).await.unwrap(); file.write_all(&[last[0] ^ 0x80]).await.unwrap(); file.flush().await.unwrap(); drop(file); assert!(ArtFile::::read_image(&path, Backend::Arctic, 7).await.is_err()); - tokio::fs::remove_file(path).await.unwrap(); + crate::fsx::remove_file(path).await.unwrap(); } } diff --git a/src/persistence/space/data.rs b/src/persistence/space/data.rs index dd200019..93338d70 100644 --- a/src/persistence/space/data.rs +++ b/src/persistence/space/data.rs @@ -1,7 +1,9 @@ -use std::collections::HashSet; -use std::io::SeekFrom; +use alloc::{string::String, string::ToString, vec::Vec}; +use hashbrown::HashSet; +use nagoya::io::SeekFrom; use std::path::Path; +use crate::fsx::File; use crate::persistence::SpaceDataOps; use crate::persistence::space::{BatchData, open_or_create_file}; use crate::prelude::WT_DATA_EXTENSION; @@ -10,6 +12,7 @@ use data_bucket::{ DataPage, GeneralHeader, GeneralPage, Link, PageType, Persistable, SizeMeasurable, SpaceInfoPage, parse_data_pages_batch, parse_general_header_by_index, parse_page, persist_page, persist_pages_batch, update_at, }; +use nagoya::io::{Seek as _, Write as _}; use rkyv::api::high::HighDeserializer; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -17,8 +20,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize}; -use tokio::fs::File; -use tokio::io::{AsyncSeekExt, AsyncWriteExt}; fn link_sort_key(link: &Link) -> (u32, u32) { (link.page_id.into(), link.offset) @@ -206,7 +207,7 @@ impl SpaceData

) -> bool { - let free_ranges = std::mem::take(&mut self.info.inner.empty_links_list); + let free_ranges = core::mem::take(&mut self.info.inner.empty_links_list); let (remaining, changed) = subtract_used_ranges(free_ranges, used_links); self.info.inner.empty_links_list = remaining; changed @@ -241,8 +242,8 @@ where } else { open_or_create_file(path).await? }; - let info = parse_page::<_, PAGE_SIZE>(&mut data_file, 0).await?; - let file_length = data_file.metadata().await?.len(); + let info = parse_page::<_, PAGE_SIZE, PAGE_SIZE>(&mut data_file, 0).await?; + let file_length = crate::fsx::file_metadata(&mut data_file).await?; // Mirror the index file's ceil logic: a file whose length is an exact // page multiple ends with a full last page, so the plain floor // division names a page id one past EOF and reopening the table fails @@ -253,7 +254,7 @@ where } else { file_length / PAGE_SIZE as u64 }; - let last_page_header = parse_general_header_by_index(&mut data_file, page_id as u32).await?; + let last_page_header = parse_general_header_by_index::(&mut data_file, page_id as u32).await?; Ok(Self { data_file, @@ -279,7 +280,7 @@ where header: GeneralHeader::new(0.into(), PageType::SpaceInfo, 0.into()), inner: info, }; - persist_page(&mut page, file).await + Ok(persist_page::<_, PAGE_SIZE>(&mut page, file).await?) } async fn save_data(&mut self, link: Link, bytes: &[u8]) -> eyre::Result<()> { @@ -294,7 +295,7 @@ where data: [0; 1], }, }; - persist_page(&mut page, &mut self.data_file).await?; + persist_page::<_, PAGE_SIZE>(&mut page, &mut self.data_file).await?; self.current_data_length = 0; // High-water mark, as in the batch path below: the new page is the // one the link names, which can be more than one past the current @@ -320,8 +321,8 @@ where self.update_data_length().await?; } } - update_at::<{ PAGE_SIZE }>(&mut self.data_file, link, bytes).await?; - // `update_at` ends with a buffered `write_all` that `tokio::fs::File` + update_at::<{ PAGE_SIZE }, PAGE_SIZE>(&mut self.data_file, link, bytes).await?; + // `update_at` ends with a `write_all` that the file behind `fsx` // completes on a background blocking task. Flush before reporting the // save done so the bytes are visible to any other handle. self.data_file.flush().await?; @@ -368,7 +369,7 @@ where }) .collect::>(); let parsed_pages = - parse_data_pages_batch::(&mut self.data_file, ids_to_parse).await?; + parse_data_pages_batch::(&mut self.data_file, ids_to_parse).await?; let updated_pages = vec![parsed_pages, created_pages] .into_iter() @@ -397,7 +398,7 @@ where self.current_data_length = page.inner.length; } - persist_pages_batch(updated_pages, &mut self.data_file).await?; + persist_pages_batch::<_, PAGE_SIZE>(updated_pages, &mut self.data_file).await?; // The batch's last page write is a buffered `write_all`; flush so the // batch is visible to other handles once it reports done. self.data_file.flush().await?; @@ -447,7 +448,7 @@ where // Single choke point for the info page reaching disk: enforce the // page-0 slot budget however the free-range list was mutated. self.bound_empty_links_list(); - persist_page(&mut self.info, &mut self.data_file).await?; + persist_page::<_, PAGE_SIZE>(&mut self.info, &mut self.data_file).await?; // A generated table may immediately reopen this file through a // separate handle. Make the updated metadata visible before reporting // success, just as `save_data` does for row bytes. diff --git a/src/persistence/space/index/mod.rs b/src/persistence/space/index/mod.rs index d4cbc625..284ecb4d 100644 --- a/src/persistence/space/index/mod.rs +++ b/src/persistence/space/index/mod.rs @@ -1,16 +1,18 @@ +use alloc::{string::String, string::ToString, vec::Vec}; mod page_aliases; mod reconstruct; mod table_of_contents; mod unsized_; mod util; -use std::collections::HashMap; -use std::fmt::Debug; -use std::hash::Hash; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::hash::Hash; +use core::sync::atomic::{AtomicU32, Ordering}; +use hashbrown::HashMap; use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use crate::fsx::File; use convert_case::{Case, Casing}; use data_bucket::page::{IndexValue, PageId}; use data_bucket::{ @@ -22,6 +24,7 @@ use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; use indexset::concurrent::multimap::BTreeMultiMap; use indexset::core::pair::Pair; +use nagoya::io::Write as _; use rkyv::de::Pool; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -29,8 +32,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; use crate::persistence::SpaceIndexOps; use crate::persistence::space::{BatchChangeEvent, open_or_create_file}; @@ -42,16 +43,16 @@ pub use unsized_::SpaceIndexUnsized; pub use util::{map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general}; #[derive(Debug)] -pub struct SpaceIndex { +pub struct SpaceIndex { space_id: SpaceId, - table_of_contents: IndexTableOfContents<(T, Link), INNER_PAGE_SIZE>, + table_of_contents: IndexTableOfContents<(T, Link), INNER_PAGE_SIZE, STRIDE>, next_page_id: Arc, index_file: File, #[allow(dead_code)] info: GeneralPage>, } -impl SpaceIndex +impl SpaceIndex where T: Archive + Ord @@ -110,9 +111,9 @@ where } else { open_or_create_file(index_file_path).await? }; - let info = parse_page::<_, INNER_PAGE_SIZE>(&mut index_file, 0).await?; + let info = parse_page::<_, INNER_PAGE_SIZE, STRIDE>(&mut index_file, 0).await?; - let file_length = index_file.metadata().await?.len(); + let file_length = crate::fsx::file_metadata(&mut index_file).await?; let page_id = if file_length % (INNER_PAGE_SIZE as u64 + GENERAL_HEADER_SIZE as u64) == 0 { file_length / (INNER_PAGE_SIZE as u64 + GENERAL_HEADER_SIZE as u64) } else { @@ -146,7 +147,7 @@ where async fn add_index_page(&mut self, node: IndexPage, page_id: PageId) -> eyre::Result<()> { let header = GeneralHeader::new(page_id, PageType::Index, self.space_id); let mut general_page = GeneralPage { inner: node, header }; - persist_page(&mut general_page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut general_page, &mut self.index_file).await?; Ok(()) } @@ -160,7 +161,7 @@ where let mut new_node_id = None; let size = get_index_page_size_from_data_length::(INNER_PAGE_SIZE as usize); - let mut utility = IndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + let mut utility = IndexPage::::parse_index_page_utility::(&mut self.index_file, page_id).await?; utility.slots.insert(index, utility.current_index); utility.slots.remove(size); utility.current_length += 1; @@ -168,16 +169,21 @@ where key: value.key.clone(), link: value.value, }; - utility.current_index = - IndexPage::::persist_value(&mut self.index_file, page_id, size, index_value, utility.current_index) - .await?; + utility.current_index = IndexPage::::persist_value::( + &mut self.index_file, + page_id, + size, + index_value, + utility.current_index, + ) + .await?; if node_id.key < value.key { utility.node_id = value.clone().into(); new_node_id = Some(value); } - IndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + IndexPage::::persist_index_page_utility::(&mut self.index_file, page_id, utility).await?; Ok(new_node_id) } @@ -192,7 +198,7 @@ where let mut new_node_id = None; let size = get_index_page_size_from_data_length::(INNER_PAGE_SIZE as usize); - let mut utility = IndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + let mut utility = IndexPage::::parse_index_page_utility::(&mut self.index_file, page_id).await?; let value_position = *utility .slots .get(index) @@ -203,7 +209,7 @@ where utility.slots.remove(index); utility.slots.push(0); utility.current_length -= 1; - IndexPage::::remove_value(&mut self.index_file, page_id, size, utility.current_index).await?; + IndexPage::::remove_value::(&mut self.index_file, page_id, size, utility.current_index).await?; if node_id.key == value.key { let index = *utility @@ -211,11 +217,12 @@ where .get(index - 1) .expect("slots always should exist in `size` bounds"); utility.node_id = - IndexPage::::read_value_with_index(&mut self.index_file, page_id, size, index as usize).await?; + IndexPage::::read_value_with_index::(&mut self.index_file, page_id, size, index as usize) + .await?; new_node_id = Some(utility.node_id.clone().into()) } - IndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + IndexPage::::persist_index_page_utility::(&mut self.index_file, page_id, utility).await?; Ok(new_node_id) } @@ -292,7 +299,8 @@ where .table_of_contents .get(&(node_id.key.clone(), node_id.value)) .ok_or(eyre!("Node with {:?} id is not found", node_id))?; - let mut page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, page_id.into()).await?; + let mut page = + parse_page::, INNER_PAGE_SIZE, STRIDE>(&mut self.index_file, page_id.into()).await?; let splitted_page = page.inner.split(split_index); let new_page_id = if let Some(id) = self.table_of_contents.pop_empty_page_id() { id @@ -316,7 +324,7 @@ where // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. self.add_index_page(splitted_page, new_page_id).await?; - persist_page(&mut page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut page, &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; Ok(()) @@ -327,7 +335,8 @@ where let indexset = BTreeMap::::with_maximum_node_size(size); let mut nodes = Vec::with_capacity(self.table_of_contents.iter().count()); for (_, page_id) in self.table_of_contents.iter() { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, (*page_id).into()).await?; + let page = + parse_page::, INNER_PAGE_SIZE, STRIDE>(&mut self.index_file, (*page_id).into()).await?; nodes.push(page.inner.get_node()); } indexset.attach_nodes(nodes); @@ -343,7 +352,8 @@ where let indexset = BTreeMultiMap::::with_maximum_node_size(size); let mut pages = Vec::with_capacity(self.table_of_contents.iter().count()); for ((key, link), page_id) in self.table_of_contents.iter() { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, (*page_id).into()).await?; + let page = + parse_page::, INNER_PAGE_SIZE, STRIDE>(&mut self.index_file, (*page_id).into()).await?; pages.push(( Pair { key: key.clone(), @@ -357,7 +367,7 @@ where } } -impl SpaceIndexOps for SpaceIndex +impl SpaceIndexOps for SpaceIndex where T: Archive + Ord @@ -410,7 +420,7 @@ where header: GeneralHeader::new(0.into(), PageType::SpaceInfo, 0.into()), inner: info, }; - persist_page(&mut page, file).await + Ok(persist_page::<_, STRIDE>(&mut page, file).await?) } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -442,7 +452,7 @@ where } => self.process_split_node(node_id, split_index).await, }?; // The partial page writes above can end with a buffered `write_all` - // that `tokio::fs::File` completes on a background blocking task. + // that the file behind `fsx` completes on the calling thread. // Flush before reporting the event processed so the bytes are visible // to any other handle that opens this file afterwards. self.index_file.flush().await?; @@ -482,8 +492,11 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, page_index.into()) - .await?; + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( + &mut self.index_file, + page_index.into(), + ) + .await?; pages.insert(page_index, page); pages .get_mut(&page_index) @@ -564,8 +577,11 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, page_index.into()) - .await?; + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( + &mut self.index_file, + page_index.into(), + ) + .await?; pages.insert(page_index, page); pages .get_mut(&page_index) @@ -628,7 +644,7 @@ where // authority (`parse_indexset` and the strict load audit iterate TOC // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. - persist_pages_batch(pages.values().cloned().collect(), &mut self.index_file).await?; + persist_pages_batch::<_, STRIDE>(pages.values().cloned().collect(), &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; // The batch's last write is buffered; flush so the batch is visible // to other handles once it reports done. @@ -639,6 +655,7 @@ where #[cfg(test)] mod test { + use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::{INNER_PAGE_SIZE, IndexPage, IndexValue, Persistable, get_index_page_size_from_data_length}; use super::*; @@ -651,7 +668,7 @@ mod test { #[tokio::test] async fn orphan_page_from_crash_between_page_and_toc_write_is_ignored_on_reload() { use indexset::cdc::change::ChangeEvent; - use tokio::io::AsyncWriteExt; + use nagoya::io::Write as _; let dir = std::env::temp_dir().join(format!("wt_orphan_crash_{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); @@ -669,7 +686,7 @@ mod test { }; { - let mut index = SpaceIndex::::new(&path, 0.into(), 1) + let mut index = SpaceIndex::::new(&path, 0.into(), 1) .await .unwrap(); index @@ -687,7 +704,7 @@ mod test { index.index_file.flush().await.unwrap(); } - let mut reloaded = SpaceIndex::::new(&path, 0.into(), 1) + let mut reloaded = SpaceIndex::::new(&path, 0.into(), 1) .await .unwrap(); let restored = reloaded.parse_indexset().await.unwrap(); diff --git a/src/persistence/space/index/page_aliases.rs b/src/persistence/space/index/page_aliases.rs index 131dd05d..627588e4 100644 --- a/src/persistence/space/index/page_aliases.rs +++ b/src/persistence/space/index/page_aliases.rs @@ -5,6 +5,7 @@ //! when a split or a max-remove re-keys a page mid-batch while later events //! still name a historical maximum. +use alloc::vec::Vec; use data_bucket::Link; use data_bucket::page::PageId; use eyre::eyre; @@ -35,7 +36,7 @@ pub(super) struct PageAliasEntry { impl Default for PageAliases { fn default() -> Self { Self { - inline: std::array::from_fn(|_| None), + inline: core::array::from_fn(|_| None), overflow: Vec::new(), } } diff --git a/src/persistence/space/index/reconstruct.rs b/src/persistence/space/index/reconstruct.rs index dab116ca..64ca1c13 100644 --- a/src/persistence/space/index/reconstruct.rs +++ b/src/persistence/space/index/reconstruct.rs @@ -4,7 +4,8 @@ //! generic function that can be unit-tested with synthetic pages; the proc //! macro only generates type plumbing and node attachment. -use std::fmt::Debug; +use alloc::vec::Vec; +use core::fmt::Debug; use indexset::core::multipair::MultiPair; use indexset::core::pair::Pair; @@ -173,7 +174,7 @@ mod tests { for b in nodes.iter().skip(i + 1) { assert_ne!( a.last().unwrap().cmp(b.last().unwrap()), - std::cmp::Ordering::Equal, + core::cmp::Ordering::Equal, "two node maxima compare Equal: {:?} vs {:?}", a.last().unwrap(), b.last().unwrap() @@ -247,7 +248,7 @@ mod tests { assert_eq!(flatten(&nodes[..1]), vec![(1, 1), (1, 2), (2, 30)]); // Every stored entry is distinct. That is what the discriminator counter was // for; identity is now the `(key, value)` pair itself. - let mut seen = std::collections::BTreeSet::new(); + let mut seen = alloc::collections::BTreeSet::new(); for p in nodes.iter().flatten() { assert!(seen.insert((p.key, p.value)), "duplicate entry {:?}", (p.key, p.value)); } diff --git a/src/persistence/space/index/table_of_contents.rs b/src/persistence/space/index/table_of_contents.rs index 633c1cf5..f6a594a9 100644 --- a/src/persistence/space/index/table_of_contents.rs +++ b/src/persistence/space/index/table_of_contents.rs @@ -1,7 +1,9 @@ -use std::fmt::Debug; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::sync::atomic::{AtomicU32, Ordering}; +use crate::fsx::File; use data_bucket::page::PageId; use data_bucket::{ GeneralHeader, GeneralPage, PageType, SizeMeasurable, SpaceId, TableOfContentsPage, parse_page, persist_page, @@ -13,7 +15,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; /// A table-of-contents entry whose serialized size can never fit a segment. /// @@ -26,8 +27,8 @@ pub struct TocEntryOversizedError { pub segment_capacity: usize, } -impl std::fmt::Display for TocEntryOversizedError { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl core::fmt::Display for TocEntryOversizedError { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( formatter, "table-of-contents entry needs {} bytes but a whole empty segment holds only {}", @@ -36,16 +37,16 @@ impl std::fmt::Display for TocEntryOversizedError { } } -impl std::error::Error for TocEntryOversizedError {} +impl core::error::Error for TocEntryOversizedError {} #[derive(Debug)] -pub struct IndexTableOfContents { +pub struct IndexTableOfContents { current_page: usize, next_page_id: Arc, pub pages: Vec>>, } -impl IndexTableOfContents +impl IndexTableOfContents where T: Debug + SizeMeasurable + Ord + Eq, { @@ -248,7 +249,7 @@ where + for<'a> rkyv::bytecheck::CheckBytes>, { for page in &mut self.pages { - persist_page(page, file).await?; + persist_page::<_, STRIDE>(page, file).await?; } Ok(()) @@ -265,7 +266,7 @@ where + Eq + for<'a> rkyv::bytecheck::CheckBytes>, { - let first_page = parse_page::, DATA_LENGTH>(file, 1).await; + let first_page = parse_page::, DATA_LENGTH, STRIDE>(file, 1).await; let page = match first_page { Ok(page) => page, Err(error) => { @@ -275,11 +276,14 @@ where // file extends into page 1's slot, the parse failure means a // torn or truncated table of contents, and silently starting // empty would discard the whole index. - let file_length = file.metadata().await?.len(); + let file_length = crate::fsx::file_metadata(file).await?; if file_length <= data_bucket::PAGE_SIZE as u64 { return Ok(Self::new(space_id, next_page_id)); } - return Err(error.wrap_err(format!( + // `wrap_err` belonged to `eyre::Report`. The parse error is a + // concrete `data_bucket::error::Error` now, so it becomes a report + // first and keeps the same context message. + return Err(eyre::Report::new(error).wrap_err(format!( "table of contents page 1 failed to parse in a {file_length}-byte index file that should contain it" ))); } @@ -297,7 +301,7 @@ where let mut ind = false; while !ind { - let page = parse_page::, DATA_LENGTH>(file, index).await?; + let page = parse_page::, DATA_LENGTH, STRIDE>(file, index).await?; ind = page.header.next_id.is_empty(); index = page.header.next_id.into(); table_of_contents_pages.push(page); @@ -316,13 +320,14 @@ where #[cfg(test)] mod tests { use crate::persistence::space::index::table_of_contents::IndexTableOfContents; + use alloc::sync::Arc; + use core::sync::atomic::AtomicU32; + use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::page::PageId; - use std::sync::Arc; - use std::sync::atomic::AtomicU32; #[test] fn empty() { - let toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); assert_eq!( toc.current_page, 0, "`current_page` is not set to 0, it is {}", @@ -333,7 +338,7 @@ mod tests { #[test] fn insert_to_empty() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); let key = 1; toc.insert(key, 1.into()); @@ -352,7 +357,7 @@ mod tests { #[test] fn checked_update_reports_a_missing_identity_without_mutating_the_toc() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); toc.insert(7, 2.into()); assert!(!toc.try_update_key(&8, 9).unwrap()); @@ -363,7 +368,10 @@ mod tests { #[test] fn growing_key_update_moves_the_entry_instead_of_overflowing_the_segment() { const DATA_LENGTH: u32 = 128; - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new( + 0.into(), + Arc::new(AtomicU32::new(1)), + ); // Fill the first segment close to capacity with short keys. let mut key = 0; @@ -395,7 +403,10 @@ mod tests { use crate::persistence::TocEntryOversizedError; const DATA_LENGTH: u32 = 128; - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new( + 0.into(), + Arc::new(AtomicU32::new(1)), + ); toc.insert("small".to_string(), PageId::from(9)); let oversized = "x".repeat(4 * DATA_LENGTH as usize); @@ -411,7 +422,7 @@ mod tests { #[test] fn insert_more_than_one_page() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); let mut keys = vec![]; for key in 0..10 { toc.insert(key, 1.into()); @@ -437,7 +448,7 @@ mod tests { #[test] fn insert_reaches_existing_tail_after_reload_resets_cursor() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); for key in 0..10 { toc.insert(key, u32::from(key).into()); } @@ -465,7 +476,7 @@ mod tests { #[test] fn insert_reports_a_truncated_segment_chain() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); for key in 0..10 { toc.insert(key, u32::from(key).into()); } @@ -480,7 +491,7 @@ mod tests { #[test] fn reinsert_on_empty_space() { - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(0))); let mut keys = vec![]; for key in 0..10 { toc.insert(key, 1.into()); diff --git a/src/persistence/space/index/unsized_.rs b/src/persistence/space/index/unsized_.rs index dd7cdad0..babafe6b 100644 --- a/src/persistence/space/index/unsized_.rs +++ b/src/persistence/space/index/unsized_.rs @@ -1,9 +1,11 @@ -use std::collections::HashMap; -use std::fmt::Debug; -use std::hash::Hash; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; - +use alloc::sync::Arc; +use alloc::{string::String, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::sync::atomic::{AtomicU32, Ordering}; +use hashbrown::HashMap; + +use crate::fsx::File; use data_bucket::page::PageId; use data_bucket::{ GeneralHeader, GeneralPage, IndexPageUtility, IndexValue, Link, PageType, SizeMeasurable, SpaceId, SpaceInfoPage, @@ -14,6 +16,7 @@ use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; use indexset::concurrent::multimap::BTreeMultiMap; use indexset::core::pair::Pair; +use nagoya::io::Write as _; use rkyv::de::Pool; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; @@ -21,8 +24,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; use super::page_aliases::PageAliases; use crate::UnsizedNode; @@ -31,16 +32,16 @@ use crate::persistence::{IndexTableOfContents, SpaceIndex, SpaceIndexOps, recons use crate::prelude::WT_INDEX_EXTENSION; #[derive(Debug)] -pub struct SpaceIndexUnsized { +pub struct SpaceIndexUnsized { space_id: SpaceId, - table_of_contents: IndexTableOfContents<(T, Link), DATA_LENGTH>, + table_of_contents: IndexTableOfContents<(T, Link), DATA_LENGTH, STRIDE>, next_page_id: Arc, index_file: File, #[allow(dead_code)] info: GeneralPage>, } -impl SpaceIndexUnsized +impl SpaceIndexUnsized where T: Archive + Ord @@ -103,7 +104,7 @@ where } pub async fn new>(index_file_path: S, space_id: SpaceId, version: u32) -> eyre::Result { - let space_index = SpaceIndex::::new(index_file_path, space_id, version).await?; + let space_index = SpaceIndex::::new(index_file_path, space_id, version).await?; Ok(Self { space_id, table_of_contents: space_index.table_of_contents, @@ -126,7 +127,7 @@ where // happened to create the page. let header = GeneralHeader::new(page_id, PageType::IndexUnsized, self.space_id); let mut general_page = GeneralPage { inner: node, header }; - persist_page(&mut general_page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut general_page, &mut self.index_file).await?; Ok(()) } @@ -187,7 +188,8 @@ where let mut new_node_id = None; let mut utility = - UnsizedIndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + UnsizedIndexPage::::parse_index_page_utility::(&mut self.index_file, page_id) + .await?; let index_value = IndexValue { key: value.key.clone(), link: value.value, @@ -201,9 +203,11 @@ where let future_utility_size = UnsizedIndexPageUtility::::persisted_size(utility.slots_size as usize + 1, future_node_id_size); if future_utility_size + utility.last_value_offset as usize + value_size > DATA_LENGTH as usize { - let mut page = - parse_page::, DATA_LENGTH>(&mut self.index_file, page_id.into()) - .await?; + let mut page = parse_page::, DATA_LENGTH, STRIDE>( + &mut self.index_file, + page_id.into(), + ) + .await?; page.inner.apply_change_event(ChangeEvent::InsertAt { // The page mutation ignores event ids; this synthetic event // exists only to reuse the same insertion accounting. @@ -215,11 +219,11 @@ where Self::compact_page_if_needed(&mut page.inner)?; let changed_node_id = (page.inner.node_id.key != utility.node_id.key).then(|| Pair::from(page.inner.node_id.clone())); - persist_page(&mut page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut page, &mut self.index_file).await?; return Ok(changed_node_id); } let previous_offset = utility.last_value_offset; - let value_offset = UnsizedIndexPage::::persist_value( + let value_offset = UnsizedIndexPage::::persist_value::( &mut self.index_file, page_id, previous_offset, @@ -237,7 +241,12 @@ where new_node_id = Some(value); } - UnsizedIndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + UnsizedIndexPage::::persist_index_page_utility::( + &mut self.index_file, + page_id, + utility, + ) + .await?; Ok(new_node_id) } @@ -273,7 +282,8 @@ where let mut new_node_id = None; let mut utility = - UnsizedIndexPage::::parse_index_page_utility(&mut self.index_file, page_id).await?; + UnsizedIndexPage::::parse_index_page_utility::(&mut self.index_file, page_id) + .await?; utility.slots.remove(index); utility.slots_size -= 1; @@ -282,14 +292,23 @@ where .slots .get(index - 1) .expect("slots always should exist in `size` bounds"); - let node_id = - UnsizedIndexPage::::read_value_with_offset(&mut self.index_file, page_id, offset, len) - .await?; + let node_id = UnsizedIndexPage::::read_value_with_offset::( + &mut self.index_file, + page_id, + offset, + len, + ) + .await?; utility.update_node_id(node_id)?; new_node_id = Some(utility.node_id.clone().into()) } - UnsizedIndexPage::::persist_index_page_utility(&mut self.index_file, page_id, utility).await?; + UnsizedIndexPage::::persist_index_page_utility::( + &mut self.index_file, + page_id, + utility, + ) + .await?; Ok(new_node_id) } @@ -300,7 +319,8 @@ where .get(&(node_id.key.clone(), node_id.value)) .ok_or(eyre!("Node with {:?} id is not found", node_id))?; let mut page = - parse_page::, DATA_LENGTH>(&mut self.index_file, page_id.into()).await?; + parse_page::, DATA_LENGTH, STRIDE>(&mut self.index_file, page_id.into()) + .await?; let splitted_page = page.inner.split(split_index); let new_page_id = if let Some(id) = self.table_of_contents.pop_empty_page_id() { id @@ -324,7 +344,7 @@ where // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. self.add_index_page(splitted_page, new_page_id).await?; - persist_page(&mut page, &mut self.index_file).await?; + persist_page::<_, STRIDE>(&mut page, &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; Ok(()) @@ -334,9 +354,11 @@ where let indexset = BTreeMap::>>::with_maximum_node_size(DATA_LENGTH as usize); let mut nodes = Vec::with_capacity(self.table_of_contents.iter().count()); for (_, page_id) in self.table_of_contents.iter() { - let page = - parse_page::, DATA_LENGTH>(&mut self.index_file, (*page_id).into()) - .await?; + let page = parse_page::, DATA_LENGTH, STRIDE>( + &mut self.index_file, + (*page_id).into(), + ) + .await?; let node = page.inner.get_node(); nodes.push(UnsizedNode::from_inner(node, DATA_LENGTH as usize)); } @@ -353,9 +375,11 @@ where let indexset = BTreeMultiMap::>::with_maximum_node_size(DATA_LENGTH as usize); let mut pages = Vec::with_capacity(self.table_of_contents.iter().count()); for ((key, link), page_id) in self.table_of_contents.iter() { - let page = - parse_page::, DATA_LENGTH>(&mut self.index_file, (*page_id).into()) - .await?; + let page = parse_page::, DATA_LENGTH, STRIDE>( + &mut self.index_file, + (*page_id).into(), + ) + .await?; pages.push(( Pair { key: key.clone(), @@ -373,7 +397,8 @@ where } } -impl SpaceIndexOps for SpaceIndexUnsized +impl SpaceIndexOps + for SpaceIndexUnsized where T: Archive + Ord @@ -412,7 +437,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndex::::bootstrap(file, table_name, version).await + SpaceIndex::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -444,7 +469,7 @@ where } => self.process_split_node(node_id, split_index).await, }?; // The partial page writes above can end with a buffered `write_all` - // that `tokio::fs::File` completes on a background blocking task. + // that the file behind `fsx` completes on the calling thread. // Flush before reporting the event processed so the bytes are visible // to any other handle that opens this file afterwards. self.index_file.flush().await?; @@ -484,7 +509,7 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>( + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( &mut self.index_file, page_index.into(), ) @@ -562,7 +587,7 @@ where let page_to_update = if let Some(page) = page { page } else { - let page = parse_page::, INNER_PAGE_SIZE>( + let page = parse_page::, INNER_PAGE_SIZE, STRIDE>( &mut self.index_file, page_index.into(), ) @@ -633,7 +658,7 @@ where // authority (`parse_indexset` and the strict load audit iterate TOC // entries only). The reverse order left a durable TOC entry pointing // at absent or stale bytes. - persist_pages_batch(pages.values().cloned().collect(), &mut self.index_file).await?; + persist_pages_batch::<_, STRIDE>(pages.values().cloned().collect(), &mut self.index_file).await?; self.table_of_contents.persist(&mut self.index_file).await?; // The batch's last write is buffered; flush so the batch is visible // to other handles once it reports done. diff --git a/src/persistence/space/index/util.rs b/src/persistence/space/index/util.rs index cde98019..b45fb700 100644 --- a/src/persistence/space/index/util.rs +++ b/src/persistence/space/index/util.rs @@ -1,16 +1,17 @@ use crate::prelude::IndexTableOfContents; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::sync::atomic::{AtomicU32, Ordering}; use data_bucket::{ GeneralHeader, GeneralPage, IndexPage, Link, PageType, SizeMeasurable, UnsizedIndexPage, VariableSizeMeasurable, }; -use std::fmt::Debug; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; #[allow(clippy::type_complexity)] -pub fn map_index_pages_to_toc_and_general( +pub fn map_index_pages_to_toc_and_general( pages: Vec>, ) -> ( - IndexTableOfContents<(T, Link), DATA_LENGTH>, + IndexTableOfContents<(T, Link), DATA_LENGTH, STRIDE>, Vec>>, ) where @@ -31,10 +32,10 @@ where } #[allow(clippy::type_complexity)] -pub fn map_unsized_index_pages_to_toc_and_general( +pub fn map_unsized_index_pages_to_toc_and_general( pages: Vec>, ) -> ( - IndexTableOfContents<(T, Link), DATA_LENGTH>, + IndexTableOfContents<(T, Link), DATA_LENGTH, STRIDE>, Vec>>, ) where diff --git a/src/persistence/space/logical_index.rs b/src/persistence/space/logical_index.rs index e2e948fc..7c3faa75 100644 --- a/src/persistence/space/logical_index.rs +++ b/src/persistence/space/logical_index.rs @@ -5,10 +5,12 @@ //! this persistence-worker-owned index derives the structural events required //! by the unchanged WTI disk format. -use std::fmt::Debug; -use std::hash::Hash; +use alloc::{string::String, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; use std::path::{Path, PathBuf}; +use crate::fsx::File; use data_bucket::{Link, SizeMeasurable, SpaceId, VariableSizeMeasurable}; use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; @@ -23,7 +25,6 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize, rancor}; -use tokio::fs::File; use crate::UnsizedNode; use crate::convert_multi_change_events; @@ -158,25 +159,25 @@ where /// Sized-key WTI persistence with foreground logical CDC and a background /// structural shadow. The wrapped `SpaceIndex` retains the existing file /// layout byte-for-byte. -pub struct SpaceLogicalIndex +pub struct SpaceLogicalIndex where T: Send + Ord + Eq + Clone + 'static, { index_path: PathBuf, shadow: BTreeMap, - disk: SpaceIndex, + disk: SpaceIndex, } -impl Debug for SpaceLogicalIndex +impl Debug for SpaceLogicalIndex where T: Send + Ord + Eq + Clone + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter.debug_struct("SpaceLogicalIndex").finish_non_exhaustive() } } -impl SpaceLogicalIndex +impl SpaceLogicalIndex where T: Archive + Ord @@ -208,7 +209,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalIndex +impl SpaceIndexOps + for SpaceLogicalIndex where T: Archive + Ord @@ -245,7 +247,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndex::::bootstrap(file, table_name, version).await + SpaceIndex::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -260,27 +262,27 @@ where } /// Variable-sized-key counterpart to [`SpaceLogicalIndex`]. -pub struct SpaceLogicalIndexUnsized +pub struct SpaceLogicalIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { index_path: PathBuf, shadow: BTreeMap>>, - disk: SpaceIndexUnsized, + disk: SpaceIndexUnsized, } -impl Debug for SpaceLogicalIndexUnsized +impl Debug for SpaceLogicalIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_struct("SpaceLogicalIndexUnsized") .finish_non_exhaustive() } } -impl SpaceLogicalIndexUnsized +impl SpaceLogicalIndexUnsized where T: Archive + Ord @@ -313,7 +315,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalIndexUnsized +impl SpaceIndexOps + for SpaceLogicalIndexUnsized where T: Archive + Ord @@ -351,7 +354,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndexUnsized::::bootstrap(file, table_name, version).await + SpaceIndexUnsized::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -368,25 +371,25 @@ where /// Sized-key WTI persistence for a non-unique runtime backend that emits /// logical `(key, link)` mutations. The WTI file layout and its node topology /// remain compatible with earlier WorkTable releases. -pub struct SpaceLogicalMultiIndex +pub struct SpaceLogicalMultiIndex where T: Debug + Send + Ord + Eq + Clone + 'static, { index_path: PathBuf, shadow: BTreeMultiMap, - disk: SpaceIndex, + disk: SpaceIndex, } -impl Debug for SpaceLogicalMultiIndex +impl Debug for SpaceLogicalMultiIndex where T: Debug + Send + Ord + Eq + Clone + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter.debug_struct("SpaceLogicalMultiIndex").finish_non_exhaustive() } } -impl SpaceLogicalMultiIndex +impl SpaceLogicalMultiIndex where T: Archive + Ord @@ -418,7 +421,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalMultiIndex +impl SpaceIndexOps + for SpaceLogicalMultiIndex where T: Archive + Ord @@ -455,7 +459,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndex::::bootstrap(file, table_name, version).await + SpaceIndex::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -470,27 +474,28 @@ where } /// Variable-sized-key counterpart to [`SpaceLogicalMultiIndex`]. -pub struct SpaceLogicalMultiIndexUnsized +pub struct SpaceLogicalMultiIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { index_path: PathBuf, shadow: BTreeMultiMap>>, - disk: SpaceIndexUnsized, + disk: SpaceIndexUnsized, } -impl Debug for SpaceLogicalMultiIndexUnsized +impl Debug + for SpaceLogicalMultiIndexUnsized where T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_struct("SpaceLogicalMultiIndexUnsized") .finish_non_exhaustive() } } -impl SpaceLogicalMultiIndexUnsized +impl SpaceLogicalMultiIndexUnsized where T: Archive + Ord @@ -523,7 +528,8 @@ where } } -impl SpaceIndexOps for SpaceLogicalMultiIndexUnsized +impl SpaceIndexOps + for SpaceLogicalMultiIndexUnsized where T: Archive + Ord @@ -561,7 +567,7 @@ where } async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { - SpaceIndexUnsized::::bootstrap(file, table_name, version).await + SpaceIndexUnsized::::bootstrap(file, table_name, version).await } async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { @@ -577,7 +583,7 @@ where #[cfg(test)] mod tests { - use std::collections::BTreeMap as StdBTreeMap; + use alloc::collections::BTreeMap as StdBTreeMap; use data_bucket::page::PageId; diff --git a/src/persistence/space/mod.rs b/src/persistence/space/mod.rs index 4f9d280b..35bb6f62 100644 --- a/src/persistence/space/mod.rs +++ b/src/persistence/space/mod.rs @@ -1,17 +1,18 @@ +use alloc::{string::String, vec::Vec}; mod art_index; mod data; mod index; mod logical_index; -use std::collections::HashMap; -use std::future::Future; +use core::future::Future; +use hashbrown::HashMap; use std::path::Path; +use crate::fsx::File; use data_bucket::page::PageId; use data_bucket::{GeneralPage, Link, SpaceInfoPage}; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; -use tokio::fs::{File, OpenOptions}; pub use art_index::{ ArtPersistenceKey, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, @@ -88,10 +89,9 @@ pub trait SpaceSecondaryIndexOps { pub async fn open_or_create_file>(path: S) -> eyre::Result { let path = Path::new(path.as_ref()); - Ok(OpenOptions::new() - .write(true) - .read(true) - .create(!path.exists()) - .open(path) - .await?) + Ok(if path.exists() { + crate::fsx::open(path).await? + } else { + crate::fsx::open_or_create(path).await? + }) } diff --git a/src/persistence/task.rs b/src/persistence/task.rs index fc67f09f..4aa521d7 100644 --- a/src/persistence/task.rs +++ b/src/persistence/task.rs @@ -1,10 +1,14 @@ -use std::collections::{HashMap, HashSet, VecDeque}; -use std::fmt::Debug; -use std::hash::Hash; -use std::marker::PhantomData; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::time::Duration; +use alloc::boxed::Box; +use alloc::collections::VecDeque; +use alloc::sync::Arc; +use alloc::{borrow::ToOwned, string::String, string::ToString, vec::Vec}; +use core::fmt::Debug; +use core::hash::Hash; +use core::marker::PhantomData; +use core::panic::Location; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use core::time::Duration; +use hashbrown::{HashMap, HashSet}; use data_bucket::page::PageId; use parking_lot::Mutex as ParkingMutex; @@ -12,7 +16,8 @@ use tokio::sync::Notify; use tokio::task::JoinHandle; use worktable_codegen::worktable; -use crate::persistence::operation::{BatchInnerRow, BatchInnerWorkTable, BatchOperation, OperationId}; +use crate::persistence::event_ledger::{self, EventLedger, EventStream, Stages}; +use crate::persistence::operation::{BatchInnerRow, BatchInnerWorkTable, BatchOperation, OperationId, OperationType}; use crate::persistence::{ PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceResult, PersistenceState, }; @@ -207,6 +212,12 @@ pub struct QueueAnalyzer, } #[derive(Debug)] @@ -260,9 +271,19 @@ where page_limit: MAX_PAGE_AMOUNT, attempts: 0, no_progress: 0, + event_ledger: Arc::new(EventLedger::detached()), } } + /// Shares the feeding queue's event bookkeeping with this analyzer. + /// + /// Only the producer side records who pushed an event, so the analyzer has + /// to read the *same* ledger the queue writes for its gap reports to mean + /// anything. + pub fn attach_event_ledger(&mut self, ledger: Arc) { + self.event_ledger = ledger; + } + pub fn push(&mut self, value: Operation) -> eyre::Result<()> { let link = value.link(); let mut row = QueueInnerRow { @@ -301,6 +322,39 @@ where .map(|(id, _)| id) } + /// Records `stage` against every event id carried by `ops`. + /// + /// Diagnostics only. The whole body is skipped when bookkeeping is off, + /// which keeps the `Debug` formatting of secondary index labels off the + /// path of a release build entirely. + fn record_ops_stage(&self, ops: &[Operation], stage: Stages) + where + SecondaryKeys: TableSecondaryIndexEventsOps, + { + if !event_ledger::enabled() { + return; + } + for op in ops { + if let Some(evs) = op.primary_key_events() { + self.event_ledger + .record_stage_for_events(EventStream::Primary, evs, stage); + } + // `Stages::QUEUED` is unioned in for secondary streams because the + // producer side records primary ids only: an operation reaching + // the analyzer at all proves it was queued, and without this the + // secondary gap report would call every id it knows about leaked. + // The cost is that a secondary record carries no producer call + // site, which the report prints as ``. + for (index, id) in op.secondary_key_events().iter_event_ids() { + self.event_ledger.record_stage( + EventStream::Secondary(format!("{index:?}")), + id.inner(), + stage.union(Stages::QUEUED), + ); + } + } + } + pub async fn collect_batch_from_op_id( &mut self, op_id: OperationId, @@ -440,13 +494,23 @@ where ops.push(op); } - let mut op = BatchOperation::new(ops, info_wt); + self.record_ops_stage(&ops, Stages::COLLECTED); + let mut op = BatchOperation::new(ops, info_wt).with_event_ledger(self.event_ledger.clone()); let invalid_for_this_batch_ops = op.validate(&self.last_events_ids, self.attempts).await?; if let Some(invalid_for_this_batch_ops) = invalid_for_this_batch_ops { + self.record_ops_stage(&invalid_for_this_batch_ops, Stages::REQUEUED); self.extend_from_iter(invalid_for_this_batch_ops.into_iter())?; let previous_primary = self.last_events_ids.primary_id; let last_ids = op.get_last_event_ids(); let advanced = last_ids.primary_id > previous_primary; + self.event_ledger + .record_applied_upto(EventStream::Primary, last_ids.primary_id.inner()); + if event_ledger::enabled() { + for (index, id) in &last_ids.secondary_ids { + self.event_ledger + .record_applied_upto(EventStream::Secondary(format!("{index:?}")), id.inner()); + } + } self.last_events_ids.merge(last_ids); self.last_invalid_batch_size = 0; self.page_limit = MAX_PAGE_AMOUNT; @@ -461,6 +525,7 @@ where } else { // can't collect batch for now let ops = op.ops(); + self.record_ops_stage(&ops, Stages::REQUEUED); self.attempts += 1; self.no_progress += 1; if self.last_invalid_batch_size == ops.len() { @@ -480,8 +545,8 @@ where #[cfg(test)] mod lifecycle_tests { - use std::collections::HashMap; - use std::sync::atomic::{AtomicUsize, Ordering}; + use core::sync::atomic::{AtomicUsize, Ordering}; + use hashbrown::HashMap; use super::*; @@ -508,7 +573,7 @@ mod lifecycle_tests { } fn iter_event_ids(&self) -> impl Iterator { - std::iter::empty() + core::iter::empty() } fn sort(&mut self) {} @@ -727,7 +792,7 @@ mod lifecycle_tests { .unwrap(); assert_eq!( - batch.get(&1.into()).unwrap(), + batch.get(&PageId::from(1u32)).unwrap(), &vec![ ( Link { @@ -791,7 +856,7 @@ mod lifecycle_tests { .get_batch_data_op() .unwrap(); - let page_one_writes = batch.get(&1.into()).unwrap(); + let page_one_writes = batch.get(&PageId::from(1u32)).unwrap(); assert_eq!( page_one_writes, &vec![ @@ -815,7 +880,7 @@ mod lifecycle_tests { "the complete earlier group must be applied" ); assert!( - !batch.contains_key(&2.into()), + !batch.contains_key(&PageId::from(2u32)), "the blocking group must stay queued, not be applied without its earlier events" ); assert_eq!(analyzer.len(), 2, "both rows of the blocked group remain queued"); @@ -918,7 +983,7 @@ mod lifecycle_tests { #[tokio::test] async fn wake_landing_inside_the_pop_race_window_is_not_lost() { let lifecycle = Arc::new(PersistenceLifecycle::new()); - let mut queue = Queue::<(), u64, TestEvents>::new(lifecycle.clone()); + let mut queue = Queue::<(), u64, TestEvents>::new(lifecycle.clone(), "tests/queue"); let gate = Arc::new(PopRaceWindowGate::new()); queue.pop_race_window_gate = Some(gate.clone()); let queue = Arc::new(queue); @@ -1148,6 +1213,41 @@ impl PopRaceWindowGate { } } +/// Primary index event ids lifted off an operation before it is moved into the +/// queue, so they can be recorded once the push is known to have been accepted. +/// +/// Primary only: `Queue` is generic over the secondary event type with no bound +/// that could iterate it, and the primary stream is the one whose gap guard +/// stalls the engine. The analyzer records secondary ids at collection time, +/// where that bound does exist. +/// +/// Cheap when bookkeeping is off: `EventLedger::event_ids` returns an empty +/// `Vec`, which allocates nothing, and the rest is two `Copy` field reads. +struct QueuedEventIds { + ids: Vec, + op_id: OperationId, + op_type: OperationType, +} + +impl QueuedEventIds { + fn of( + value: &Operation, + ) -> Self { + Self { + ids: value + .primary_key_events() + .map(|evs| EventLedger::event_ids(evs.as_slice())) + .unwrap_or_default(), + op_id: value.operation_id(), + op_type: value.operation_type(), + } + } + + fn record(&self, ledger: &EventLedger, site: &'static Location<'static>) { + ledger.record_queued(EventStream::Primary, &self.ids, self.op_id, self.op_type, site); + } +} + #[derive(Debug)] pub struct Queue { // Not `lockfree::queue::Queue`: its `Removable::empty` materializes the @@ -1161,38 +1261,71 @@ pub struct Queue { // queue that still holds work. len: Arc, lifecycle: Arc, + /// Producer-side half of the event-gap bookkeeping: every operation that + /// reaches persistence passes through this queue, so an event id the + /// engine is waiting for that never appears here was assigned by the index + /// and dropped before it was queued. Shared with the analyzer, which reads + /// it when the gap guard fires. Diagnostics only, and inert unless + /// [`event_ledger::enabled`]. + event_ledger: Arc, #[cfg(test)] - pop_race_window_gate: Option>, + pop_race_window_gate: Option>, } impl Queue { - fn new(lifecycle: Arc) -> Self { + fn new(lifecycle: Arc, table_path: &str) -> Self { Self { queue: ParkingMutex::new(VecDeque::new()), notify: Notify::new(), len: Arc::new(AtomicUsize::new(0)), lifecycle, + event_ledger: Arc::new(EventLedger::new(table_path)), #[cfg(test)] pop_race_window_gate: None, } } - pub fn push(&self, value: Operation) -> PersistenceResult { - self.push_message(PersistenceMessage::Operation(value)) + /// The event bookkeeping this queue writes, for sharing with the analyzer. + pub fn event_ledger(&self) -> Arc { + self.event_ledger.clone() + } + + /// Enqueues one operation, naming the producer's call site. + /// + /// The site is passed explicitly rather than taken with `#[track_caller]`, + /// because that only reaches one frame up: a wrapper that wants its own + /// caller named in a gap report has to forward a location through here. + /// through here rather than call `push` and lose it. + pub fn push_at( + &self, + value: Operation, + site: &'static Location<'static>, + ) -> PersistenceResult { + // The ids have to be lifted out before the operation is moved into the + // queue, but they are only recorded once the push is accepted: a + // refused push (the engine is closing or already failed) genuinely + // does not queue its events, and recording it as queued would hide + // exactly that leak mode. + let queued = QueuedEventIds::of(&value); + self.push_message(PersistenceMessage::Operation(value))?; + queued.record(&self.event_ledger, site); + Ok(()) } - /// Enqueues a whole batch of operations under one lifecycle check, one - /// queue lock acquisition and one worker wake-up, so callers producing - /// many operations at once (`insert_many`) pay the intake overhead once - /// instead of per row. All-or-nothing: either every operation is accepted - /// or none is. - pub fn push_many( + /// Enqueues a whole batch under one lifecycle check, one queue lock and one + /// worker wake-up, so a caller producing many operations at once + /// (`insert_many`) pays the intake overhead once instead of per row. + /// All-or-nothing: either every operation is accepted or none is. Takes the + /// producer's call site for the same reason as [`Queue::push_at`]. + pub fn push_many_at( &self, values: Vec>, + site: &'static Location<'static>, ) -> PersistenceResult { if values.is_empty() { return Ok(()); } + let queued = values.iter().map(QueuedEventIds::of).collect::>(); let state = self.lifecycle.state.lock(); match &*state { PersistenceState::Running => {} @@ -1205,6 +1338,11 @@ impl Queue>>, secondary_keys_events: SecondaryKeys, ) -> PersistenceResult { - self.push(Operation::Update(UpdateOperation { - id: OperationId::Single(uuid::Uuid::now_v7()), - primary_key_events, - secondary_keys_events, - bytes, - link: new_link, - })) + // `Location::caller()` without `#[track_caller]` resolves to this line + // rather than to vacuum's call site. That is deliberate: the trait + // declaration lives outside this module and cannot be annotated, and + // this line is already a unique producer label, because `apply_move` + // is only ever reached from a vacuum row move. + self.push_at( + Operation::Update(UpdateOperation { + id: OperationId::Single(uuid::Uuid::now_v7()), + primary_key_events, + secondary_keys_events, + bytes, + link: new_link, + }), + Location::caller(), + ) } fn reclaim_pages(&self, page_ids: Vec) -> PersistenceResult { @@ -1391,17 +1537,21 @@ impl Drop impl PersistenceTask { + /// `#[track_caller]` so an event-gap report names the producer that + /// pushed the operation rather than this forwarding line. + #[track_caller] pub fn apply_operation(&self, op: Operation) -> PersistenceResult { - self.queue.push(op) + self.queue.push_at(op, Location::caller()) } /// Enqueues a batch of operations atomically with a single worker /// wake-up. See [`Queue::push_many`]. + #[track_caller] pub fn apply_operations( &self, ops: Vec>, ) -> PersistenceResult { - self.queue.push_many(ops) + self.queue.push_many_at(ops, Location::caller()) } pub fn ensure_running(&self) -> PersistenceResult { @@ -1417,14 +1567,15 @@ impl /// This is intentionally separate from `VacuumStats`: online vacuum makes /// freed pages durably reusable, but does not truncate `.wt.data`. /// Operators can sample this value to observe physical growth and reuse. - pub async fn persisted_data_file_size_bytes(&self) -> std::io::Result { - tokio::fs::metadata(format!( + pub async fn persisted_data_file_size_bytes(&self) -> Result { + // `metadata` answers with the length, which is the only thing anything + // here ever asked a metadata handle for. + crate::fsx::metadata(format!( "{}/{}", self.table_path.trim_end_matches('/'), WT_DATA_EXTENSION )) .await - .map(|metadata| metadata.len()) } /// Returns a sink that lets vacuum queue persistence operations for row @@ -1448,12 +1599,15 @@ impl { let table_path = engine.config().table_path().to_owned(); let lifecycle = Arc::new(PersistenceLifecycle::new()); - let queue = Arc::new(Queue::new(lifecycle.clone())); + let queue = Arc::new(Queue::new(lifecycle.clone(), &table_path)); let engine_queue = queue.clone(); let engine_lifecycle = lifecycle.clone(); let analyzer_inner_wt: Arc = Default::default(); let mut analyzer = QueueAnalyzer::new(analyzer_inner_wt.clone()); + // Producer and consumer must share one ledger: the queue records who + // pushed an event, the analyzer's gap guard reads it back. + analyzer.attach_event_ledger(queue.event_ledger()); let analyzer_in_progress = Arc::new(AtomicBool::new(true)); let task_analyzer_in_progress = analyzer_in_progress.clone(); diff --git a/src/primary_key.rs b/src/primary_key.rs index ff73bbec..ab4469bd 100644 --- a/src/primary_key.rs +++ b/src/primary_key.rs @@ -1,4 +1,4 @@ -use std::sync::atomic::{ +use core::sync::atomic::{ AtomicI8, AtomicI16, AtomicI32, AtomicI64, AtomicU8, AtomicU16, AtomicU32, AtomicU64, Ordering, }; @@ -21,7 +21,7 @@ pub trait PrimaryKeyGeneratorRange { /// /// Concurrent `reserve` and [`PrimaryKeyGenerator::next`] calls never /// observe overlapping keys. - fn reserve(&self, count: usize) -> std::ops::Range; + fn reserve(&self, count: usize) -> core::ops::Range; } pub trait PrimaryKeyGeneratorState { @@ -52,7 +52,7 @@ macro_rules! atomic_primary_key { } impl PrimaryKeyGeneratorRange<$ty> for $atomic_ty { - fn reserve(&self, count: usize) -> std::ops::Range<$ty> { + fn reserve(&self, count: usize) -> core::ops::Range<$ty> { let count = <$ty>::try_from(count).unwrap_or_else(|_| { panic!( "autoincrement primary key space exhausted: cannot reserve {count} {} keys", @@ -142,7 +142,7 @@ mod tests { #[test] fn concurrent_reservations_never_overlap() { - use std::sync::Arc; + use alloc::sync::Arc; let generator = Arc::new(AtomicU64::from_state(0)); let mut handles = vec![]; diff --git a/src/table/mod.rs b/src/table/mod.rs index eeba4d01..9c1d77f8 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -1,9 +1,13 @@ +use alloc::{string::String, vec::Vec}; pub mod select; pub mod system_info; +#[cfg(feature = "std")] pub mod vacuum; use crate::in_memory::{ArchivedRowWrapper, DataPages, RowWrapper, StorableRow}; -use crate::persistence::{AcknowledgeOperation, InsertOperation, Operation, PersistenceLoadError}; +#[cfg(feature = "std")] +use crate::persistence::PersistenceLoadError; +use crate::persistence::{AcknowledgeOperation, InsertOperation, Operation}; use crate::prelude::{Link, LockMap, OperationId, PrimaryKeyGeneratorState}; use crate::primary_key::{PrimaryKeyGenerator, TablePrimaryKey}; use crate::util::OffsetEqLink; @@ -11,8 +15,13 @@ use crate::{ AvailableIndex, IndexError, IndexMap, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, convert_change_events, in_memory, }; +use alloc::sync::Arc; +use core::fmt::Debug; +use core::marker::PhantomData; use data_bucket::INNER_PAGE_SIZE; use derive_more::{Display, Error, From}; +#[cfg(feature = "std")] +use hashbrown::HashSet; use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; #[cfg(feature = "perf_measurements")] @@ -24,11 +33,8 @@ use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Portable, Serialize}; -use std::collections::HashSet; -use std::fmt::Debug; -use std::marker::PhantomData; +#[cfg(feature = "std")] use std::path::Path; -use std::sync::Arc; use uuid::Uuid; /// Keys per chunk when a bulk delete takes its mutation guards. /// @@ -51,7 +57,7 @@ pub struct WorkTable< const DATA_LENGTH: usize = INNER_PAGE_SIZE, PkMap = IndexMap>, > where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static, PkMap: crate::UniqueIndex>, { @@ -94,7 +100,7 @@ impl< PkMap, > where - PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + core::hash::Hash, SecondaryIndexes: Default, PkGen: Default, PkMap: crate::UniqueIndex>, @@ -127,7 +133,7 @@ impl< > WorkTable where Row: TableRow, - PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + core::hash::Hash, PkMap: crate::UniqueIndex>, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, @@ -138,6 +144,7 @@ where /// This load-only scan prevents a torn index link from turning zeroed or /// unrelated bytes into a plausible row. It deliberately does not run on /// steady-state operations. + #[cfg(feature = "std")] pub fn validate_persisted_state(&self, path: impl AsRef) -> Result<(), PersistenceLoadError> where <::WrappedRow as Archive>::Archived: Portable @@ -146,7 +153,7 @@ where { let path = path.as_ref(); let mut links = HashSet::with_capacity(self.primary_index.pk_map.len()); - let mut cells_by_page = std::collections::HashMap::::new(); + let mut cells_by_page = hashbrown::HashMap::::new(); for (primary_key, offset_link) in self.primary_index.pk_map.iter_values() { if !links.insert(offset_link) { @@ -211,7 +218,7 @@ where /// caller can iterate it directly while pre-assigning contiguous keys to a /// batch of rows for `insert_many`. Interleaved [`Self::get_next_pk`] /// calls keep working and never overlap a reservation. - pub fn reserve_pks(&self, count: usize) -> std::ops::Range + pub fn reserve_pks(&self, count: usize) -> core::ops::Range where PkGen: crate::primary_key::PrimaryKeyGeneratorRange, { @@ -238,7 +245,7 @@ where if current_link == Some(link) { return None; } - std::hint::spin_loop(); + core::hint::spin_loop(); } None } @@ -475,7 +482,7 @@ where /// Returns the keys actually deleted, in key order. pub fn delete_range(&self, range: R) -> Result, BatchDeleteError> where - R: std::ops::RangeBounds, + R: core::ops::RangeBounds, Row: Archive + Clone + for<'a> Serialize, Share>, rkyv::rancor::Error>>, @@ -531,8 +538,8 @@ where .primary_index .pk_map .range_values(( - std::ops::Bound::Included(chunk[0].clone()), - std::ops::Bound::Included(chunk[chunk.len() - 1].clone()), + core::ops::Bound::Included(chunk[0].clone()), + core::ops::Bound::Included(chunk[chunk.len() - 1].clone()), )) .map(|(key, link)| (key, link.into())) .collect(); @@ -1089,8 +1096,8 @@ where ops.push(Operation::Insert(InsertOperation { id: OperationId::Multi(batch_id), pk_gen_state: self.pk_gen.get_state(), - primary_key_events: std::mem::take(&mut forward_primary[row_index]), - secondary_keys_events: std::mem::take(&mut forward_secondary[row_index]), + primary_key_events: core::mem::take(&mut forward_primary[row_index]), + secondary_keys_events: core::mem::take(&mut forward_secondary[row_index]), bytes, link: *link, })); @@ -1349,7 +1356,7 @@ pub enum BatchInsertError { /// batch needs to know the prefix already succeeded rather than assume nothing /// happened. #[derive(Debug, Display, Error)] -pub enum BatchDeleteError { +pub enum BatchDeleteError { /// One key could not be deleted. Everything before it was. #[display("batch delete stopped at {key:?} after {deleted} deleted: {source}")] Key { @@ -1376,5 +1383,6 @@ pub enum WorkTableError { PrimaryUpdateTry, PagesError(in_memory::PagesExecutionError), #[display("{}", _0)] - PersistenceError(#[error(not(source))] std::sync::Arc), + #[cfg(feature = "std")] + PersistenceError(#[error(not(source))] alloc::sync::Arc), } diff --git a/src/table/select/mod.rs b/src/table/select/mod.rs index 5b9fc6e6..fe7de3ed 100644 --- a/src/table/select/mod.rs +++ b/src/table/select/mod.rs @@ -1,4 +1,4 @@ -use std::collections::VecDeque; +use alloc::collections::VecDeque; mod query; diff --git a/src/table/select/query.rs b/src/table/select/query.rs index 2b2f3f66..41bd6a01 100644 --- a/src/table/select/query.rs +++ b/src/table/select/query.rs @@ -1,7 +1,8 @@ use crate::WorkTableError; use crate::select::{Order, QueryParams}; +use alloc::vec::Vec; -use std::collections::VecDeque; +use alloc::collections::VecDeque; pub struct SelectQueryBuilder where diff --git a/src/table/system_info.rs b/src/table/system_info.rs index 49e95761..109f9e7c 100644 --- a/src/table/system_info.rs +++ b/src/table/system_info.rs @@ -1,5 +1,5 @@ -use prettytable::{Table, format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR, row}; -use std::fmt::{self, Debug, Display, Formatter}; +use alloc::{string::String, string::ToString, vec::Vec}; +use core::fmt::{self, Debug, Display, Formatter}; use crate::in_memory::{RowWrapper, StorableRow}; use crate::mem_stat::MemStat; @@ -55,7 +55,7 @@ impl< PkMap, > WorkTable where - PrimaryKey: Debug + Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, PkMap: UniqueIndex>, @@ -118,31 +118,76 @@ impl Display for SystemInfo { "Allocated Memory: {mem_fmt} (data) + {idx_fmt} (indexes) = {total_fmt} total\n" )?; - let mut table = Table::new(); - table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR); - table.add_row(row!["Index", "Type", "Keys", "Capacity", "Node Count", "Heap", "Used"]); - + // **Padded by hand rather than by a table crate.** + // + // This used to be `prettytable-rs`, which reaches `csv` and then + // `memchr` and fails to build without `std` in 505 places. Every + // alternative measured puts its usable API behind `std` too: `tabled` + // compiles without `std` but exposes no `Table` at all in that mode, + // and `comfy-table` and `ascii_table` do not compile. Seven columns of + // short strings are not worth a dependency that decides whether this + // crate can be embedded. + let mut rows: Vec<[String; COLUMNS]> = Vec::with_capacity(self.indexes_info.len() + 1); + rows.push([ + "Index".to_string(), + "Type".to_string(), + "Keys".to_string(), + "Capacity".to_string(), + "Node Count".to_string(), + "Heap".to_string(), + "Used".to_string(), + ]); for idx in &self.indexes_info { - table.add_row(row![ - idx.name, + rows.push([ + idx.name.to_string(), idx.index_type.to_string(), - idx.key_count, - idx.capacity, - idx.node_count, + idx.key_count.to_string(), + idx.capacity.to_string(), + idx.node_count.to_string(), fmt_bytes(idx.heap_size), fmt_bytes(idx.used_size), ]); } - let mut buffer = Vec::new(); - table.print(&mut buffer).unwrap(); - let table_str = String::from_utf8(buffer).unwrap(); - writeln!(f, "{}", table_str.trim_end())?; + // Width by character count, not byte length: an index named with any + // multi-byte character would otherwise pad short and skew every column + // after it. + let mut widths = [0usize; COLUMNS]; + for row in &rows { + for (width, cell) in widths.iter_mut().zip(row) { + *width = (*width).max(cell.chars().count()); + } + } + + for (index, row) in rows.iter().enumerate() { + for (column, (cell, width)) in row.iter().zip(&widths).enumerate() { + if column + 1 == COLUMNS { + write!(f, "{cell}")?; + } else { + let padding = width - cell.chars().count(); + write!(f, "{cell}{:padding$}{COLUMN_GAP}", "")?; + } + } + writeln!(f)?; + // A rule under the header, and nothing between the rows: the same + // shape `FORMAT_NO_BORDER_LINE_SEPARATOR` produced. + if index == 0 { + let rule: usize = widths.iter().sum::() + COLUMN_GAP.len() * (COLUMNS - 1); + writeln!(f, "{:- String { const KB: f64 = 1024.0; const MB: f64 = 1024.0 * KB; diff --git a/src/table/vacuum/fragmentation_info.rs b/src/table/vacuum/fragmentation_info.rs index 6541e014..68d0fb92 100644 --- a/src/table/vacuum/fragmentation_info.rs +++ b/src/table/vacuum/fragmentation_info.rs @@ -12,7 +12,8 @@ //! //! [`WorkTable`]: crate::table::WorkTable -use std::collections::HashMap; +use alloc::vec::Vec; +use hashbrown::HashMap; use data_bucket::page::PageId; use data_bucket::{INNER_PAGE_SIZE, Link}; diff --git a/src/table/vacuum/manager.rs b/src/table/vacuum/manager.rs index 566ed452..c7520644 100644 --- a/src/table/vacuum/manager.rs +++ b/src/table/vacuum/manager.rs @@ -1,7 +1,8 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; +use alloc::sync::Arc; +use alloc::{string::ToString, vec::Vec}; +use core::sync::atomic::{AtomicU64, Ordering}; +use core::time::Duration; +use hashbrown::HashMap; use tokio::task::AbortHandle; use parking_lot::RwLock; diff --git a/src/table/vacuum/mod.rs b/src/table/vacuum/mod.rs index 74fc55db..1c94e20f 100644 --- a/src/table/vacuum/mod.rs +++ b/src/table/vacuum/mod.rs @@ -1,3 +1,5 @@ +use alloc::boxed::Box; +use alloc::vec::Vec; use async_trait::async_trait; use data_bucket::Link; diff --git a/src/table/vacuum/pacing.rs b/src/table/vacuum/pacing.rs index 1ff64c9d..ea4bf81f 100644 --- a/src/table/vacuum/pacing.rs +++ b/src/table/vacuum/pacing.rs @@ -15,8 +15,8 @@ //! the preceding check. Every insert, delete and upsert passes through those //! stripes, including mutations that never ask for reclaimable space. -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::time::Duration; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use core::time::Duration; use smart_default::SmartDefault; @@ -114,7 +114,7 @@ pub trait ForegroundActivity { impl ForegroundActivity for crate::lock::LockMap where - PrimaryKey: Clone + std::fmt::Debug + Eq + std::hash::Hash, + PrimaryKey: Clone + core::fmt::Debug + Eq + core::hash::Hash, { fn mutations_in_flight(&self) -> usize { crate::lock::LockMap::mutations_in_flight(self) @@ -163,8 +163,8 @@ impl VacuumPacing { #[cfg(test)] mod tests { - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use super::*; diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index 47030bca..b95b787c 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -1,9 +1,12 @@ -use std::collections::VecDeque; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; +use alloc::boxed::Box; +use alloc::collections::VecDeque; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicU64, Ordering}; +use core::time::Duration; +use std::time::Instant; /// How long retirements have to stop arriving for a delete burst to count as /// over. Short enough that a sweep still follows a delete promptly, long @@ -85,7 +88,7 @@ pub struct EmptyDataVacuum< const DATA_LENGTH: usize, SecondaryEvents = (), > where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + core::hash::Hash, Row: StorableRow + Send + Clone + 'static + Debug, PkMap: UniqueIndex>, { @@ -139,7 +142,7 @@ impl< > where Row: TableRow + StorableRow + Send + Clone + 'static, - PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + core::hash::Hash, PkMap: UniqueIndex>, ::WrappedRow: RowWrapper, Row: Archive @@ -684,7 +687,7 @@ impl< > where Row: TableRow + StorableRow + Send + Sync + Clone + 'static, - PrimaryKey: Debug + Clone + Ord + Send + Sync + TablePrimaryKey + std::hash::Hash, + PrimaryKey: Debug + Clone + Ord + Send + Sync + TablePrimaryKey + core::hash::Hash, PkMap: UniqueIndex> + Send + Sync + 'static, ::WrappedRow: RowWrapper, Row: Archive @@ -737,9 +740,10 @@ where #[cfg(test)] mod tests { - use std::collections::{HashMap, VecDeque}; - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use alloc::collections::VecDeque; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use hashbrown::HashMap; use data_bucket::Link; use data_bucket::page::PageId; @@ -747,7 +751,7 @@ mod tests { use crate::in_memory::{ArchivedRowWrapper, RowWrapper, StorableRow}; use crate::prelude::*; - use std::time::Duration; + use core::time::Duration; use crate::vacuum::vacuum::{CandidateMove, EmptyDataVacuum}; use crate::vacuum::{VacuumGate, VacuumPacing, WorkTableVacuum}; diff --git a/src/util/mod.rs b/src/util/mod.rs index 7b1adf10..ac5cc7a4 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,8 +1,22 @@ pub(crate) mod epoch; mod offset_eq_link; +#[cfg(feature = "std")] mod optimized_vec; mod ordered_float; pub use offset_eq_link::OffsetEqLink; +#[cfg(feature = "std")] pub use optimized_vec::OptimizedVec; pub use ordered_float::{OrderedF32Def, OrderedF64Def}; + +/// Give up the rest of the timeslice after a spin has stopped paying. +/// +/// Under `std` that is the operating system's yield. Without one there is no +/// scheduler to yield to, so the spin hint is the whole of what can be done. +#[inline] +pub(crate) fn yield_now() { + #[cfg(feature = "std")] + std::thread::yield_now(); + #[cfg(not(feature = "std"))] + core::hint::spin_loop(); +} diff --git a/src/util/offset_eq_link.rs b/src/util/offset_eq_link.rs index 0b39d0db..a1c3c3c5 100644 --- a/src/util/offset_eq_link.rs +++ b/src/util/offset_eq_link.rs @@ -22,20 +22,20 @@ impl OffsetEqLink { } } -impl std::hash::Hash for OffsetEqLink { - fn hash(&self, state: &mut H) { +impl core::hash::Hash for OffsetEqLink { + fn hash(&self, state: &mut H) { self.absolute_index().hash(state); } } impl PartialOrd for OffsetEqLink { - fn partial_cmp(&self, other: &Self) -> Option { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for OffsetEqLink { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { self.absolute_index().cmp(&other.absolute_index()) } } @@ -48,7 +48,7 @@ impl PartialEq for OffsetEqLink { impl Eq for OffsetEqLink {} -impl std::ops::Deref for OffsetEqLink { +impl core::ops::Deref for OffsetEqLink { type Target = Link; fn deref(&self) -> &Self::Target { @@ -85,7 +85,7 @@ impl SizeMeasurable for OffsetEqLink { mod tests { use super::*; use data_bucket::page::PageId; - use std::collections::HashSet; + use hashbrown::HashSet; const TEST_DATA_LENGTH: usize = 4096; diff --git a/src/util/optimized_vec.rs b/src/util/optimized_vec.rs index c26114d0..9570ec9b 100644 --- a/src/util/optimized_vec.rs +++ b/src/util/optimized_vec.rs @@ -1,3 +1,4 @@ +use alloc::vec::Vec; /// Struct for storing data in a vector with stable indexes and slot reuse. /// Slots are `Option`: `remove` is `Option::take`, so the value moves out /// without a `Clone` bound and the slot is freed immediately. The previous @@ -201,7 +202,7 @@ mod tests { /// count proves the removed value is the only remaining owner. #[test] fn test_optimized_vec_remove_moves_without_clone() { - use std::rc::Rc; + use alloc::rc::Rc; struct NotClone(#[allow(dead_code)] Rc<()>); diff --git a/src/util/ordered_float.rs b/src/util/ordered_float.rs index 598be510..29ff13e3 100644 --- a/src/util/ordered_float.rs +++ b/src/util/ordered_float.rs @@ -4,7 +4,7 @@ use rkyv::{Archive, Deserialize, Serialize}; #[rkyv(remote = ordered_float::OrderedFloat, archived = ArchivedF64)] #[rkyv(derive(Debug))] pub struct OrderedF64Def { - #[rkyv(getter = std::ops::Deref::deref)] + #[rkyv(getter = core::ops::Deref::deref)] value: f64, } @@ -18,7 +18,7 @@ impl From for ordered_float::OrderedFloat { #[rkyv(remote = ordered_float::OrderedFloat, archived = ArchivedF32)] #[rkyv(derive(Debug))] pub struct OrderedF32Def { - #[rkyv(getter = std::ops::Deref::deref)] + #[rkyv(getter = core::ops::Deref::deref)] value: f32, } diff --git a/tests/mod.rs b/tests/mod.rs index 86ec0ed7..2e1e3256 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -46,12 +46,12 @@ pub fn check_if_dirs_are_same(got: String, expected: String) -> bool { pub async fn remove_file_if_exists(path: String) { if Path::new(path.as_str()).exists() { - tokio::fs::remove_file(path.as_str()).await.unwrap(); + ::worktable::prelude::fsx::remove_file(path.as_str()).await.unwrap(); } } pub async fn remove_dir_if_exists(path: String) { if Path::new(path.as_str()).exists() { - tokio::fs::remove_dir_all(path).await.unwrap() + ::worktable::prelude::fsx::remove_dir_all(path).await.unwrap() } } diff --git a/tests/persistence/custom_page_size.rs b/tests/persistence/custom_page_size.rs new file mode 100644 index 00000000..b7563cae --- /dev/null +++ b/tests/persistence/custom_page_size.rs @@ -0,0 +1,70 @@ +use worktable::prelude::*; +use worktable::worktable; + +use crate::remove_dir_if_exists; + +/// Half the default stride. Chosen so a page written at this size lands where +/// the default would put the middle of a page: a table that quietly fell back +/// to the default could not produce the file lengths asserted below. +const HALF: u32 = 8192; + +worktable! ( + name: HalfPage, + persist: true, + columns: { + id: u64 primary_key, + payload: u64, + }, + config: { + page_size: 8192 + } +); + +/// `page_size` beside `persist: true` used to be refused outright: the seeks +/// computed every offset from a hardcoded stride while the generated table +/// threaded the configured one, and the two disagreeing corrupted the file. +/// +/// A round trip alone would not prove the setting took effect, because a table +/// that fell back to the default would still reload its own writes. So the +/// file length is checked too, and it can only be a multiple of the configured +/// stride. +#[tokio::test] +async fn a_persisted_table_with_a_custom_page_size_reloads_what_it_wrote() { + let dir = "tests/data/custom_page_size/persisted"; + remove_dir_if_exists(dir.to_string()).await; + let config = + DiskConfig::new_with_table_name(dir, HalfPageWorkTable::name_snake_case(), HalfPageWorkTable::version()); + + let mut expected = Vec::new(); + { + let engine = HalfPagePersistenceEngine::new(config.clone()).await.unwrap(); + let table = HalfPageWorkTable::load(engine).await.unwrap(); + // Enough rows to need several pages at this stride, so the page + // transitions are exercised and not just the first page. + for i in 0..2_000u64 { + let row = HalfPageRow { id: i, payload: i }; + table.insert(row.clone()).await.unwrap(); + expected.push(row); + } + table.wait_for_ops().await.unwrap(); + } + + let data_file_path = format!("{dir}/{}/.wt.data", HalfPageWorkTable::name_snake_case()); + let length = std::fs::metadata(&data_file_path).unwrap().len(); + let pages = length.div_ceil(u64::from(HALF)); + assert!( + pages >= 2, + "expected several {HALF}-byte pages, got {pages} from {length} bytes" + ); + + let engine = HalfPagePersistenceEngine::new(config).await.unwrap(); + let table = HalfPageWorkTable::load(engine) + .await + .expect("a table with a custom page size must reload"); + assert_eq!(table.select_all().execute().unwrap().len(), expected.len()); + for row in &expected { + assert_eq!(table.select(row.id).as_ref(), Some(row)); + } + + remove_dir_if_exists(dir.to_string()).await; +} diff --git a/tests/persistence/duplicate_key_index_reload.rs b/tests/persistence/duplicate_key_index_reload.rs index d3b41405..c241c9b9 100644 --- a/tests/persistence/duplicate_key_index_reload.rs +++ b/tests/persistence/duplicate_key_index_reload.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; @@ -111,12 +112,11 @@ async fn assert_straddling_topology(dir: &str) { use data_bucket::INNER_PAGE_SIZE; use std::sync::Arc; use std::sync::atomic::AtomicU32; - use tokio::fs::OpenOptions; let path = format!("{dir}/duplicate_key_reload/score_idx.wt.idx"); - let mut file = OpenOptions::new().read(true).write(true).open(&path).await.unwrap(); + let mut file = worktable::prelude::fsx::open(&path).await.unwrap(); let next_id_gen = Arc::new(AtomicU32::new(1)); - let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -133,9 +133,12 @@ async fn assert_straddling_topology(dir: &str) { let mut pages_per_key: BTreeMap = BTreeMap::new(); for page_id in mappings { - let page = parse_page::, { DUPLICATE_KEY_RELOAD_PAGE_SIZE as u32 }>(&mut file, page_id.into()) - .await - .unwrap(); + let page = parse_page::, { DUPLICATE_KEY_RELOAD_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, + page_id.into(), + ) + .await + .unwrap(); let keys: BTreeSet = page.inner.index_values[..page.inner.current_length as usize] .iter() .map(|v| v.key) diff --git a/tests/persistence/index_page/read.rs b/tests/persistence/index_page/read.rs index 5cc90005..ebc19d28 100644 --- a/tests/persistence/index_page/read.rs +++ b/tests/persistence/index_page/read.rs @@ -1,16 +1,13 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::{INNER_PAGE_SIZE, IndexPage, parse_page}; -use tokio::fs::OpenOptions; #[tokio::test] async fn test_index_page_read_in_space() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/test_persist/primary.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/primary.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 99); @@ -20,14 +17,11 @@ async fn test_index_page_read_in_space() { #[tokio::test] async fn test_index_page_read_after_create_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -40,14 +34,11 @@ async fn test_index_page_read_after_create_node_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -64,14 +55,12 @@ async fn test_index_page_read_after_insert_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") + .await + .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 7); @@ -88,14 +77,11 @@ async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index #[tokio::test] async fn test_index_page_read_after_remove_at_node_id_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 3); @@ -109,14 +95,11 @@ async fn test_index_page_read_after_remove_at_node_id_in_space_index() { #[tokio::test] async fn test_index_page_read_after_remove_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_at.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -130,14 +113,12 @@ async fn test_index_page_read_after_remove_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_removed_place.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_removed_place.wt.idx") + .await + .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 7); @@ -159,14 +140,11 @@ async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_second_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_second_node.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 5); @@ -176,7 +154,7 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { assert_eq!(page.inner.index_values.first().unwrap().key, 5); assert_eq!(page.inner.index_values.first().unwrap().link.length, 24); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 3) .await .unwrap(); assert_eq!(page.inner.node_id.key, 15); @@ -189,14 +167,12 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_creation_of_node_after_remove_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") + .await + .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 10); @@ -206,7 +182,7 @@ async fn test_index_pages_read_after_creation_of_node_after_remove_node_in_space assert_eq!(page.inner.index_values.first().unwrap().key, 10); assert_eq!(page.inner.index_values.first().unwrap().link.length, 24); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 3) .await .unwrap(); assert_eq!(page.inner.node_id.key, 15); @@ -219,14 +195,11 @@ async fn test_index_pages_read_after_creation_of_node_after_remove_node_in_space #[tokio::test] async fn test_index_pages_read_full_page() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_big_amount.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_big_amount.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 1000); @@ -236,21 +209,18 @@ async fn test_index_pages_read_full_page() { #[tokio::test] async fn test_index_pages_read_after_node_split() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_split_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_split_node.wt.idx") .await .unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); assert_eq!(page.inner.node_id.key, 457); assert_eq!(page.inner.current_index, 0); assert_eq!(page.inner.current_length, 453); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) + let page = parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 3) .await .unwrap(); assert_eq!(page.inner.node_id.key, 1000); diff --git a/tests/persistence/index_page/unsized_read.rs b/tests/persistence/index_page/unsized_read.rs index e4673d55..c2e2fec4 100644 --- a/tests/persistence/index_page/unsized_read.rs +++ b/tests/persistence/index_page/unsized_read.rs @@ -1,19 +1,18 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::{INNER_PAGE_SIZE, Link, UnsizedIndexPage, parse_page}; -use tokio::fs::OpenOptions; #[tokio::test] async fn test_index_page_read_after_create_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -31,18 +30,18 @@ async fn test_index_page_read_after_create_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") - .await - .unwrap(); - - let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") .await .unwrap(); + let page = + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); + assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 1); let value = page.inner.index_values.first().unwrap(); @@ -57,9 +56,11 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { ); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 3, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Someone from somewhere".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -77,17 +78,16 @@ async fn test_index_pages_read_after_creation_of_second_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_remove_node_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 3, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Someone from somewhere".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -105,17 +105,16 @@ async fn test_index_pages_read_after_remove_node_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_insert_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 2); @@ -143,17 +142,16 @@ async fn test_index_pages_read_after_insert_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_remove_at_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone".to_string()); assert_eq!(page.inner.index_values.len(), 1); @@ -171,18 +169,18 @@ async fn test_index_page_read_after_remove_at_in_space_index() { #[tokio::test] async fn test_index_page_read_after_remove_at_node_id_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") - .await - .unwrap(); - - let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") .await .unwrap(); + let page = + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); + assert_eq!(page.inner.node_id.key, "Something else".to_string()); assert_eq!(page.inner.index_values.len(), 1); let value = page.inner.index_values.first().unwrap(); @@ -199,17 +197,18 @@ async fn test_index_page_read_after_remove_at_node_id_in_space_index() { #[tokio::test] async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx") - .await - .unwrap(); + let mut file = worktable::prelude::fsx::open( + "tests/data/expected/space_index_unsized/process_insert_at_with_node_id_update.wt.idx", + ) + .await + .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone 1".to_string()); assert_eq!(page.inner.index_values.len(), 2); @@ -237,18 +236,18 @@ async fn test_index_page_read_after_insert_at_with_node_id_update_in_space_index #[tokio::test] async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx") - .await - .unwrap(); - - let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_insert_at_removed_place.wt.idx") .await .unwrap(); + let page = + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); + assert_eq!(page.inner.node_id.key, "Something from someone 1".to_string()); assert_eq!(page.inner.index_values.len(), 3); let first_value = &page.inner.index_values[0]; @@ -285,24 +284,25 @@ async fn test_index_page_read_after_insert_at_removed_place_in_space_index() { #[tokio::test] async fn test_index_pages_read_after_node_split() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_split_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_split_node.wt.idx") .await .unwrap(); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 2) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 2, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone 52"); assert_eq!(page.inner.slots_size, 53); let page = - parse_page::, { INNER_PAGE_SIZE as u32 }>(&mut file, 3) - .await - .unwrap(); + parse_page::, { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>( + &mut file, 3, + ) + .await + .unwrap(); assert_eq!(page.inner.node_id.key, "Something from someone _100"); assert_eq!(page.inner.slots_size, 48); } diff --git a/tests/persistence/insert_cost_shape.rs b/tests/persistence/insert_cost_shape.rs new file mode 100644 index 00000000..72cc3ba9 --- /dev/null +++ b/tests/persistence/insert_cost_shape.rs @@ -0,0 +1,216 @@ +//! Where the time in an insert goes, since it is not the disk. +//! +//! WorkTable's bulk load runs at about 200 MB/s while the write path under it +//! does gigabytes, and removing persistence entirely changes nothing. So the +//! cost is in the insert. This asks the first question that splits the +//! candidates: does it scale with the size of the row, or is it a fixed price +//! per row? + +use worktable::prelude::*; +use worktable_codegen::worktable; + +worktable!( + name: InsertShape, + columns: { + id: u64 primary_key, + payload: String, + } +); + +#[test] +#[ignore = "a measurement, not an assertion"] +fn does_insert_cost_scale_with_row_size() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + const ROWS: u64 = 25_000; + println!(" payload rows/s us/row MB/s"); + for size in [8usize, 64, 256, 1024, 2048, 4096, 8192] { + let payload = "x".repeat(size); + // Warm: allocator and the table's first growth are not the subject. + { + let warm = InsertShapeWorkTable::default(); + for id in 0..1_000u64 { + warm.insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + } + let table = InsertShapeWorkTable::default(); + let at = std::time::Instant::now(); + for id in 0..ROWS { + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + let elapsed = at.elapsed().as_secs_f64(); + println!( + " {size:>7} {:>9.0} {:>8.2} {:>7.0}", + ROWS as f64 / elapsed, + elapsed * 1e6 / ROWS as f64, + (ROWS as usize * size) as f64 / 1e6 / elapsed, + ); + } + }); +} + +/// The same rows, with the table taken out of it: what the loop costs when all +/// it does is build the row and drop it. Anything the insert arm spends beyond +/// this is the table. +#[test] +#[ignore = "a measurement, not an assertion"] +fn what_the_loop_costs_without_the_table() { + const ROWS: u64 = 25_000; + println!(" payload rows/s us/row"); + for size in [8usize, 4096] { + let payload = "x".repeat(size); + let at = std::time::Instant::now(); + let mut sink = 0usize; + for id in 0..ROWS { + let row = InsertShapeRow { + id, + payload: payload.clone(), + }; + sink = sink.wrapping_add(row.payload.len()); + std::hint::black_box(&row); + } + let elapsed = at.elapsed().as_secs_f64(); + std::hint::black_box(sink); + println!( + " {size:>7} {:>9.0} {:>8.3}", + ROWS as f64 / elapsed, + elapsed * 1e6 / ROWS as f64, + ); + } +} + +/// A long run of the expensive case, so a sampling profiler has something to +/// look at. Not a measurement in itself. +#[test] +#[ignore = "for profiling only"] +fn keep_inserting_four_kilobyte_rows() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let payload = "x".repeat(4096); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while std::time::Instant::now() < deadline { + let table = InsertShapeWorkTable::default(); + for id in 0..20_000u64 { + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + } + }); +} + +/// Does an insert get slower as the table fills? +/// +/// A cost that scales with rows already present is O(n) per insert and O(n^2) +/// overall, which is what a super-linear response to row size would look like +/// if bigger rows simply reach any given page count sooner. +#[test] +#[ignore = "a measurement, not an assertion"] +fn does_insert_slow_down_as_the_table_fills() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + for size in [256usize, 4096] { + let payload = "x".repeat(size); + let table = InsertShapeWorkTable::default(); + const BLOCK: u64 = 2_000; + const BLOCKS: u64 = 10; + println!(" payload {size}: us/row by block of {BLOCK}"); + let mut line = String::new(); + for block in 0..BLOCKS { + let at = std::time::Instant::now(); + for n in 0..BLOCK { + let id = block * BLOCK + n; + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + let per_row = at.elapsed().as_secs_f64() * 1e6 / BLOCK as f64; + line.push_str(&format!("{per_row:>8.2}")); + } + println!(" {line}"); + } + }); +} + +/// The per-row cost with the page-list clone nearly absent, which is the floor +/// a fix for it would approach. +/// +/// The clone is O(pages), so a table that has barely any pages barely pays it. +/// Timing small tables gives the cost of everything else: the row clone, the +/// rkyv serialize, and the copy into the page. +#[test] +#[ignore = "a measurement, not an assertion"] +fn the_floor_with_almost_no_pages() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + println!(" payload rows pages us/row MB/s"); + for size in [256usize, 1024, 4096] { + let payload = "x".repeat(size); + let per_page = (16356 / size).max(1); + for rows in [30u64, 120, 480] { + // Median of several fresh tables: a single small run is noise. + let mut samples = Vec::new(); + for _ in 0..25 { + let table = InsertShapeWorkTable::default(); + let at = std::time::Instant::now(); + for id in 0..rows { + table + .insert(InsertShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + samples.push(at.elapsed().as_secs_f64() / rows as f64); + } + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let per_row = samples[samples.len() / 2]; + println!( + " {size:>7} {rows:>5} {:>5} {:>7.3} {:>7.0}", + rows as usize / per_page, + per_row * 1e6, + size as f64 / 1e6 / per_row, + ); + } + } + }); +} diff --git a/tests/persistence/insert_latency.rs b/tests/persistence/insert_latency.rs new file mode 100644 index 00000000..00afe6a0 --- /dev/null +++ b/tests/persistence/insert_latency.rs @@ -0,0 +1,124 @@ +//! Per-insert latency, split by whether the insert had to allocate a page. +//! +//! These are two populations, not one distribution. An insert that fits in the +//! page already open is cheap; one that has to add a page pays for the page +//! list as well. Blending them hides the second behind the first, and how much +//! it hides depends on row size: at 4 KiB a page holds three rows, so a third +//! of all inserts allocate and the expensive population is not a tail at all. + +use worktable::prelude::PersistedWorkTable; +use worktable::prelude::*; +use worktable_codegen::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: InsertLatency, + persist: true, + columns: { + id: u64 primary_key, + payload: String, + } +); + +worktable!( + name: InsertLatencyMemory, + columns: { + id: u64 primary_key, + payload: String, + } +); + +fn report(label: &str, mut us: Vec, of: usize) { + if us.is_empty() { + println!(" {label:<34} (none)"); + return; + } + us.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let at = |q: f64| us[((us.len() as f64 * q) as usize).min(us.len() - 1)]; + println!( + " {label:<34} {:>5.1}% of inserts p50 {:>7.2} p99 {:>8.2} max {:>9.2} (us)", + 100.0 * us.len() as f64 / of as f64, + at(0.50), + at(0.99), + us[us.len() - 1], + ); +} + +#[test] +#[ignore = "a measurement, not an assertion"] +fn insert_latency_split_by_page_allocation() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + const ROWS: u64 = 20_000; + for size in [256usize, 4096] { + let payload = "x".repeat(size); + println!("payload {size} B, {ROWS} rows"); + + // ---- no persistence + let table = InsertLatencyMemoryWorkTable::default(); + let (mut same, mut fresh) = (Vec::new(), Vec::new()); + let mut pages = table.0.data.get_page_count(); + for id in 0..ROWS { + let at = std::time::Instant::now(); + table + .insert(InsertLatencyMemoryRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + let took = at.elapsed().as_secs_f64() * 1e6; + let now = table.0.data.get_page_count(); + if now == pages { + same.push(took) + } else { + fresh.push(took) + } + pages = now; + } + report("in memory, existing page", same, ROWS as usize); + report("in memory, allocated a page", fresh, ROWS as usize); + + // ---- persisted + let dir = "tests/data/insert_latency"; + remove_dir_if_exists(dir.to_string()).await; + let config = DiskConfig::new_with_table_name( + dir, + InsertLatencyWorkTable::name_snake_case(), + InsertLatencyWorkTable::version(), + ); + let engine = InsertLatencyPersistenceEngine::new(config).await.unwrap(); + let persisted = InsertLatencyWorkTable::load(engine).await.unwrap(); + let (mut same, mut fresh) = (Vec::new(), Vec::new()); + let mut pages = persisted.0.data.get_page_count(); + for id in 0..ROWS { + let at = std::time::Instant::now(); + persisted + .insert(InsertLatencyRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + let took = at.elapsed().as_secs_f64() * 1e6; + let now = persisted.0.data.get_page_count(); + if now == pages { + same.push(took) + } else { + fresh.push(took) + } + pages = now; + } + report("persisted, existing page", same, ROWS as usize); + report("persisted, allocated a page", fresh, ROWS as usize); + persisted.wait_for_ops().await.expect("the queue drains"); + remove_dir_if_exists(dir.to_string()).await; + } + }); +} diff --git a/tests/persistence/local_write_bandwidth.rs b/tests/persistence/local_write_bandwidth.rs new file mode 100644 index 00000000..5d74f201 --- /dev/null +++ b/tests/persistence/local_write_bandwidth.rs @@ -0,0 +1,201 @@ +//! What WorkTable's local persistence path sustains, in bytes per second. +//! +//! Measured through `insert` and `wait_for_ops` rather than against +//! `persist_page` directly, so it counts everything the engine does to make a +//! write durable and not just the call at the bottom of it. + +use worktable::prelude::PersistedWorkTable; +use worktable::prelude::*; +use worktable_codegen::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: WriteBandwidth, + persist: true, + columns: { + id: u64 primary_key, + payload: String, + } +); + +// The same table without persistence, so the cost of being a WorkTable can be +// told apart from the cost of writing to disk. Anything this arm spends is +// spent by the persisted one too, before any page is written. +worktable!( + name: WriteBandwidthMemory, + columns: { + id: u64 primary_key, + payload: String, + } +); + +/// A page of the on-disk format, which is the unit a write actually lands in. +const PAGE: usize = 4096 * 4; + +/// Per-page checksums of every file, so two snapshots say how many pages a +/// stretch of work really wrote. +fn page_checksums(dir: &str) -> Vec<(String, Vec)> { + fn walk(dir: &std::path::Path, into: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.is_dir() { + walk(&path, into); + } else if path.is_file() { + into.push(path); + } + } + } + let mut paths = Vec::new(); + walk(std::path::Path::new(dir), &mut paths); + paths.sort(); + paths + .into_iter() + .map(|path| { + let bytes = std::fs::read(&path).expect("a table file"); + ( + path.to_string_lossy().into_owned(), + bytes.chunks(PAGE).map(crc32fast::hash).collect(), + ) + }) + .collect() +} + +/// Bytes that differ between two snapshots, counted a page at a time. +fn written_bytes(before: &[(String, Vec)], after: &[(String, Vec)]) -> u64 { + let mut pages = 0u64; + for (name, now) in after { + let then = before + .iter() + .find(|(n, _)| n == name) + .map(|(_, c)| c.as_slice()) + .unwrap_or(&[]); + pages += now + .iter() + .enumerate() + .filter(|(index, checksum)| then.get(*index) != Some(*checksum)) + .count() as u64; + } + pages * PAGE as u64 +} + +fn table_bytes(dir: &str) -> u64 { + fn walk(dir: &std::path::Path, total: &mut u64) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + if path.is_dir() { + walk(&path, total); + } else if let Ok(meta) = entry.metadata() { + *total += meta.len(); + } + } + } + let mut total = 0; + walk(std::path::Path::new(dir), &mut total); + total +} + +#[test] +#[ignore = "a measurement, not an assertion"] +fn local_write_bandwidth() { + let dir = "tests/data/local_write_bandwidth"; + let config = DiskConfig::new_with_table_name( + dir, + WriteBandwidthWorkTable::name_snake_case(), + WriteBandwidthWorkTable::version(), + ); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_io() + .enable_time() + .build() + .unwrap(); + + runtime.block_on(async { + remove_dir_if_exists(dir.to_string()).await; + let engine = WriteBandwidthPersistenceEngine::new(config.clone()).await.unwrap(); + let table = WriteBandwidthWorkTable::load(engine).await.unwrap(); + let payload = "x".repeat(4096); + + // ---- bulk load: consecutive pages, which is the batch path's shape + const ROWS: u64 = 25_000; + let at = std::time::Instant::now(); + for id in 0..ROWS { + table + .insert(WriteBandwidthRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + table.wait_for_ops().await.expect("the queue drains"); + let bulk = at.elapsed().as_secs_f64(); + let bytes = table_bytes(dir); + + // ---- scattered updates into what already exists + const UPDATES: u64 = 2_000; + let replacement = "y".repeat(4096); + let pages_before = page_checksums(dir); + let at = std::time::Instant::now(); + for n in 0..UPDATES { + // Spread across the whole table rather than a contiguous run. + let id = (n * (ROWS / UPDATES)) % ROWS; + table + .update(WriteBandwidthRow { + id, + payload: replacement.clone(), + }) + .await + .unwrap(); + } + table.wait_for_ops().await.expect("the queue drains"); + let scattered = at.elapsed().as_secs_f64(); + let scattered_bytes = written_bytes(&pages_before, &page_checksums(dir)); + + println!("table {:.1} MB on disk", bytes as f64 / 1e6); + println!( + " bulk insert, {ROWS} rows : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s", + bulk * 1e3, + bytes as f64 / 1e6 / bulk, + ROWS as f64 / bulk, + ); + println!( + " scattered update, {UPDATES} rows : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s {:.1} MB written", + scattered * 1e3, + scattered_bytes as f64 / 1e6 / scattered, + UPDATES as f64 / scattered, + scattered_bytes as f64 / 1e6, + ); + + // ---- the same inserts with nothing underneath them + let memory = WriteBandwidthMemoryWorkTable::default(); + let at = std::time::Instant::now(); + for id in 0..ROWS { + memory + .insert(WriteBandwidthMemoryRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + let in_memory = at.elapsed().as_secs_f64(); + println!( + " in memory, no persistence : {:>8.1} ms {:>7.0} MB/s {:>8.0} rows/s", + in_memory * 1e3, + bytes as f64 / 1e6 / in_memory, + ROWS as f64 / in_memory, + ); + println!( + " ^ persistence adds {:.1} ms on top of {:.1} ms of table work", + (bulk - in_memory) * 1e3, + in_memory * 1e3, + ); + + remove_dir_if_exists(dir.to_string()).await; + }); +} diff --git a/tests/persistence/mod.rs b/tests/persistence/mod.rs index e282cc9f..1bdb95ea 100644 --- a/tests/persistence/mod.rs +++ b/tests/persistence/mod.rs @@ -5,15 +5,20 @@ use worktable::worktable; mod bulk_delete_durability; mod bulk_load_stall; mod concurrent; +mod custom_page_size; mod duplicate_key_index_reload; mod exact_boundary_load; mod failure; mod in_place_durability; mod index_page; +mod insert_cost_shape; +mod insert_latency; mod insert_many; mod insert_many_bench; mod loaded_index_growth; +mod local_write_bandwidth; mod multi_row_backend_order; +mod persistence_is_what; mod read; mod recovery_load; mod same_size_in_place; diff --git a/tests/persistence/persistence_is_what.rs b/tests/persistence/persistence_is_what.rs new file mode 100644 index 00000000..ca17484a --- /dev/null +++ b/tests/persistence/persistence_is_what.rs @@ -0,0 +1,60 @@ +//! Is the persistence path waiting, or working? +//! +//! Bulk load persists at about 310 MB/s while the disk under it does gigabytes +//! and DataBucket's own write path does 500+ MB/s single threaded. So something +//! between them is the limit. CPU time against wall time says which kind of +//! limit it is: near or above wall means it is computing, well under means it +//! is waiting. + +use worktable::prelude::PersistedWorkTable; +use worktable::prelude::*; +use worktable_codegen::worktable; + +use crate::remove_dir_if_exists; + +worktable!( + name: PersistShape, + persist: true, + columns: { id: u64 primary_key, payload: String } +); + +#[test] +#[ignore = "a measurement, not an assertion"] +fn is_persistence_waiting_or_working() { + let dir = "tests/data/persistence_is_what"; + let config = DiskConfig::new_with_table_name( + dir, + PersistShapeWorkTable::name_snake_case(), + PersistShapeWorkTable::version(), + ); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + remove_dir_if_exists(dir.to_string()).await; + let engine = PersistShapePersistenceEngine::new(config).await.unwrap(); + let table = PersistShapeWorkTable::load(engine).await.unwrap(); + let payload = "x".repeat(4096); + + // Marked so the times either side can be attributed to this and not to + // building the table or tearing it down. + println!("MARK begin"); + let at = std::time::Instant::now(); + for id in 0..25_000u64 { + table + .insert(PersistShapeRow { + id, + payload: payload.clone(), + }) + .await + .unwrap(); + } + table.wait_for_ops().await.expect("the queue drains"); + println!("MARK end {:.1} ms", at.elapsed().as_secs_f64() * 1e3); + + remove_dir_if_exists(dir.to_string()).await; + }); +} diff --git a/tests/persistence/read.rs b/tests/persistence/read.rs index d56ec83c..49faf282 100644 --- a/tests/persistence/read.rs +++ b/tests/persistence/read.rs @@ -1,4 +1,4 @@ -use tokio::fs::File; +use data_bucket::DEFAULT_PAGE_STRIDE; use worktable::prelude::PersistedWorkTable; use worktable::prelude::*; @@ -10,8 +10,10 @@ use crate::remove_dir_if_exists; #[tokio::test] async fn test_info_parse() { - let mut file = File::open("tests/data/expected/test_persist/.wt.data").await.unwrap(); - let info = parse_page::, { TEST_PERSIST_INNER_SIZE as u32 }>(&mut file, 0) + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/.wt.data") + .await + .unwrap(); + let info = parse_page::, { TEST_PERSIST_INNER_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 0) .await .unwrap(); @@ -31,10 +33,10 @@ async fn test_info_parse() { #[tokio::test] async fn test_primary_index_parse() { - let mut file = File::open("tests/data/expected/test_persist/primary.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/primary.wt.idx") .await .unwrap(); - let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }>(&mut file, 2) + let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); @@ -66,10 +68,10 @@ async fn test_primary_index_parse() { #[tokio::test] async fn test_another_idx_index_parse() { - let mut file = File::open("tests/data/expected/test_persist/another_idx.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/another_idx.wt.idx") .await .unwrap(); - let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }>(&mut file, 2) + let index = parse_page::, { TEST_PERSIST_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 2) .await .unwrap(); @@ -101,10 +103,16 @@ async fn test_another_idx_index_parse() { #[tokio::test] async fn test_data_parse() { - let mut file = File::open("tests/data/expected/test_persist/.wt.data").await.unwrap(); - let data = parse_data_page::<{ TEST_PERSIST_PAGE_SIZE as u32 }, { TEST_PERSIST_INNER_SIZE }>(&mut file, 1) + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/.wt.data") .await .unwrap(); + let data = parse_data_page::< + { TEST_PERSIST_PAGE_SIZE as u32 }, + { TEST_PERSIST_INNER_SIZE }, + { TEST_PERSIST_PAGE_SIZE as u32 }, + >(&mut file, 1) + .await + .unwrap(); assert_eq!(data.header.space_id, 0.into()); assert_eq!(data.header.page_id, 1.into()); diff --git a/tests/persistence/recovery_load.rs b/tests/persistence/recovery_load.rs index 3cf8feaf..63726804 100644 --- a/tests/persistence/recovery_load.rs +++ b/tests/persistence/recovery_load.rs @@ -52,7 +52,7 @@ async fn recovery_mode_reads_valid_rows_through_a_surviving_secondary_index() { let table_dir = format!("{DIR}/{}", RecoveryLoadWorkTable::name_snake_case()); let primary_path = format!("{table_dir}/primary{WT_INDEX_EXTENSION}"); - tokio::fs::rename(&primary_path, format!("{primary_path}.damaged")) + worktable::prelude::fsx::rename(&primary_path, format!("{primary_path}.damaged")) .await .unwrap(); @@ -104,7 +104,7 @@ async fn recovery_mode_rejects_corrupt_rows_reached_through_a_secondary_index() let table_dir = format!("{CORRUPT_DIR}/{}", RecoveryLoadWorkTable::name_snake_case()); let primary_path = format!("{table_dir}/primary{WT_INDEX_EXTENSION}"); - tokio::fs::rename(&primary_path, format!("{primary_path}.damaged")) + worktable::prelude::fsx::rename(&primary_path, format!("{primary_path}.damaged")) .await .unwrap(); diff --git a/tests/persistence/s3/mod.rs b/tests/persistence/s3/mod.rs index 04957c96..c2f9040d 100644 --- a/tests/persistence/s3/mod.rs +++ b/tests/persistence/s3/mod.rs @@ -1,4 +1,6 @@ use crate::remove_dir_if_exists; +// A tokio `TcpStream`, so tokio's extension traits: this is the mock S3 +// server the test talks to, not the storage path the crate took off tokio. use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tokio::task::JoinHandle; diff --git a/tests/persistence/schema.rs b/tests/persistence/schema.rs index c815a18a..33d886c3 100644 --- a/tests/persistence/schema.rs +++ b/tests/persistence/schema.rs @@ -1,7 +1,6 @@ -use tokio::fs::File; - use super::*; use crate::remove_dir_if_exists; +use data_bucket::DEFAULT_PAGE_STRIDE; worktable!( name: SchemaMetadata, @@ -37,8 +36,10 @@ async fn generated_schema_is_persisted_and_mismatches_are_rejected() { let table = SchemaMetadataWorkTable::load(engine).await.unwrap(); table.close().await.unwrap(); - let mut file = File::open(format!("{table_path}/{}", WT_DATA_EXTENSION)).await.unwrap(); - let info = parse_page::, { PAGE_SIZE as u32 }>(&mut file, 0) + let mut file = worktable::prelude::fsx::open(format!("{table_path}/{}", WT_DATA_EXTENSION)) + .await + .unwrap(); + let info = parse_page::, { PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 0) .await .unwrap(); assert_eq!( @@ -80,8 +81,10 @@ async fn loading_a_legacy_empty_schema_does_not_rewrite_the_file() { let table = SchemaMetadataWorkTable::load(engine).await.unwrap(); table.close().await.unwrap(); - let mut file = File::open(format!("{table_path}/{}", WT_DATA_EXTENSION)).await.unwrap(); - let info = parse_page::, { PAGE_SIZE as u32 }>(&mut file, 0) + let mut file = worktable::prelude::fsx::open(format!("{table_path}/{}", WT_DATA_EXTENSION)) + .await + .unwrap(); + let info = parse_page::, { PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>(&mut file, 0) .await .unwrap(); assert!(info.inner.row_schema.is_empty()); diff --git a/tests/persistence/space_data.rs b/tests/persistence/space_data.rs index 191c0322..3abe53af 100644 --- a/tests/persistence/space_data.rs +++ b/tests/persistence/space_data.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use data_bucket::DEFAULT_PAGE_STRIDE; +use worktable::prelude::HashMap; use data_bucket::{INNER_PAGE_SIZE, Link, PAGE_SIZE, parse_general_header_by_index}; use worktable::prelude::{SpaceData, SpaceDataOps}; @@ -46,7 +47,9 @@ async fn rewriting_one_link_does_not_inflate_the_persisted_data_length() { space.save_batch_data(batch).await.unwrap(); assert_eq!(space.current_data_length, 48); - let header = parse_general_header_by_index(&mut space.data_file, 1).await.unwrap(); + let header = parse_general_header_by_index::(&mut space.data_file, 1) + .await + .unwrap(); assert_eq!(header.data_length, 48); drop(space); @@ -86,7 +89,9 @@ async fn reclaiming_thousands_of_pages_bounds_the_info_page_instead_of_corruptin assert!(kept < 2_000, "the overflow condition was not constructed"); // Page 1 must be untouched by the info persist. - let header = parse_general_header_by_index(&mut space.data_file, 1).await.unwrap(); + let header = parse_general_header_by_index::(&mut space.data_file, 1) + .await + .unwrap(); assert_eq!(header.page_type, PageType::Data); drop(space); diff --git a/tests/persistence/space_index/indexset_compatibility.rs b/tests/persistence/space_index/indexset_compatibility.rs index c9c9ba9a..6f702a5a 100644 --- a/tests/persistence/space_index/indexset_compatibility.rs +++ b/tests/persistence/space_index/indexset_compatibility.rs @@ -1,4 +1,5 @@ mod sized { + use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use data_bucket::{INNER_PAGE_SIZE, Link}; @@ -11,7 +12,7 @@ mod sized { async fn test_indexset_node_creation() { remove_file_if_exists("tests/data/space_index/indexset/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/indexset/process_create_node.wt.idx", 0.into(), 1, @@ -46,7 +47,7 @@ mod sized { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/indexset/process_insert_at.wt.idx", 0.into(), 1, @@ -81,7 +82,7 @@ mod sized { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/indexset/process_insert_at_big_amount.wt.idx", 0.into(), 1, @@ -124,6 +125,7 @@ mod sized { } mod unsized_ { + use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use crate::{check_if_files_are_same, remove_file_if_exists}; @@ -137,7 +139,7 @@ mod unsized_ { async fn test_indexset_node_creation() { remove_file_if_exists("tests/data/space_index_unsized/indexset/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/indexset/process_create_node.wt.idx", 0.into(), 1, @@ -172,7 +174,7 @@ mod unsized_ { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/indexset/process_insert_at.wt.idx", 0.into(), 1, @@ -210,7 +212,7 @@ mod unsized_ { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/indexset/process_insert_at_big_amount.wt.idx", 0.into(), 1, diff --git a/tests/persistence/space_index/unsized_write.rs b/tests/persistence/space_index/unsized_write.rs index 61175c97..da84b980 100644 --- a/tests/persistence/space_index/unsized_write.rs +++ b/tests/persistence/space_index/unsized_write.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use data_bucket::{INNER_PAGE_SIZE, Link}; @@ -14,7 +15,7 @@ mod run_first { async fn test_space_index_process_create_node() { remove_file_if_exists("tests/data/space_index_unsized/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_create_node.wt.idx", 0.into(), 1, @@ -52,7 +53,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_create_second_node.wt.idx", 0.into(), 1, @@ -90,7 +91,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_remove_node.wt.idx", 0.into(), 1, @@ -128,7 +129,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at.wt.idx", 0.into(), 1, @@ -175,7 +176,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at_big_amount.wt.idx", 0.into(), 1, @@ -249,7 +250,7 @@ async fn test_space_index_process_remove_at() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_remove_at.wt.idx", 0.into(), 1, @@ -296,7 +297,7 @@ async fn test_space_index_process_remove_at_node_id() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_remove_at_node_id.wt.idx", 0.into(), 1, @@ -344,7 +345,7 @@ async fn test_space_index_process_insert_at_with_node_id_update() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at_with_node_id_update.wt.idx", 0.into(), 1, @@ -391,7 +392,7 @@ async fn test_space_index_process_insert_at_removed_place() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_insert_at_removed_place.wt.idx", 0.into(), 1, @@ -484,7 +485,7 @@ async fn test_space_index_process_create_node_after_remove() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_create_node_after_remove.wt.idx", 0.into(), 1, @@ -522,7 +523,7 @@ async fn test_space_index_process_split_node() { ) .unwrap(); - let mut space_index = SpaceIndexUnsized::::new( + let mut space_index = SpaceIndexUnsized::::new( "tests/data/space_index_unsized/process_split_node.wt.idx", 0.into(), 1, diff --git a/tests/persistence/space_index/write.rs b/tests/persistence/space_index/write.rs index d6f8d2c6..81a0befa 100644 --- a/tests/persistence/space_index/write.rs +++ b/tests/persistence/space_index/write.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::fs::copy; use data_bucket::{INNER_PAGE_SIZE, Link}; @@ -14,7 +15,7 @@ mod run_first { async fn test_space_index_process_create_node() { remove_file_if_exists("tests/data/space_index/process_create_node.wt.idx".to_string()).await; - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_create_node.wt.idx", 0.into(), 1, @@ -52,7 +53,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_create_second_node.wt.idx", 0.into(), 1, @@ -90,7 +91,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at.wt.idx", 0.into(), 1, @@ -137,7 +138,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at_big_amount.wt.idx", 0.into(), 1, @@ -210,7 +211,7 @@ mod run_first { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_remove_node.wt.idx", 0.into(), 1, @@ -249,7 +250,7 @@ async fn test_space_index_process_insert_at_with_node_id_update() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at_with_node_id_update.wt.idx", 0.into(), 1, @@ -296,7 +297,7 @@ async fn test_space_index_process_remove_at() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_remove_at.wt.idx", 0.into(), 1, @@ -343,7 +344,7 @@ async fn test_space_index_process_remove_at_node_id() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_remove_at_node_id.wt.idx", 0.into(), 1, @@ -390,7 +391,7 @@ async fn test_space_index_process_insert_at_removed_place() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_insert_at_removed_place.wt.idx", 0.into(), 1, @@ -483,7 +484,7 @@ async fn test_space_index_process_create_node_after_remove() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_create_node_after_remove.wt.idx", 0.into(), 1, @@ -521,7 +522,7 @@ async fn test_space_index_process_split_node() { ) .unwrap(); - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/process_split_node.wt.idx", 0.into(), 1, @@ -560,10 +561,13 @@ async fn test_space_index_process_split_node() { async fn batch_split_then_max_remove_then_historical_identity_insert_applies() { remove_file_if_exists("tests/data/space_index/batch_alias.wt.idx".to_string()).await; - let mut space_index = - SpaceIndex::::new("tests/data/space_index/batch_alias.wt.idx", 0.into(), 1) - .await - .unwrap(); + let mut space_index = SpaceIndex::::new( + "tests/data/space_index/batch_alias.wt.idx", + 0.into(), + 1, + ) + .await + .unwrap(); fn link(offset: u32) -> Link { Link { @@ -636,7 +640,7 @@ async fn batch_split_then_max_remove_then_historical_identity_insert_applies() { async fn batch_replay_of_real_cdc_stream_with_splits_matches_the_source() { remove_file_if_exists("tests/data/space_index/batch_cdc_replay.wt.idx".to_string()).await; - let mut space_index = SpaceIndex::::new( + let mut space_index = SpaceIndex::::new( "tests/data/space_index/batch_cdc_replay.wt.idx", 0.into(), 1, diff --git a/tests/persistence/sync/string_secondary_index.rs b/tests/persistence/sync/string_secondary_index.rs index 93a6fafc..e4b65ffc 100644 --- a/tests/persistence/sync/string_secondary_index.rs +++ b/tests/persistence/sync/string_secondary_index.rs @@ -1,3 +1,4 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use worktable::prelude::PersistedWorkTable; use worktable::prelude::*; use worktable_codegen::worktable; @@ -104,11 +105,12 @@ fn fragmented_string_index_compacts_after_restart_before_appending() { } let index_path = format!("{path}/fragmented_string_secondary/project_idx.wt.idx"); - let mut index_file = tokio::fs::File::open(index_path).await.unwrap(); - let page = parse_page::, { INNER_PAGE_SIZE as u32 }>( - &mut index_file, - 2, - ) + let mut index_file = worktable::prelude::fsx::open(index_path).await.unwrap(); + let page = parse_page::< + UnsizedIndexPage, + { INNER_PAGE_SIZE as u32 }, + DEFAULT_PAGE_STRIDE, + >(&mut index_file, 2) .await .unwrap(); let utility_size = worktable::data_bucket::UnsizedIndexPageUtility::::persisted_size( diff --git a/tests/persistence/toc/read.rs b/tests/persistence/toc/read.rs index d36d59b0..6460f931 100644 --- a/tests/persistence/toc/read.rs +++ b/tests/persistence/toc/read.rs @@ -1,37 +1,34 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::sync::Arc; use std::sync::atomic::AtomicU32; use data_bucket::{INNER_PAGE_SIZE, Link}; -use tokio::fs::OpenOptions; use worktable::prelude::IndexTableOfContents; #[tokio::test] async fn test_index_table_of_contents_read() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/persist_index_table_of_contents.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/persist_index_table_of_contents.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(1)); - let toc = - IndexTableOfContents::::parse_from_file(&mut file, 0.into(), next_id_gen) - .await - .unwrap(); + let toc = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + next_id_gen, + ) + .await + .unwrap(); assert_eq!(toc.get(&13), Some(1.into())) } #[tokio::test] async fn test_index_table_of_contents_read_from_space() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/test_persist/primary.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/test_persist/primary.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(1)); - let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u64, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -53,14 +50,11 @@ async fn test_index_table_of_contents_read_from_space() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -83,14 +77,11 @@ async fn test_index_table_of_contents_read_from_space_index() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_after_insert() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -113,14 +104,12 @@ async fn test_index_table_of_contents_read_from_space_index_after_insert() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_updated_node_id() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_insert_at_with_node_id_update.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -143,14 +132,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_updated_node_id #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_remove_at_node_id() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_at_node_id.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -173,14 +159,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_remove_at_node_ #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_remove_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_remove_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -214,14 +197,12 @@ async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_create_node_after_remove_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index/process_create_node_after_remove.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -255,14 +236,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_create_node_aft #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_after_split_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index/process_split_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index/process_split_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(u32, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -297,36 +275,39 @@ async fn test_index_table_of_contents_read_from_space_index_after_split_node() { #[tokio::test] async fn test_truncated_table_of_contents_is_an_error_not_an_empty_index() { let path = std::env::temp_dir().join(format!("worktable-toc-truncated-{}.wt.idx", uuid::Uuid::new_v4())); - let mut file = OpenOptions::new() - .write(true) - .read(true) - .create(true) - .truncate(true) - .open(&path) - .await - .unwrap(); + let mut file = worktable::prelude::fsx::create(&path).await.unwrap(); // A DATA_LENGTH of 20 forces the table of contents to span several pages. - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); for key in 0..10 { toc.insert(key, u32::from(key).into()); } assert!(toc.pages.len() > 1, "fixture must span TOC pages"); toc.persist(&mut file).await.unwrap(); - file.sync_all().await.unwrap(); + worktable::prelude::fsx::sync_all(&mut file).await.unwrap(); // The intact file must round-trip. - let reloaded = IndexTableOfContents::::parse_from_file(&mut file, 0.into(), Arc::new(AtomicU32::new(1))) - .await - .unwrap(); + let reloaded = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + Arc::new(AtomicU32::new(1)), + ) + .await + .unwrap(); assert_eq!(reloaded.pages.len(), toc.pages.len()); // Tear the first table-of-contents page: the file still extends into its // slot, so the load must fail loudly instead of yielding an empty index. - file.set_len(data_bucket::PAGE_SIZE as u64 + 10).await.unwrap(); - let error = IndexTableOfContents::::parse_from_file(&mut file, 0.into(), Arc::new(AtomicU32::new(1))) + worktable::prelude::fsx::set_len(&mut file, data_bucket::PAGE_SIZE as u64 + 10) .await - .unwrap_err(); + .unwrap(); + let error = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + Arc::new(AtomicU32::new(1)), + ) + .await + .unwrap_err(); assert!( error.to_string().contains("table of contents page 1 failed to parse"), "unexpected error: {error:#}" @@ -334,13 +315,16 @@ async fn test_truncated_table_of_contents_is_an_error_not_an_empty_index() { // A file that never grew past page 0 is a real bootstrap and still loads // as a fresh empty table of contents. - file.set_len(0).await.unwrap(); - let bootstrapped = - IndexTableOfContents::::parse_from_file(&mut file, 0.into(), Arc::new(AtomicU32::new(1))) - .await - .unwrap(); + worktable::prelude::fsx::set_len(&mut file, 0).await.unwrap(); + let bootstrapped = IndexTableOfContents::::parse_from_file( + &mut file, + 0.into(), + Arc::new(AtomicU32::new(1)), + ) + .await + .unwrap(); assert_eq!(bootstrapped.pages.len(), 1); drop(file); - tokio::fs::remove_file(&path).await.unwrap(); + worktable::prelude::fsx::remove_file(&path).await.unwrap(); } diff --git a/tests/persistence/toc/unsized_read.rs b/tests/persistence/toc/unsized_read.rs index 1dffc0ad..ed460df7 100644 --- a/tests/persistence/toc/unsized_read.rs +++ b/tests/persistence/toc/unsized_read.rs @@ -1,20 +1,17 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use std::sync::Arc; use std::sync::atomic::AtomicU32; use data_bucket::{INNER_PAGE_SIZE, Link}; -use tokio::fs::OpenOptions; use worktable::prelude::IndexTableOfContents; #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -37,14 +34,12 @@ async fn test_index_table_of_contents_read_from_space_index_unsized() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_with_two_nodes() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_create_second_node.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(3)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -78,14 +73,11 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_with_two_nod #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_node.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -119,14 +111,11 @@ async fn test_index_table_of_contents_read_from_space_index_with_remove_node() { #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_insert_at() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_insert_at.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -149,14 +138,11 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_after_insert #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove_at() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") + let mut file = worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at.wt.idx") .await .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -179,14 +165,12 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove_at_node_id() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") - .await - .unwrap(); + let mut file = + worktable::prelude::fsx::open("tests/data/expected/space_index_unsized/process_remove_at_node_id.wt.idx") + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, @@ -209,14 +193,13 @@ async fn test_index_table_of_contents_read_from_space_index_unsized_after_remove #[tokio::test] async fn test_index_table_of_contents_read_from_space_index_unsized_after_create_node_after_remove() { - let mut file = OpenOptions::new() - .write(true) - .read(true) - .open("tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx") - .await - .unwrap(); + let mut file = worktable::prelude::fsx::open( + "tests/data/expected/space_index_unsized/process_create_node_after_remove.wt.idx", + ) + .await + .unwrap(); let next_id_gen = Arc::new(AtomicU32::new(2)); - let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }>::parse_from_file( + let toc = IndexTableOfContents::<(String, Link), { INNER_PAGE_SIZE as u32 }, DEFAULT_PAGE_STRIDE>::parse_from_file( &mut file, 0.into(), next_id_gen, diff --git a/tests/persistence/toc/write.rs b/tests/persistence/toc/write.rs index 0bb04709..910ec0e9 100644 --- a/tests/persistence/toc/write.rs +++ b/tests/persistence/toc/write.rs @@ -1,7 +1,7 @@ +use data_bucket::DEFAULT_PAGE_STRIDE; use data_bucket::INNER_PAGE_SIZE; use std::sync::Arc; use std::sync::atomic::AtomicU32; -use tokio::fs::File; use worktable::prelude::IndexTableOfContents; use crate::{check_if_files_are_same, remove_file_if_exists}; @@ -10,11 +10,14 @@ use crate::{check_if_files_are_same, remove_file_if_exists}; async fn test_persist_index_table_of_contents() { remove_file_if_exists("tests/data/persist_index_table_of_contents.wt.idx".to_string()).await; - let mut toc = IndexTableOfContents::::new(0.into(), Arc::new(AtomicU32::new(1))); + let mut toc = IndexTableOfContents::::new( + 0.into(), + Arc::new(AtomicU32::new(1)), + ); // Compile-time compatibility regression: the public API before PR #63 // returned unit, including for callers that bind the expression's type. let _: () = toc.insert(13, 1.into()); - let mut file = File::create("tests/data/persist_index_table_of_contents.wt.idx") + let mut file = worktable::prelude::fsx::create("tests/data/persist_index_table_of_contents.wt.idx") .await .unwrap(); toc.persist(&mut file).await.unwrap(); diff --git a/tests/slotted_page_requirement.rs b/tests/slotted_page_requirement.rs new file mode 100644 index 00000000..8b21460d --- /dev/null +++ b/tests/slotted_page_requirement.rs @@ -0,0 +1,204 @@ +//! What a data page has to say about itself, for beta.20. +//! +//! # Where this comes from +//! +//! beta.19 changed the on-disk format and every existing `.wt.data` had to be +//! thrown away and rebuilt, because nothing could read the old shape. That is a +//! regeneration event, and it happened on 6 September 2026 across every store on +//! this machine. +//! +//! It does not have to happen again. The reason it did is that a data page +//! cannot be read without the index that points into it: +//! +//! ```ignore +//! pub struct DataPage { +//! pub length: u32, +//! pub data: [u8; DATA_LENGTH], +//! } +//! ``` +//! +//! Rows are bump allocated into `data` and `length` is a high water mark. There +//! are no delimiters, so nothing can tell where one row ends and the next +//! begins. `empty_links_list` in the `SpaceInfoPage` records freed ranges and is +//! explicitly lossy: `bound_empty_links_list` truncates it when it outgrows the +//! info page and logs "space leak, not corruption". +//! +//! **The schema is already there and this is not asking for it again.** +//! `SpaceInfoPage` carries `row_schema`, `primary_key_fields` and +//! `secondary_index_types`, and `ensure_schema` refuses a mismatch by name. A +//! reader already knows how to decode a row. What it cannot do is find one. +//! +//! # The two things, in order of how much they matter +//! +//! 1. **A row directory in the data page**, the usual slotted layout: an +//! `(offset, length)` per row growing down from the end of the page, with a +//! count. Then a page describes itself, a reader needs no index, and the CRC +//! on that page validates the directory together with the rows it points at. +//! +//! 2. **A reader for the format beta.19 writes**, so beta.20 is an upgrade +//! rather than another regeneration. One already exists and is switched off: +//! `src/page/iterators.rs` in DataBucket, where `LinksIterator` walks index +//! pages for links and `DataIterator` follows them, decoding through +//! `row_schema`. It is 226 lines, commented out at `src/page/mod.rs:4`, and +//! enabling it produces nine errors that are bit rot rather than design: +//! `crate::IndexData` and `super::SpaceInfo` were renamed, and one call site +//! predates the API going async. +//! +//! # How the two fit together +//! +//! `DATA_VERSION` is 2 today and lives in every page's `GeneralHeader`, so it is +//! per page rather than per file. +//! +//! - **beta.20 ships both.** It writes 3 and reads 2 and 3. +//! - **beta.21 ships neither of the old ones.** The v2 path is deleted. +//! +//! So v2 is a one way ramp rather than dual support: a store is loaded through +//! it once, written back as v3, and never read that way again. It does not need +//! to be fast and it never needs append, which is most of why it is cheap. +//! +//! Two things to settle rather than discover: +//! +//! - Once a page has a directory and an index, both know where a row is and they +//! can disagree. One has to be authoritative. The directory is the better +//! candidate: it is local to the page and validated by the same CRC, where the +//! index is a separate structure with a different topology per backend. Under +//! `validate-reads` a load can compare the two and name a disagreement instead +//! of silently preferring one. +//! - Whether one file may hold both v2 and v3 pages. Per page versioning allows +//! it, which makes migration an append rather than a rewrite, but then no +//! reader may assume uniformity. +//! +//! # What is missing here, and is the next piece of work +//! +//! A committed `.wt.data` written by beta.19, so the ramp can be tested against +//! a real old file rather than against one this build just wrote. Until that +//! fixture exists, `a_store_reopens_without_being_rebuilt` below only proves the +//! current version reopens, which is the weaker half. + +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: SlottedRow, + version: 1, + persist: true, + columns: { + id: u64 primary_key autoincrement, + blob: String, + }, +); + +/// Enough rows to fill more than one page, so a directory would be doing real +/// work rather than describing a single row. +const ROWS: u64 = 4_000; + +async fn filled(dir: &str) -> SlottedRowWorkTable { + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir).expect("a directory"); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("an engine"); + let table = SlottedRowWorkTable::load(engine).await.expect("a table"); + + for n in 0..ROWS { + table + .insert(SlottedRowRow { + id: table.get_next_pk().into(), + blob: format!("row {n}, long enough to make the page boundaries interesting"), + }) + .await + .expect("a row"); + } + table.wait_for_ops().await.expect("the queue drains"); + table +} + +fn data_file(dir: &str) -> std::path::PathBuf { + std::path::Path::new(dir) + .join(SlottedRowWorkTable::name_snake_case()) + .join(".wt.data") +} + +/// A data page should say where its rows are, without an index. +/// +/// **This is the beta.20 requirement.** The check goes straight at the bytes on +/// purpose. Reading the page through the engine would prove only that the index +/// still works, and the index is exactly what a self describing page is supposed +/// to make unnecessary. +/// +/// Written against bytes rather than against an API that does not exist yet, so +/// this file compiles today and fails on the missing behaviour rather than on a +/// missing symbol. +#[tokio::test] +#[ignore = "beta.20: a data page carries no row directory"] +async fn a_data_page_says_where_its_rows_are() { + let dir = "tests/data/slotted_page/self_describing"; + let table = filled(dir).await; + table.close().await.expect("the table closes"); + + let bytes = std::fs::read(data_file(dir)).expect("the file"); + assert!( + bytes.len() > PAGE_SIZE, + "the fixture has to span pages: {} bytes", + bytes.len() + ); + + // A slotted page keeps its directory at the end: a row count in the last + // four bytes, then that many (offset, length) pairs growing back up. Any + // layout would do; what matters is that something in the page delimits the + // rows. Today the tail is write padding, so this reads zero. + let mut described = 0usize; + let (pages, _) = bytes.as_chunks::(); + for page in pages.iter().skip(1) { + let mut tail = [0u8; 4]; + tail.copy_from_slice(&page[PAGE_SIZE - 4..]); + described += u32::from_le_bytes(tail) as usize; + } + + assert_eq!( + described, ROWS as usize, + "no page says how many rows it holds, so the {ROWS} rows in this file \ + cannot be found without the index. A row directory in the data page is \ + what makes a page readable on its own, and what makes the next format \ + change an upgrade instead of a regeneration." + ); + let _ = std::fs::remove_dir_all(dir); +} + +/// A store reopens without being deleted first. +/// +/// **Not ignored, and passing.** It guards the property at the current version, +/// so a format change that breaks reopening trips here rather than in somebody's +/// deploy. It is the weaker half of the requirement: proving beta.20 can read +/// beta.19 needs a beta.19 file committed as a fixture, which does not exist +/// yet. +#[tokio::test] +async fn a_store_reopens_without_being_rebuilt() { + let dir = "tests/data/slotted_page/reopen"; + let table = filled(dir).await; + table.close().await.expect("the table closes"); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("an engine"); + let reopened = SlottedRowWorkTable::load(engine) + .await + .expect("a store reopens rather than needing to be rebuilt"); + + assert_eq!( + reopened.select_all().execute().expect("a read").len(), + ROWS as usize, + "no rows are lost reopening a store" + ); + reopened.close().await.expect("the table closes"); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/tests/worktable/concurrency.rs b/tests/worktable/concurrency.rs index 51eceb4f..75382664 100644 --- a/tests/worktable/concurrency.rs +++ b/tests/worktable/concurrency.rs @@ -115,7 +115,7 @@ macro_rules! backend_suite { scope.spawn(move || { for n in 0..per_writer { let id = w * per_writer + n; - futures::executor::block_on(table.insert(row(id))).expect("insert"); + nagoya::block_on(table.insert(row(id))).expect("insert"); // Read while the others write, so the scan and // the mutations actually overlap. let _ = table.select(id); @@ -166,7 +166,7 @@ macro_rules! backend_suite { scope.spawn(move || { for n in 0..per_writer { let id = seed + w * per_writer + n; - futures::executor::block_on(table.insert(row(id))).expect("insert"); + nagoya::block_on(table.insert(row(id))).expect("insert"); } }); } @@ -218,11 +218,11 @@ macro_rules! backend_suite { let rows: Vec<_> = (chunk..(chunk + 16).min(per_writer)) .map(|n| row(base + n)) .collect(); - futures::executor::block_on(table.insert_many(rows)).expect("insert_many"); + nagoya::block_on(table.insert_many(rows)).expect("insert_many"); } } else { for n in 0..per_writer { - futures::executor::block_on(table.insert(row(base + n))).expect("insert"); + nagoya::block_on(table.insert(row(base + n))).expect("insert"); } } }); @@ -263,7 +263,7 @@ macro_rules! backend_suite { scope.spawn(move || { // Distinct primary keys, one shared payload. let contended = ConcRow { id: w, payload: 42, bucket: 0 }; - if futures::executor::block_on(table.insert(contended)).is_ok() { + if nagoya::block_on(table.insert(contended)).is_ok() { winners.fetch_add(1, Ordering::Release); } }); @@ -363,8 +363,8 @@ macro_rules! backend_suite { (base + WINDOW, base) }; for i in 0..WINDOW { - futures::executor::block_on(table.delete(from + i)).expect("delete"); - futures::executor::block_on(table.insert(row(to + i))).expect("insert"); + nagoya::block_on(table.delete(from + i)).expect("delete"); + nagoya::block_on(table.insert(row(to + i))).expect("insert"); } } finished.fetch_add(1, Ordering::Release); @@ -452,7 +452,7 @@ macro_rules! backend_suite { let table = Arc::clone(&table); scope.spawn(move || { for n in 0..per_writer { - futures::executor::block_on(table.insert(row(w * per_writer + n))).expect("insert"); + nagoya::block_on(table.insert(row(w * per_writer + n))).expect("insert"); } }); } diff --git a/tests/worktable/index/insert.rs b/tests/worktable/index/insert.rs index f0ae7c85..b0615d77 100644 --- a/tests/worktable/index/insert.rs +++ b/tests/worktable/index/insert.rs @@ -228,7 +228,7 @@ async fn insert_when_unique_violated() { attr3: 123456789, attr4: row_new_attr_4.clone(), }; - assert!(futures::executor::block_on(shared.insert(row)).is_err()); + assert!(nagoya::block_on(shared.insert(row)).is_err()); } }); @@ -315,7 +315,7 @@ async fn insert_when_pk_violated() { attr3: 123456789, attr4: "Attribute__4".to_string(), }; - assert!(futures::executor::block_on(shared.insert(row)).is_err()); + assert!(nagoya::block_on(shared.insert(row)).is_err()); } }); diff --git a/tests/worktable/key_widths.rs b/tests/worktable/key_widths.rs index f0429fcb..349d0a35 100644 --- a/tests/worktable/key_widths.rs +++ b/tests/worktable/key_widths.rs @@ -66,7 +66,7 @@ macro_rules! width_case { ); // And the entry comes out again. - futures::executor::block_on(table.delete(2u64)).expect("delete"); + nagoya::block_on(table.delete(2u64)).expect("delete"); assert!( table.select_by_key(42 as $key).is_none(), "{}: {} index still resolves a deleted row", diff --git a/tests/worktable/nonunique_arctic.rs b/tests/worktable/nonunique_arctic.rs index bb9505ec..fcefb796 100644 --- a/tests/worktable/nonunique_arctic.rs +++ b/tests/worktable/nonunique_arctic.rs @@ -165,7 +165,7 @@ async fn concurrent_inserts_and_deletes_keep_the_index_consistent() { for n in 0..per_writer { let edge = (writer as u128) << 64 | n as u128; let source = (n as u128) % keys; - futures::executor::block_on(table.insert(row(&table, source, edge, n))).unwrap(); + nagoya::block_on(table.insert(row(&table, source, edge, n))).unwrap(); // Interleave point reads to race the writers. let _ = table.select_by_source_hash(source).execute().unwrap(); } diff --git a/tests/worktable/partitioned.rs b/tests/worktable/partitioned.rs index 56df81c2..672c1e32 100644 --- a/tests/worktable/partitioned.rs +++ b/tests/worktable/partitioned.rs @@ -101,7 +101,7 @@ async fn insert_with_a_custom_initialiser_runs_once_per_key() { .partition_or_insert_with(11, || { let t = PriceWorkTable::default(); for e in 0..3u8 { - futures::executor::block_on(t.insert(row(e, e as f64))).unwrap(); + nagoya::block_on(t.insert(row(e, e as f64))).unwrap(); } t }) @@ -177,7 +177,7 @@ fn concurrent_creation_and_reading_is_sound() { let table = prices.partition_or_create(k).unwrap(); // Every thread writes the same row for a key, so whichever // wins the insert the value must match the key. - let _ = futures::executor::block_on(table.insert(row(0, k as f64))); + let _ = nagoya::block_on(table.insert(row(0, k as f64))); let got = prices.partition(k).unwrap().select(0).unwrap(); assert_eq!(got.bid, k as f64, "thread {t} saw a torn partition at {k}"); } @@ -426,7 +426,7 @@ async fn concurrent_writers_on_disjoint_partitions_do_not_interfere() { std::thread::spawn(move || { let table = prices.partition_or_create(t).unwrap(); for e in 0..ROWS { - futures::executor::block_on(table.insert(row(e, t as f64 * 1000.0 + e as f64))).unwrap(); + nagoya::block_on(table.insert(row(e, t as f64 * 1000.0 + e as f64))).unwrap(); } }) }) @@ -490,7 +490,7 @@ async fn readers_survive_partitions_being_removed_under_them() { let t = prices .partition_or_insert_with(k, || { let t = PriceWorkTable::default(); - futures::executor::block_on(t.insert(row(0, k as f64))).unwrap(); + nagoya::block_on(t.insert(row(0, k as f64))).unwrap(); t }) .unwrap();