From 7068c8d082b386ca3f0aab3eb727f981efcd1a16 Mon Sep 17 00:00:00 2001 From: meh Date: Sat, 5 Sep 2026 06:48:39 +0700 Subject: [PATCH 1/5] Resolve the beta18 stack through local caret dependencies --- Cargo.toml | 40 +++++++++++++++++++++------------------- codegen/Cargo.toml | 4 ++-- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c074df20..1108852a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["codegen", "dsl", "examples", "performance_measurement", "performance [package] name = "worktable" -version = "1.0.0-beta.17" +version = "1.0.0-beta.18" edition = "2024" authors = ["Handy-caT"] license = "MIT" @@ -15,7 +15,11 @@ keywords = ["database", "embedded", "in-memory", "index", "storage"] categories = ["database-implementations", "data-structures", "caching"] [features] -default = ["wti-predictable-search"] +default = ["wti-predictable-search", "arctic-ps-reclaim"] +# Compatibility name: WorkTable's Arctic adapter always uses ps-reclaim. Keep +# this feature so existing manifests do not break, but do not allow a +# `--no-default-features` build to silently select Arctic's no-op SMR. +arctic-ps-reclaim = [] perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"] s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktable_codegen/s3-support"] # Moves unique WorkTablesIndex structural CDC work out of the table mutation @@ -33,34 +37,31 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +arc-swap = "1" async-trait = "0.1" -arctic = { package = "arctic-wt", version = "0.1" } -congee = { package = "congee-wt", version = "0.4" } +arctic = { package = "arctic-wt", version = "^0.1", path = "../arctic-wt", default-features = false, features = ["smr-ps-reclaim"] } +congee = { package = "congee-wt", version = "^0.4", path = "../congee-wt" } convert_case = "0.6" crc32fast = "1" -# Already in the dependency graph transitively (indexset's concurrent -# structures); used directly for read-side grace periods. -data_bucket = "0.5" -# data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" } -# data_bucket = { path = "../DataBucket", version = "0.3.14" } +data_bucket = { version = "^0.5", path = "../DataBucket" } derive_more = { version = "2", features = ["from", "error", "display", "debug", "into"] } eyre = "0.6" fastrand = "2" futures = "0.3" -indexset = { package = "WorkTablesIndex", version = "0.0", default-features = false, features = ["concurrent", "cdc", "multimap"] } +indexset = { package = "WorkTablesIndex", version = "^0.0", path = "../WorkTablesIndex", default-features = false, features = ["concurrent", "cdc", "multimap"] } vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"] } # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] } -# indexset = { package = "wt-indexset", version = "=0.12.12", features = ["concurrent", "cdc", "multimap"] } +# indexset = { package = "wt-indexset", version = "^0.12", features = ["concurrent", "cdc", "multimap"] } log = "0.4" ordered-float = "5" parking_lot = "0.12" -performance_measurement = { path = "performance_measurement", version = "0.1.0", optional = true } -performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true } +performance_measurement = { path = "performance_measurement", version = "^0.1", optional = true } +performance_measurement_codegen = { path = "performance_measurement/codegen", version = "^0.1", optional = true } prettytable-rs = "0.10" psc-nanoid = { version = "3", features = ["rkyv", "packed"] } rkyv = { version = "0.8", features = ["uuid-1"] } reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] } -ps-reclaim = "0.1" +ps-reclaim = { version = "^0.1", path = "../ps-reclaim" } rustc-hash = "2" rusty-s3 = { version = "0.10", optional = true } smart-default = "0.7" @@ -69,14 +70,15 @@ tracing = "0.1" url = { version = "2", optional = true } uuid = { version = "1", features = ["v4", "v7"] } walkdir = { version = "2", optional = true } -# Exact for the same reason as `worktable_dsl` in codegen/Cargo.toml: carets do -# not match pre-releases, and these move as one train. -worktable_codegen = { path = "codegen", version = "=1.0.0-beta.17" } +# These pre-release workspace crates move as one train. The explicit caret +# keeps the dependency policy consistent while the local path selects this +# checkout during validation. +worktable_codegen = { path = "codegen", version = "^1.0.0-beta.18" } # Re-exported below. Each generated table carries its declaration as a const # whose documentation says to read it with `worktable_dsl::Schema::parse`; that # instruction is only true if a plain `worktable` dependency can reach the -# crate. Exact for the same pre-release reason as codegen. -worktable_dsl = { path = "dsl", version = "=1.0.0-beta.18" } +# crate. +worktable_dsl = { path = "dsl", version = "^1.0.0-beta.18" } [dev-dependencies] chrono = "0.4" diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index ec43fb4f..bafbabba 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "worktable_codegen" -version = "1.0.0-beta.17" +version = "1.0.0-beta.18" edition = "2024" license = "MIT" description = "Proc-macro companion crate for worktable: the worktable! macro and its derives." @@ -25,7 +25,7 @@ proc-macro = true # these three crates are one release train: the macro generates code against a # specific runtime, so they must move together. Every other dependency here is # a caret at minor granularity. -worktable_dsl = { path = "../dsl", version = "=1.0.0-beta.18" } +worktable_dsl = { path = "../dsl", version = "^1.0.0-beta.18" } rkyv = { version = "0.8" } syn = { version = "2", features = ["full"] } quote = "1" From 7b22c2b6cc5dfb79594c2c35024e4cb7c1e0d989 Mon Sep 17 00:00:00 2001 From: meh Date: Sat, 5 Sep 2026 06:52:32 +0700 Subject: [PATCH 2/5] Complete beta18 memory reclamation and backend validation --- CHANGELOG.md | 41 +- codegen/src/generators/in_memory/wrapper.rs | 28 +- codegen/src/generators/index_backend.rs | 7 +- codegen/src/generators/persist/table/impls.rs | 43 +- codegen/src/generators/persist/table/mod.rs | 21 + codegen/src/generators/persist/wrapper.rs | 28 +- .../src/generators/read_only/table/impls.rs | 37 +- codegen/src/generators/read_only/table/mod.rs | 39 +- codegen/src/generators/read_only/wrapper.rs | 28 +- codegen/src/persist_table/generator/mod.rs | 1 + codegen/src/persist_table/generator/space.rs | 6 +- .../persist_table/generator/space_file/mod.rs | 55 +- .../generator/space_file/worktable_impls.rs | 39 +- codegen/src/persist_table/parser.rs | 5 + codegen/src/worktable/mod.rs | 10 +- docs/beta18-validation.md | 298 +++++++++ dsl/src/validate.rs | 2 +- src/in_memory/data.rs | 237 +++++++- src/in_memory/mod.rs | 3 +- src/in_memory/pages.rs | 573 +++++++++--------- src/in_memory/publication.rs | 93 --- src/in_memory/row.rs | 44 +- src/index/arctic.rs | 230 ++++--- src/index/arctic_multi.rs | 20 +- src/index/mod.rs | 2 +- src/index/primary_index.rs | 433 ++----------- src/lib.rs | 26 +- src/mem_stat/mod.rs | 45 +- src/persistence/mod.rs | 13 +- src/persistence/space/art_index.rs | 223 ++++++- src/persistence/space/mod.rs | 4 +- src/table/mod.rs | 30 +- src/table/vacuum/vacuum.rs | 84 +-- src/util/epoch.rs | 6 +- tests/generation_swap_requirement.rs | 167 +++++ tests/worktable/upsert.rs | 17 +- tests/worktable/vacuum_invariants.rs | 28 +- 37 files changed, 1861 insertions(+), 1105 deletions(-) create mode 100644 docs/beta18-validation.md delete mode 100644 src/in_memory/publication.rs create mode 100644 tests/generation_swap_requirement.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2caaa26f..1672a46b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,45 @@ Change Log ========== +## [1.0.0-beta.18] + +### Added + +- Selectable `worktables_index`, `arctic`, and `congee` backends for generated + primary and supported secondary indexes, including persisted topology load. +- Async and batched insert/delete paths, with bulk-mutation signaling used by + reactive vacuum scheduling. +- `MemStat` for generated persisted/read-only tables and Arc-owned + `unload_gracefully` for generation swaps. +- Strict persisted-state validation, schema metadata, recovery loading, and + Arctic string/non-unique index support. + +### Changed + +- WorkTable row/page reclamation uses the local `ps-reclaim` domain for every + index backend; the WorkTable Arctic adapter also selects Arctic's + `ps-reclaim` SMR exclusively. +- Readers now synchronize on the exact physical cell. Unrelated rows cannot + block because of a hashed lock collision. +- Vacuum discovers move candidates from a transient primary-index snapshot and + keeps only one live-cell counter per page, removing the previous four-byte + per-row directory. +- Vacuum waits for two quiet observations after mutation activity and yields + throughout a bulk mutation instead of competing with foreground work. +- The archived wrapper retains the beta.17 inner-row position so legacy stores + without bundled schema metadata remain readable. + +### Fixed + +- Torn reads and premature physical-link reuse during concurrent update, + delete, and vacuum activity. +- In-place replacement now preserves the embedded cell lock byte while copying + the rest of the archived row. +- Whole-map Arctic destruction uses an unordered physical drain instead of + repeatedly searching for the next logical key. +- Persisted primary/secondary index reconstruction and validation failures that + could otherwise expose missing, duplicate, or mismatched rows. + ## [0.4.1] ### Added @@ -57,4 +96,4 @@ Change Log ### Fixed -- `Clippy` errors in macro declaration about unused `Result`'s. \ No newline at end of file +- `Clippy` errors in macro declaration about unused `Result`'s. diff --git a/codegen/src/generators/in_memory/wrapper.rs b/codegen/src/generators/in_memory/wrapper.rs index 48c31da7..1c40617a 100644 --- a/codegen/src/generators/in_memory/wrapper.rs +++ b/codegen/src/generators/in_memory/wrapper.rs @@ -25,12 +25,12 @@ impl InMemoryGenerator { quote! { #[derive(rkyv::Archive, Debug, rkyv::Deserialize, rkyv::Serialize)] + #[rkyv(attr(repr(C)))] #[repr(C)] pub struct #wrapper_ident { inner: #row_ident, - is_ghosted: bool, - is_deleted: bool, - is_in_vacuum_process: bool, + publication_flags: u8, + cell_state: CellState, } } } @@ -48,23 +48,22 @@ impl InMemoryGenerator { } fn is_ghosted(&self) -> bool { - self.is_ghosted + self.publication_flags & 1 != 0 } fn is_vacuumed(&self) -> bool { - self.is_in_vacuum_process + self.publication_flags & 4 != 0 } fn is_deleted(&self) -> bool { - self.is_deleted + self.publication_flags & 2 != 0 } fn from_inner(inner: #row_ident) -> Self { Self { inner, - is_ghosted: true, - is_deleted: false, - is_in_vacuum_process: false, + publication_flags: 1, + cell_state: CellState, } } } @@ -89,17 +88,20 @@ impl InMemoryGenerator { quote! { impl ArchivedRowWrapper for #row_ident { + unsafe fn cell_state_ptr(this: *mut Self) -> *mut std::sync::atomic::AtomicU8 { + unsafe { std::ptr::addr_of_mut!((*this).cell_state).cast() } + } fn unghost(&mut self) { - self.is_ghosted = false; + self.publication_flags &= !1; } fn set_in_vacuum_process(&mut self) { - self.is_in_vacuum_process = true; + self.publication_flags |= 4; } fn delete(&mut self) { - self.is_deleted = true; + self.publication_flags |= 2; } fn is_deleted(&self) -> bool { - self.is_deleted + self.publication_flags & 2 != 0 } } } diff --git a/codegen/src/generators/index_backend.rs b/codegen/src/generators/index_backend.rs index 833e7422..edb61fc2 100644 --- a/codegen/src/generators/index_backend.rs +++ b/codegen/src/generators/index_backend.rs @@ -108,8 +108,13 @@ pub(crate) fn primary_key_backend_impl( } IndexBackend::Arctic => { let field = single_supported_field(backend, fields, supported_types(backend))?; + let derive = if primitive_name(field).as_deref() == Some("String") { + quote! {} + } else { + quote! { Copy, } + }; Ok(( - quote! { Copy, }, + derive, quote! { impl ArcticKey for #primary_key { type Raw = <#field as ArcticKey>::Raw; diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index b3f0a06f..39e453b2 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -252,40 +252,37 @@ impl PersistGenerator { } else { quote! { IndexMap } }; - let index_setup = if pk_types_unsized { + let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex { - pk_map: #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name), - reverse_pk_map: IndexMap::new(), - }); + inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + PersistentArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default() + )); + } + } else if pk_types_unsized { + quote! { + inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name) + )); } } else { match self.columns.primary_index_backend { crate::common::model::IndexBackend::WorktablesIndex => quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex { - pk_map: #wti_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size), - reverse_pk_map: IndexMap::new(), - }); + inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + #wti_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) + )); }, crate::common::model::IndexBackend::Indexset => quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex { - pk_map: UpstreamIndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size), - reverse_pk_map: IndexMap::new(), - }); - }, - crate::common::model::IndexBackend::Arctic => quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex { - pk_map: PersistentArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default(), - reverse_pk_map: IndexMap::new(), - }); + inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + UpstreamIndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) + )); }, + crate::common::model::IndexBackend::Arctic => unreachable!("handled before variable-size dispatch"), crate::common::model::IndexBackend::Congee => quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex { - pk_map: PersistentCongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default(), - reverse_pk_map: IndexMap::new(), - }); + inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + PersistentCongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default() + )); }, } }; diff --git a/codegen/src/generators/persist/table/mod.rs b/codegen/src/generators/persist/table/mod.rs index 2efb5714..745ca8d1 100644 --- a/codegen/src/generators/persist/table/mod.rs +++ b/codegen/src/generators/persist/table/mod.rs @@ -14,6 +14,7 @@ impl PersistGenerator { let page_size_consts = self.gen_page_size_consts(); let version_const = self.gen_version_const(); let type_ = self.gen_table_type()?; + let mem_stat_impl = self.gen_table_mem_stat_impl(); let impl_ = self.gen_table_impl(); let index_fns = self.gen_table_index_fns()?; let select_query_executor_impl = self.gen_table_select_query_executor_impl(); @@ -23,6 +24,7 @@ impl PersistGenerator { #page_size_consts #version_const #type_ + #mem_stat_impl #impl_ #index_fns #select_query_executor_impl @@ -30,6 +32,21 @@ impl PersistGenerator { }) } + fn gen_table_mem_stat_impl(&self) -> TokenStream { + let ident = WorktableNameGenerator::from_table_name(self.name.to_string()).get_work_table_ident(); + quote! { + impl MemStat for #ident { + fn heap_size(&self) -> usize { + self.0.heap_size() + } + + fn used_size(&self) -> usize { + self.0.used_size() + } + } + } + } + fn gen_page_size_consts(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let page_const_name = name_generator.get_page_size_const_ident(); @@ -97,6 +114,10 @@ impl PersistGenerator { #[table(pk_unsized, pk_wti_logical)] } } + (true, crate::common::model::IndexBackend::Arctic) => quote! { + #[derive(Debug, PersistTable)] + #[table(pk_arctic_string)] + }, (true, _) => quote! { #[derive(Debug, PersistTable)] #[table(pk_unsized)] diff --git a/codegen/src/generators/persist/wrapper.rs b/codegen/src/generators/persist/wrapper.rs index 8308495a..222c2dd1 100644 --- a/codegen/src/generators/persist/wrapper.rs +++ b/codegen/src/generators/persist/wrapper.rs @@ -25,12 +25,12 @@ impl PersistGenerator { quote! { #[derive(rkyv::Archive, Debug, rkyv::Deserialize, rkyv::Serialize)] + #[rkyv(attr(repr(C)))] #[repr(C)] pub struct #wrapper_ident { inner: #row_ident, - is_ghosted: bool, - is_deleted: bool, - is_in_vacuum_process: bool, + publication_flags: u8, + cell_state: CellState, } } } @@ -48,23 +48,22 @@ impl PersistGenerator { } fn is_ghosted(&self) -> bool { - self.is_ghosted + self.publication_flags & 1 != 0 } fn is_vacuumed(&self) -> bool { - self.is_in_vacuum_process + self.publication_flags & 4 != 0 } fn is_deleted(&self) -> bool { - self.is_deleted + self.publication_flags & 2 != 0 } fn from_inner(inner: #row_ident) -> Self { Self { inner, - is_ghosted: true, - is_deleted: false, - is_in_vacuum_process: false, + publication_flags: 1, + cell_state: CellState, } } } @@ -89,17 +88,20 @@ impl PersistGenerator { quote! { impl ArchivedRowWrapper for #row_ident { + unsafe fn cell_state_ptr(this: *mut Self) -> *mut std::sync::atomic::AtomicU8 { + unsafe { std::ptr::addr_of_mut!((*this).cell_state).cast() } + } fn unghost(&mut self) { - self.is_ghosted = false; + self.publication_flags &= !1; } fn set_in_vacuum_process(&mut self) { - self.is_in_vacuum_process = true; + self.publication_flags |= 4; } fn delete(&mut self) { - self.is_deleted = true; + self.publication_flags |= 2; } fn is_deleted(&self) -> bool { - self.is_deleted + self.publication_flags & 2 != 0 } } } diff --git a/codegen/src/generators/read_only/table/impls.rs b/codegen/src/generators/read_only/table/impls.rs index d23bbf51..6e4577b6 100644 --- a/codegen/src/generators/read_only/table/impls.rs +++ b/codegen/src/generators/read_only/table/impls.rs @@ -235,25 +235,34 @@ impl ReadOnlyGenerator { }) .collect::>(); let pk_types_unsized = is_unsized_vec(pk_types); - let pk_map = match self.columns.primary_index_backend { - crate::common::model::IndexBackend::Indexset => quote! { UpstreamIndexMap }, - _ => quote! { IndexMap }, - }; - - let index_setup = if pk_types_unsized { + let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic { + quote! { + inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + ArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default() + )); + } + } else if self.columns.primary_index_backend == crate::common::model::IndexBackend::Congee { quote! { - inner.primary_index = std::sync::Arc::new(PrimaryIndex { - pk_map: IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name), - reverse_pk_map: IndexMap::new(), - }); + inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + CongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default() + )); + } + } else if pk_types_unsized { + quote! { + inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name) + )); } } else { + let pk_map = match self.columns.primary_index_backend { + crate::common::model::IndexBackend::Indexset => quote! { UpstreamIndexMap }, + _ => quote! { IndexMap }, + }; quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - inner.primary_index = std::sync::Arc::new(PrimaryIndex { - pk_map: #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size), - reverse_pk_map: IndexMap::new(), - }); + inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map( + #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size) + )); } }; diff --git a/codegen/src/generators/read_only/table/mod.rs b/codegen/src/generators/read_only/table/mod.rs index 72bc8c0d..4079e981 100644 --- a/codegen/src/generators/read_only/table/mod.rs +++ b/codegen/src/generators/read_only/table/mod.rs @@ -14,6 +14,7 @@ impl ReadOnlyGenerator { let page_size_consts = self.gen_page_size_consts(); let version_const = self.gen_version_const(); let type_ = self.gen_table_type()?; + let mem_stat_impl = self.gen_table_mem_stat_impl(); let impl_ = self.gen_table_impl(); let index_fns = self.gen_table_index_fns()?; let select_query_executor_impl = self.gen_table_select_query_executor_impl(); @@ -23,6 +24,7 @@ impl ReadOnlyGenerator { #page_size_consts #version_const #type_ + #mem_stat_impl #impl_ #index_fns #select_query_executor_impl @@ -30,6 +32,21 @@ impl ReadOnlyGenerator { }) } + fn gen_table_mem_stat_impl(&self) -> TokenStream { + let ident = WorktableNameGenerator::from_table_name(self.name.to_string()).get_work_table_ident(); + quote! { + impl MemStat for #ident { + fn heap_size(&self) -> usize { + self.0.heap_size() + } + + fn used_size(&self) -> usize { + self.0.used_size() + } + } + } + } + fn gen_page_size_consts(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let page_const_name = name_generator.get_page_size_const_ident(); @@ -84,20 +101,32 @@ impl ReadOnlyGenerator { // shape of the table (no persistence engine or task, sync `into_worktable`), the // second selects the unsized primary index. A read-only table with an unsized key // needs both, so `read_only` is unconditional here. - let derive = match (pk_types_unsized, pk_upstream) { - (true, true) => quote! { + let derive = match (pk_types_unsized, self.columns.primary_index_backend, pk_upstream) { + (true, crate::common::model::IndexBackend::Arctic, _) => quote! { + #[derive(Debug, PersistTable)] + #[table(read_only, pk_arctic_string)] + }, + (false, crate::common::model::IndexBackend::Arctic, _) => quote! { + #[derive(Debug, PersistTable)] + #[table(read_only, pk_arctic)] + }, + (false, crate::common::model::IndexBackend::Congee, _) => quote! { + #[derive(Debug, PersistTable)] + #[table(read_only, pk_congee)] + }, + (true, _, true) => quote! { #[derive(Debug, PersistTable)] #[table(read_only, pk_unsized, pk_upstream)] }, - (true, false) => quote! { + (true, _, false) => quote! { #[derive(Debug, PersistTable)] #[table(read_only, pk_unsized)] }, - (false, true) => quote! { + (false, _, true) => quote! { #[derive(Debug, PersistTable)] #[table(read_only, pk_upstream)] }, - (false, false) => quote! { + (false, _, false) => quote! { #[derive(Debug, PersistTable)] #[table(read_only)] }, diff --git a/codegen/src/generators/read_only/wrapper.rs b/codegen/src/generators/read_only/wrapper.rs index 9f69d5b4..236b0cff 100644 --- a/codegen/src/generators/read_only/wrapper.rs +++ b/codegen/src/generators/read_only/wrapper.rs @@ -25,12 +25,12 @@ impl ReadOnlyGenerator { quote! { #[derive(rkyv::Archive, Debug, rkyv::Deserialize, rkyv::Serialize)] + #[rkyv(attr(repr(C)))] #[repr(C)] pub struct #wrapper_ident { inner: #row_ident, - is_ghosted: bool, - is_deleted: bool, - is_in_vacuum_process: bool, + publication_flags: u8, + cell_state: CellState, } } } @@ -48,23 +48,22 @@ impl ReadOnlyGenerator { } fn is_ghosted(&self) -> bool { - self.is_ghosted + self.publication_flags & 1 != 0 } fn is_vacuumed(&self) -> bool { - self.is_in_vacuum_process + self.publication_flags & 4 != 0 } fn is_deleted(&self) -> bool { - self.is_deleted + self.publication_flags & 2 != 0 } fn from_inner(inner: #row_ident) -> Self { Self { inner, - is_ghosted: true, - is_deleted: false, - is_in_vacuum_process: false, + publication_flags: 1, + cell_state: CellState, } } } @@ -89,17 +88,20 @@ impl ReadOnlyGenerator { quote! { impl ArchivedRowWrapper for #row_ident { + unsafe fn cell_state_ptr(this: *mut Self) -> *mut std::sync::atomic::AtomicU8 { + unsafe { std::ptr::addr_of_mut!((*this).cell_state).cast() } + } fn unghost(&mut self) { - self.is_ghosted = false; + self.publication_flags &= !1; } fn set_in_vacuum_process(&mut self) { - self.is_in_vacuum_process = true; + self.publication_flags |= 4; } fn delete(&mut self) { - self.is_deleted = true; + self.publication_flags |= 2; } fn is_deleted(&self) -> bool { - self.is_deleted + self.publication_flags & 2 != 0 } } } diff --git a/codegen/src/persist_table/generator/mod.rs b/codegen/src/persist_table/generator/mod.rs index c93e091e..1472000e 100644 --- a/codegen/src/persist_table/generator/mod.rs +++ b/codegen/src/persist_table/generator/mod.rs @@ -13,6 +13,7 @@ pub struct PersistTableAttributes { pub read_only: bool, pub pk_upstream: bool, pub pk_arctic: bool, + pub pk_arctic_string: bool, pub pk_congee: bool, pub pk_wti_logical: bool, pub row_schema: Vec<(String, String)>, diff --git a/codegen/src/persist_table/generator/space.rs b/codegen/src/persist_table/generator/space.rs index dc2d279d..2a9e1718 100644 --- a/codegen/src/persist_table/generator/space.rs +++ b/codegen/src/persist_table/generator/space.rs @@ -36,7 +36,11 @@ impl Generator { let space_secondary_indexes = name_generator.get_space_secondary_index_ident(); let space_secondary_indexes_events = name_generator.get_space_secondary_index_events_ident(); let avt_index_ident = name_generator.get_available_indexes_ident(); - let space_index_type = if self.attributes.pk_unsized && self.attributes.pk_wti_logical { + let space_index_type = if self.attributes.pk_arctic_string { + quote! { + SpaceArcticStringIndex<#primary_key_type, { #inner_const_name as u32 }>, + } + } else if self.attributes.pk_unsized && self.attributes.pk_wti_logical { quote! { SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }>, } diff --git a/codegen/src/persist_table/generator/space_file/mod.rs b/codegen/src/persist_table/generator/space_file/mod.rs index 027b58c3..c1f99e5d 100644 --- a/codegen/src/persist_table/generator/space_file/mod.rs +++ b/codegen/src/persist_table/generator/space_file/mod.rs @@ -28,7 +28,11 @@ impl Generator { let inner_const_name = name_generator.get_page_inner_size_const_ident(); let pk_type = name_generator.get_primary_key_type_ident(); let space_file_ident = name_generator.get_space_file_ident(); - let primary_index = if self.attributes.pk_unsized { + let primary_index = if self.attributes.pk_arctic_string { + quote! { + pub primary_index: PersistentArcticIndex<#pk_type, OffsetEqLink<#inner_const_name>>, + } + } else if self.attributes.pk_unsized { quote! { pub primary_index: (Vec>>, Vec>>), } @@ -61,11 +65,12 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let literal_name = name_generator.get_work_table_literal_name(); let version_const = name_generator.get_version_const_ident(); - let primary_page_count = if self.attributes.pk_arctic || self.attributes.pk_congee { - quote! { 1 } - } else { - quote! { self.primary_index.0.len() as u32 + self.primary_index.1.len() as u32 } - }; + let primary_page_count = + if self.attributes.pk_arctic || self.attributes.pk_arctic_string || self.attributes.pk_congee { + quote! { 1 } + } else { + quote! { self.primary_index.0.len() as u32 + self.primary_index.1.len() as u32 } + }; let row_schema = self.attributes.row_schema.iter().map(|(name, type_name)| { quote! { (#name.to_string(), #type_name.to_string()) } }); @@ -141,7 +146,12 @@ impl Generator { let secondary_index_events = name_generator.get_space_secondary_index_events_ident(); let avt_index_ident = name_generator.get_available_indexes_ident(); - let primary_index_init = if self.attributes.pk_unsized { + let primary_index_init = if self.attributes.pk_arctic_string { + quote! { + let pk_map = self.primary_index; + let primary_index = PrimaryIndex::from_map(pk_map); + } + } else if self.attributes.pk_unsized { let pk_ident = &self.pk_ident; let map_type = if self.attributes.pk_wti_logical { quote! { PersistentWtiIndex } @@ -162,21 +172,12 @@ impl Generator { .collect(); pk_map.attach_node(UnsizedNode::from_inner(node, #const_name)); } - // Reconstruct reverse_pk_map by iterating over pk_map - let mut reverse_pk_map = IndexMap::, #pk_ident>::new(); - for (pk, link) in pk_map.iter() { - reverse_pk_map.insert(link, pk); - } - let primary_index = PrimaryIndex { pk_map, reverse_pk_map }; + let primary_index = PrimaryIndex::from_map(pk_map); } } else if self.attributes.pk_arctic || self.attributes.pk_congee { quote! { let pk_map = self.primary_index; - let reverse_pk_map = IndexMap::, #pk_type>::new(); - for (pk, link) in pk_map.iter_values() { - reverse_pk_map.insert(link, pk); - } - let primary_index = PrimaryIndex { pk_map, reverse_pk_map }; + let primary_index = PrimaryIndex::from_map(pk_map); } } else { let map_type = if self.attributes.pk_wti_logical { @@ -206,12 +207,7 @@ impl Generator { .collect(); pk_map.attach_node(node); } - // Reconstruct reverse_pk_map by iterating over pk_map - let mut reverse_pk_map = IndexMap::, #pk_type>::new(); - for (pk, link) in pk_map.iter_values() { - reverse_pk_map.insert(link, pk); - } - let primary_index = PrimaryIndex { pk_map, reverse_pk_map }; + let primary_index = PrimaryIndex::from_map(pk_map); } }; @@ -342,7 +338,7 @@ impl Generator { let index_extension = Literal::string(WT_INDEX_EXTENSION); let data_extension = Literal::string(WT_DATA_EXTENSION); - let parse_pk_page = if self.attributes.pk_unsized { + let parse_pk_page = if self.attributes.pk_unsized && !self.attributes.pk_arctic_string { quote! { let index = parse_page::, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; } @@ -352,7 +348,14 @@ impl Generator { } }; - let parse_primary = if self.attributes.pk_arctic { + let parse_primary = if self.attributes.pk_arctic_string { + quote! { + SpaceArcticStringIndex::<#pk_type, { #inner_const_name as u32 }>::load_index::<#inner_const_name>( + format!("{}/primary{}", path, #index_extension), + #version_const_name, + ).await? + } + } else if self.attributes.pk_arctic { quote! { SpaceArcticIndex::<#pk_type, { #inner_const_name as u32 }>::load_index::<#inner_const_name>( format!("{}/primary{}", path, #index_extension), diff --git a/codegen/src/persist_table/generator/space_file/worktable_impls.rs b/codegen/src/persist_table/generator/space_file/worktable_impls.rs index bd3dac79..38ef6d2e 100644 --- a/codegen/src/persist_table/generator/space_file/worktable_impls.rs +++ b/codegen/src/persist_table/generator/space_file/worktable_impls.rs @@ -12,6 +12,7 @@ impl Generator { let wait_for_ops_fn = self.gen_worktable_wait_for_ops_fn(); let persistence_monitor_fn = self.gen_worktable_persistence_monitor_fn(); let close_fn = self.gen_worktable_close_fn(); + let unload_fn = self.gen_worktable_unload_fn(); let persisted_data_file_size_fn = self.gen_persisted_data_file_size_fn(); quote! { @@ -21,11 +22,47 @@ impl Generator { #wait_for_ops_fn #persistence_monitor_fn #close_fn + #unload_fn #persisted_data_file_size_fn } } } + fn gen_worktable_unload_fn(&self) -> TokenStream { + quote! { + /// Retires an Arc-owned table generation after the caller's + /// quiesce barrier has stopped new leases and drained old ones. + pub async fn unload_gracefully( + self: std::sync::Arc, + timeout: std::time::Duration, + quiesce: F, + ) -> eyre::Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + let released_bytes = self.heap_size(); + tokio::time::timeout(timeout, quiesce()) + .await + .map_err(|_| eyre::eyre!("timed out waiting for generation leases to quiesce"))?; + + let outstanding = std::sync::Arc::strong_count(&self).saturating_sub(1); + if outstanding != 0 { + return Err(eyre::eyre!( + "cannot unload generation: {outstanding} Arc lease(s) remain after quiesce" + )); + } + + let owned = std::sync::Arc::try_unwrap(self).map_err(|arc| { + let outstanding = std::sync::Arc::strong_count(&arc).saturating_sub(1); + eyre::eyre!("cannot unload generation: {outstanding} Arc lease(s) remain") + })?; + owned.close().await?; + Ok(UnloadReport { released_bytes }) + } + } + } + fn gen_persisted_data_file_size_fn(&self) -> TokenStream { if self.attributes.read_only { quote! {} @@ -138,7 +175,7 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let pk_type = name_generator.get_primary_key_type_ident(); let const_name = name_generator.get_page_inner_size_const_ident(); - if self.attributes.pk_arctic || self.attributes.pk_congee { + if self.attributes.pk_arctic || self.attributes.pk_arctic_string || self.attributes.pk_congee { // ART durability is maintained incrementally by its native // checkpoint/WAL file rather than materialized as DataBucket pages. quote! {} diff --git a/codegen/src/persist_table/parser.rs b/codegen/src/persist_table/parser.rs index 5803beb8..e19f2a43 100644 --- a/codegen/src/persist_table/parser.rs +++ b/codegen/src/persist_table/parser.rs @@ -31,6 +31,7 @@ impl Parser { read_only: false, pk_upstream: false, pk_arctic: false, + pk_arctic_string: false, pk_congee: false, pk_wti_logical: false, row_schema: vec![], @@ -57,6 +58,10 @@ impl Parser { res.pk_arctic = true; return Ok(()); } + if meta.path.is_ident("pk_arctic_string") { + res.pk_arctic_string = true; + return Ok(()); + } if meta.path.is_ident("pk_congee") { res.pk_congee = true; return Ok(()); diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index 7064af61..f17c45d2 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -400,14 +400,14 @@ mod tests { #[test] fn non_unique_arctic_rejects_unsupported_key_types() { let error = expand(quote! { - name: StringNonUniqueArctic, + name: BoolNonUniqueArctic, persist: false, columns: { id: u64 primary_key, - name: String, + enabled: bool, }, indexes: { - name_idx: name using arctic, + enabled_idx: enabled using arctic, }, }) .unwrap_err(); @@ -415,7 +415,7 @@ mod tests { assert!( error .to_string() - .contains("supported types: u16, u32, u64, u128, i16, i32, i64, i128") + .contains("supported types: String, u16, u32, u64, u128, i16, i32, i64, i128") ); } @@ -480,7 +480,7 @@ mod tests { assert!( error .to_string() - .contains("supported types: u16, u32, u64, u128, i16, i32, i64, i128") + .contains("supported types: String, u16, u32, u64, u128, i16, i32, i64, i128") ); } diff --git a/docs/beta18-validation.md b/docs/beta18-validation.md new file mode 100644 index 00000000..33d24309 --- /dev/null +++ b/docs/beta18-validation.md @@ -0,0 +1,298 @@ +# Beta.18 validation + +Status: release validation in progress. + +Validated 2026-09-05 from local working trees only. No WorkTable-family result +in this document resolves a released crate. + +## Local provenance + +| package | local path | base revision | +|---|---|---:| +| WorkTable | `/Users/revenge/code/WorkTable` | `a046217` plus working-tree changes | +| Arctic | `/Users/revenge/code/arctic-wt` | `46f1895` plus working-tree changes | +| ps-reclaim | `/Users/revenge/code/ps-reclaim` | `5ee0523` | +| WorkTablesIndex | `/Users/revenge/code/WorkTablesIndex` | `c10c82a` | +| DataBucket | `/Users/revenge/code/DataBucket` | `fa0f8a9` | +| Congee | `/Users/revenge/code/congee-wt` | `35855d3` | +| benchmark suite | `/Users/revenge/code/wt-benchmarks` | `df76063` plus working-tree changes | + +WorkTable, codegen, and DSL now identify as `1.0.0-beta.18`. + +`cargo tree --workspace --all-features` confirms that WorkTable, codegen, DSL, +Arctic, ps-reclaim, WorkTablesIndex, DataBucket and Congee all resolve to the +paths above. The local Vec controls resolve Arctic to the same local checkout. + +## Corrected benchmark methodology + +The old random-lookup runner printed one construction observation for each +arm. Those figures mixed a published beta17 result with later local lookup +results and were too noisy to support a construction regression claim. The +construction lines have been removed from the lookup runner. Construction now +has a dedicated runner with rotated order, 31 samples, eight populations per +sample, plus independent-process first-population measurements. + +The random lookup runner now performs eight million queries per sample and +rotates the six arm positions. It prints min, p25, median, p75 and max rather +than presenting one absolute median without its host spread. Full detail is in +the local benchmark document `docs/benchmarks/moe-resident-index-ab.md` in the +wt-benchmarks repository. + +## Random successful point lookup + +The latest local release run used eight million queries per sample, nine +rotated samples, and identical checksums: + +| arm | median ns/query | +|---|---:| +| Vec linear scan | 205.44 | +| Vec + BTreeMap | 33.53 | +| Vec + Arctic | 9.25 | +| WorkTable + Arctic | 15.65 | +| WorkTable + WTI | 58.05 | +| WorkTable + Congee | 24.77 | + +Arctic now defaults to ps-reclaim, so both Arctic arms use the release +reclamation backend. WorkTable+Arctic is 1.69x the stripped index-plus-Vec +lower bound while still 2.14x faster than Vec+BTreeMap. + +ps-reclaim originally marked a second live domain pin as cold and scanned the +participant's atomic slots. WorkTable nests its page and Arctic domains on +every select, so that assumption was false. An exact per-thread four-bit +occupancy mask now handles nested and out-of-order guard drops without the +scan. The post-fix 15.65 ns restores the earlier 15.57 ns baseline. The +single-thread difference against Seize was a microbenchmark only; scalability +and application-level results decide the release. Miss and range distributions +remain pending. + +## Construction + +Audited local candidate results for 1,528 rows: + +| arm | reused-process median ms | independent-process median ms | +|---|---:|---:| +| Vec + BTreeMap | 0.040 | 0.116 | +| WorkTable, `block_on` per row | 0.101 | 0.124 | +| WorkTable, one executor around loop | 0.101 | 0.121 | +| WorkTable `insert_many` | 0.095 | 0.134 | + +The cold first-population WorkTable/BTree gap is 1.53–1.55x. The async wrapper +is not the cause: per-row `block_on` and one executor differ by less than 1%. + +Equivalent local historical reused-process row-loop medians are 0.698 ms for +beta13, 0.662 ms for beta15 and 0.160 ms for the candidate. This does not +reproduce a release-to-release construction regression; the candidate is about +3.9x faster than beta15. It remains slower than the stripped BTree control. + +## Memory + +The isolated-process paired measurement reproduced exactly: + +| arm | retained bytes | bytes after drop | +|---|---:|---:| +| Vec + Arctic | 105,760 | 0 | +| WorkTable + Arctic | 105,976 | 0 | + +The WorkTable delta is 216 bytes, 0.14 bytes per live row, or 1.002x the +stripped Vec+Arctic control. Both arms produced the same checksum. The prior +candidate retained 114,344 bytes; exact-cell state plus the per-page live count +removed 8,368 bytes from that result. + +## Hot-page concurrency + +The short micro-run was rejected because absolute throughput moved materially +with host load. The benchmark was enlarged eightfold to two million reads per +reader and 160,000 upserts per mixed sample. The candidate was run before and +after an unmodified `a046217` control using the same benchmark executable. + +| tree | writer ns/upsert | concurrent reader Mops/s | +|---|---:|---:| +| exact-cell candidate, run 1 | 321.88 | 80.482 | +| exact-cell candidate, run 2 | 469.29 | 49.707 | +| prior candidate, best | 1,028.96 | 86.313 | +| unmodified base | 1,535.69 | 22.218 | + +Mixed reader scheduling remains noisy, and the raw range is retained rather +than hidden. Both exact-cell writer samples are more than twice as fast as the +prior candidate. The bounded standardized concurrency grid below supersedes +this diagnostic for release disposition. + +## Standard scalability grid + +The checked-in `concurrent_mix` suite now has bounded 1/2/4/8/16/32-thread +axes for all three primary-index backends. Each thread runs 4,000 pre-generated +operations over a disjoint key range in a 20,000-row table. This avoids +measuring RNG or same-key contention. The host has 12 performance and four +efficiency cores. + +Pure-read median throughput in Mops/s: + +| backend | 1 | 2 | 4 | 8 | 16 | 32 | best/1T | +|---|---:|---:|---:|---:|---:|---:|---:| +| WTI | 8.748 | 14.042 | 24.181 | 22.968 | 31.507 | 23.382 | 3.60x | +| Arctic | 8.918 | 14.002 | 21.829 | 29.631 | 27.446 | 24.830 | 3.32x | +| Congee | 9.539 | 18.847 | 24.446 | 22.006 | 19.542 | 24.456 | 2.56x | + +The 10%-write median throughput in Mops/s: + +| backend | 1 | 2 | 4 | 8 | 16 | 32 | best/1T | +|---|---:|---:|---:|---:|---:|---:|---:| +| WTI | 4.826 | 5.426 | 9.701 | 4.632 | 4.256 | 4.055 | 2.01x | +| Arctic | 4.852 | 5.066 | 8.601 | 4.651 | 4.709 | 5.164 | 1.77x | +| Congee | 5.436 | 6.241 | 8.605 | 4.449 | 4.477 | 4.703 | 1.58x | + +The broad-table read path scales materially better than the one-page +false-sharing probe, but it is not close to linear. Mixed throughput peaks at +four threads and then drops for every backend. The same ceiling appears in +beta13 and beta15, so it is not introduced by beta18 or the ps-reclaim switch. +In adjacent equal-profile beta18/beta15 runs, beta18 is faster in every +targeted high-thread cell except Arctic at eight threads: 4.651 versus 5.022 +Mops/s, a reproducible 7.4% loss. Arctic beta18 is 4.8% and 6.5% faster at 16 +and 32 threads. The single eight-thread loss remains disclosed but is not an +overall scalability release blocker. + +## Local beta13/beta15/beta18 MoE-PGO grid + +The donor-width (`12,288`) PGO grid was built through an isolated local-only +harness. Beta13 and beta15 use local WorkTable, matching local DataBucket and +matching local WTI worktrees. Beta18 uses the candidate and every current +backend dependency from the local paths in the provenance table. The comparable +control medians are 1.5329, 1.5416 and 1.5501 ms respectively, a 1.1% span. + +Raw phase medians: + +| phase | backend | beta13 | beta15 | beta18 | +|---|---|---:|---:|---:| +| accumulate, 200k updates | WTI | 127.22 ms | 121.01 ms | 87.146 ms | +| | Congee | 96.937 ms | 76.214 ms | 63.854 ms | +| | Arctic | 85.409 ms | 75.556 ms | 73.496 ms | +| publish, 8 × 12,288 rows | WTI | 56.450 ms | 59.550 ms | 34.917 ms | +| | Congee | 37.259 ms | 39.042 ms | 11.399 ms | +| | Arctic | 34.202 ms | 34.881 ms | 7.7407 ms | +| retire, 8 maps under readers | WTI | 3.8748 µs* | 1.7007 ms | 50.655 µs | +| | Congee | 3.7604 µs* | 2.4858 ms | 1.7240 ms | +| | Arctic | 3.8184 µs* | 2.1884 ms | 135.64 µs | + +The beta13 retire cells marked `*` do not reclaim: that implementation only +appends the old `Arc` to a permanently retained vector unless an exclusive +`&mut self` GC API is called. They are leak timings and are not used as a +performance baseline. Against beta15's real reclamation, beta18 retire is +33.6x faster on WTI, 1.44x on Congee and 16.1x on Arctic. Publish plus retire +is 42.9%, 68.4% and 78.8% faster respectively. A first beta15 pass was rejected +openly because its no-WorkTable control was 1.9398 ms; the immediately repeated +1.5416 ms pass is the table above. + +## Local AgentCode generation grid + +The AgentCode benchmark now explicitly selects WTI, Arctic and Congee for both +the primary key and dedup secondary index. It writes one 14,400-symbol +generation and reports acceptance separately from persistence drain. Each cell +below is the median of three warm processes; durable columns sum acceptance and +drain inside each run before taking the median. + +| version | backend | memory row insert | memory batch | durable row insert | durable batch | readback | +|---|---|---:|---:|---:|---:|---:| +| beta13 | WTI | 1,212.14 | 1,041.73 | 6,598.21 | 7,817.54 | 130.91 | +| beta13 | Arctic | 910.42 | 728.90 | 6,159.13 | 7,243.97 | 103.27 | +| beta13 | Congee | 904.04 | 735.36 | 6,058.45 | 7,202.81 | 110.91 | +| beta15 | WTI | 1,193.00 | 987.45 | 6,634.12 | 6,475.49 | 129.73 | +| beta15 | Arctic | 899.16 | 721.78 | 6,129.71 | 5,822.36 | 107.42 | +| beta15 | Congee | 921.64 | 726.67 | 6,008.33 | 5,883.92 | 106.45 | +| beta18 | WTI | 835.06 | 728.80 | 4,416.68 | 3,971.17 | 104.06 | +| beta18 | Arctic | 506.03 | 379.56 | 3,753.34 | 3,377.61 | 80.30 | +| beta18 | Congee | 538.04 | 395.23 | 3,899.32 | 3,497.77 | 76.72 | + +All values are ns/row. Against beta15, beta18 durable single-row generation +writes improve 33.4%, 38.8% and 35.1% for WTI, Arctic and Congee. Durable +`insert_many` improves 38.7%, 42.0% and 40.6%. Arctic is the fastest beta18 +write backend; Congee is fastest for complete generation readback. + +## Exact-cell synchronization and compatibility + +The fixed hashed row stripes and four-byte per-row vacuum directory are gone. +Each archived cell carries one runtime synchronization byte, and each page +carries one live-cell counter. A writer on one cell does not block a different +cell on the same physical page. Full-row replacement deliberately skips the +active synchronization byte; the first implementation copied it and the +concurrent update/reclaim test exposed the resulting stuck cell. + +The corrected implementation passes 35 focused page/reclamation tests, the +concurrent update/reclaim regression, and the checked-in pre-schema persisted +fixture. The archived inner row remains at the beta.17 offset. + +## Generation unload + +Generated persisted and read-only tables now implement `MemStat`. An +Arc-owned `unload_gracefully` runs a caller-supplied quiesce barrier under a +timeout, rejects any remaining Arc leases, drains persistence, drops the owned +generation, and reports the attributed bytes released. Its three focused tests +pass, including a live reader draining during the swap. + +## Reclamation backend audit + +WorkTable's row/page domain uses local `ps-reclaim` regardless of whether the +index is WTI, Arctic, or Congee. WorkTable's Arctic adapter now disables +Arctic's default Seize feature and instantiates only Arctic `PsReclaim`, even +under `--no-default-features`. WTI still owns a Crossbeam skip-list internally, +and Congee still owns Crossbeam epoch internally; they are not simultaneously +selected indexes, and those internal implementations are distinct from +WorkTable's row/page reclamation layer. + +## Reactive vacuum + +The balanced local grid uses all six mode-order permutations, three backends, +100,000 starting rows, one second of uninterrupted upsert/reinsert/delete +pressure, and an independently packed 50,000-row control. + +| backend | off operations median | reactive operations median | delta | off range | reactive range | unpaced delta | +|---|---:|---:|---:|---:|---:|---:| +| WTI | 250,873 | 246,886 | -1.6% | 247,745–255,268 | 234,392–258,380 | -16.2% | +| Arctic | 253,660 | 250,832 | -1.1% | 246,944–264,580 | 246,255–254,977 | -29.9% | +| Congee | 257,808 | 244,726 | -5.1% | 241,337–263,049 | 238,078–258,549 | -41.1% | + +Every reactive range overlaps its vacuum-off range. The reactive median deltas +are smaller than the corresponding 3.0%, 7.0% and 8.4% vacuum-off host spreads. +The unpaced positive control loses 16–41% of foreground throughput, proving +that the workload can detect vacuum interference. + +Reactive vacuum reclaimed from 196 to the independently packed 98 in-use pages +in all 18 of 18 backend/repetition cells. Maximum excess pages were zero. + +The manager's sweep counter increments only after a sweep completes. Therefore +zero completed sweeps during load is not treated as proof of zero work. Three +deterministic local tests supply the structural evidence and pass: + +- a live mutation holds vacuum before its first batch; +- a mutation completed between checks resets the quiet-sample buffer; +- vacuum reaches the exact independently packed page count. + +## Remaining release gates + +- Complete the beta13/beta15/beta18 local YCSB grid across WTI, Arctic and + Congee. The local PGO, concurrency and AgentCode grids are complete. +- Add random miss and range-search distributions after the release-blocking + standard matrix; do not hold beta18 for nanosecond-only tuning. +- Rebuild AgencyZero against the local candidate and strict-open the complete + qa-profile bundle to prove persisted-format compatibility. +- Run support.cafe/S3/Fly validation after local correctness and performance + gates. Any deferral requires an explicit release choice. +- Put WorkTable, Arctic and benchmark changes on reviewable PRs and record the + dependency merge/publish order. + +## Workspace correctness gate + +The current candidate passes `cargo check --workspace --all-targets`, 35/35 +focused page/reclamation tests, the legacy persisted fixture, the concurrent +update/reclaim regression, and all three generation-swap tests. The complete +post-fix workspace gate passes: 271 core unit tests, 642 integration tests (four +explicitly ignored), 56 codegen tests, and the complete DSL suite. Arctic's +default ps-reclaim configuration passes 40 unit, 10 regression, and two +Shuttle tests. ps-reclaim passes its complete suite, including nested-pin, +out-of-order-drop, overflow-slot, and continuous-reader coverage. + +The first default-parallel run had one 30-second timeout in +`test_duplicate_key_mutations_without_reload`. It did not report an event gap +or engine failure. The same test then passed alone in 14.63, 11.05 and 6.40 +seconds, and passed in two complete four-thread integration runs. This is +recorded as suite-level persistence-task starvation under unbounded test +parallelism; it is not silently discarded or classified as a product deadlock. diff --git a/dsl/src/validate.rs b/dsl/src/validate.rs index a6e32b69..00885a37 100644 --- a/dsl/src/validate.rs +++ b/dsl/src/validate.rs @@ -235,7 +235,7 @@ pub const AUTOINCREMENT_TYPES: &[&str] = &["u8", "u16", "u32", "u64", "i8", "i16 pub fn supported_key_types(backend: IndexBackend) -> Option<&'static [&'static str]> { match backend { IndexBackend::Congee => Some(&["u8", "u16", "u32", "u64", "usize"]), - IndexBackend::Arctic => Some(&["u16", "u32", "u64", "u128", "i16", "i32", "i64", "i128"]), + IndexBackend::Arctic => Some(&["String", "u16", "u32", "u64", "u128", "i16", "i32", "i64", "i128"]), IndexBackend::WorktablesIndex | IndexBackend::Indexset => None, } } diff --git a/src/in_memory/data.rs b/src/in_memory/data.rs index 349e6ec6..93a1ad1d 100644 --- a/src/in_memory/data.rs +++ b/src/in_memory/data.rs @@ -2,7 +2,7 @@ use std::cell::UnsafeCell; use std::fmt::Debug; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicU8, AtomicU32, Ordering}; use data_bucket::page::INNER_PAGE_SIZE; use data_bucket::page::PageId; @@ -12,7 +12,7 @@ use derive_more::{Display, Error}; use performance_measurement_codegen::performance_measurement; use rkyv::{ Archive, Deserialize, Portable, Serialize, - api::high::HighDeserializer, + api::{high::HighDeserializer, root_position}, rancor::Strategy, seal::Seal, ser::{Serializer, allocator::ArenaHandle, sharing::Share}, @@ -20,8 +20,40 @@ use rkyv::{ with::{AtomicLoad, Relaxed, Skip, Unsafe}, }; +use crate::in_memory::ArchivedRowWrapper; use crate::prelude::Link; +const CELL_WRITER: u8 = 1 << 7; +const CELL_READERS: u8 = !CELL_WRITER; + +/// Shared access to one exact archived cell. +pub(crate) struct CellReadGuard<'a> { + state: *mut AtomicU8, + marker: PhantomData<&'a AtomicU8>, +} + +impl Drop for CellReadGuard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: the guard cannot outlive the page from which `state` came. + unsafe { &*self.state }.fetch_sub(1, Ordering::Release); + } +} + +/// Exclusive access to one exact archived cell. +pub(crate) struct CellWriteGuard<'a> { + state: *mut AtomicU8, + marker: PhantomData<&'a AtomicU8>, +} + +impl Drop for CellWriteGuard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: the guard cannot outlive the page from which `state` came. + unsafe { &*self.state }.store(0, Ordering::Release); + } +} + /// Length of the [`Data`] page header. pub const DATA_HEADER_LENGTH: usize = 4; @@ -58,14 +90,24 @@ pub struct Data { /// Per-page access barrier for the mutable byte image. /// - /// The exclusive side serializes mutations of this page's bytes (row - /// writes, in-place updates, reset on reuse); the shared side protects - /// low-level readers of those bytes (publication hydration, `with_ref`, - /// CDC byte capture). Runtime-only: skipped by rkyv, reconstructed - /// unlocked on load. + /// The exclusive side serializes page allocation/reset and append-position + /// changes. Existing-cell reads and writes use the exact cell byte below, + /// so unrelated rows on this page do not contend here. Runtime-only: + /// skipped by rkyv and reconstructed unlocked on load. #[rkyv(with = Skip)] pub(crate) access: parking_lot::RwLock<()>, + /// Number of live cells currently published on this page. + /// + /// Vacuum gets move candidates from a transient snapshot of the primary + /// index. It only needs permanent per-page state to prove a source became + /// empty before reclaiming it. Keeping that proof as one counter removes + /// the old four-byte entry for every row (and its locked `Vec`) without + /// weakening the final reclamation check. Runtime-only and rebuilt from + /// the primary index when a persisted table is loaded. + #[rkyv(with = Skip)] + live_cells: AtomicU32, + /// Inner array of bytes where deserialized `Row`s will be stored. #[rkyv(with = Unsafe)] inner_data: UnsafeCell>, @@ -77,12 +119,109 @@ pub struct Data { unsafe impl Sync for Data {} impl Data { + fn archived_cell_state_offset(bytes: &mut [u8]) -> Result + where + Row: Archive, + ::Archived: ArchivedRowWrapper, + { + let root_offset = root_position::<::Archived>(bytes.len()); + let base = bytes.as_mut_ptr(); + let root = unsafe { base.add(root_offset).cast::<::Archived>() }; + let state = unsafe { ::Archived::cell_state_ptr(root) }.cast::(); + let offset = unsafe { state.offset_from(base) }; + let offset = usize::try_from(offset).map_err(|_| ExecutionError::InvalidLink)?; + if offset >= bytes.len() { + return Err(ExecutionError::InvalidLink); + } + Ok(offset) + } + + fn cell_state_ptr(&self, link: Link) -> Result<*mut AtomicU8, ExecutionError> + where + Row: Archive, + ::Archived: ArchivedRowWrapper, + { + let start = link.offset as usize; + let end = start + .checked_add(link.length as usize) + .ok_or(ExecutionError::InvalidLink)?; + let initialized = self.free_offset.load(Ordering::Acquire) as usize; + if link.length == 0 || end > initialized || end > DATA_LENGTH { + return Err(ExecutionError::InvalidLink); + } + + let inner_data = unsafe { &mut *self.inner_data.get() }; + let root = unsafe { + inner_data + .as_mut_ptr() + .add(start + root_position::<::Archived>(link.length as usize)) + .cast::<::Archived>() + }; + // SAFETY: `root` points at this cell's archived wrapper. The wrapper + // contract places its atomic state at a stable archived offset. + Ok(unsafe { ::Archived::cell_state_ptr(root) }) + } + + pub(crate) fn read_cell(&self, link: Link) -> Result, ExecutionError> + where + Row: Archive, + ::Archived: ArchivedRowWrapper, + { + let state = self.cell_state_ptr(link)?; + let state_ref = unsafe { &*state }; + loop { + let current = state_ref.load(Ordering::Acquire); + if current & CELL_WRITER != 0 || current & CELL_READERS == CELL_READERS { + std::hint::spin_loop(); + continue; + } + if state_ref + .compare_exchange_weak(current, current + 1, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return Ok(CellReadGuard { + state, + marker: PhantomData, + }); + } + } + } + + pub(crate) fn write_cell(&self, link: Link) -> Result, ExecutionError> + where + Row: Archive, + ::Archived: ArchivedRowWrapper, + { + let state = self.cell_state_ptr(link)?; + let state_ref = unsafe { &*state }; + loop { + let current = state_ref.load(Ordering::Acquire); + if current & CELL_WRITER != 0 { + std::hint::spin_loop(); + continue; + } + if state_ref + .compare_exchange_weak(current, current | CELL_WRITER, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + while state_ref.load(Ordering::Acquire) != CELL_WRITER { + std::hint::spin_loop(); + } + return Ok(CellWriteGuard { + state, + marker: PhantomData, + }); + } + } + } + /// Creates new [`Data`] page. pub fn new(id: PageId) -> Self { Self { id, free_offset: AtomicU32::default(), access: parking_lot::RwLock::new(()), + live_cells: AtomicU32::new(0), inner_data: UnsafeCell::new(AlignedBytes::([0; DATA_LENGTH])), _phantom: PhantomData, } @@ -93,6 +232,7 @@ impl Data { id: page.header.page_id, free_offset: AtomicU32::from(page.header.data_length), access: parking_lot::RwLock::new(()), + live_cells: AtomicU32::new(0), inner_data: UnsafeCell::new(AlignedBytes::(page.inner.data)), _phantom: PhantomData, } @@ -138,6 +278,8 @@ impl Data { length, }; + self.register_cell(link); + Ok(link) } @@ -163,6 +305,34 @@ impl Data { Ok(link) } + /// Replaces an archived cell while preserving its active synchronization + /// byte. The caller must hold this cell's write guard for the entire call. + /// + /// The lock lives inside the archived wrapper, so copying the serialized + /// replacement wholesale would briefly publish a zero lock byte while the + /// surrounding row is only partially copied. A reader could then enter the + /// cell and observe a torn row. Copy the bytes on either side instead. + #[allow(clippy::missing_safety_doc)] + pub unsafe fn save_row_by_link_preserving_cell_state(&self, row: &Row, link: Link) -> Result + where + Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + ::Archived: ArchivedRowWrapper, + { + let mut bytes = rkyv::to_bytes(row).map_err(|_| ExecutionError::SerializeError)?; + let length = bytes.len() as u32; + if length != link.length { + return Err(ExecutionError::InvalidLink); + } + + let state = Self::archived_cell_state_offset(bytes.as_mut_slice())?; + let inner_data = unsafe { &mut *self.inner_data.get() }; + let destination = &mut inner_data[link.offset as usize..][..link.length as usize]; + destination[..state].copy_from_slice(&bytes[..state]); + destination[state + 1..].copy_from_slice(&bytes[state + 1..]); + + Ok(link) + } + #[allow(clippy::missing_safety_doc)] pub unsafe fn try_save_row_by_link(&self, row: &Row, mut link: Link) -> Result<(Link, Option), ExecutionError> where @@ -189,6 +359,8 @@ impl Data { let inner_data = unsafe { &mut *self.inner_data.get() }; inner_data[link.offset as usize..][..link.length as usize].copy_from_slice(bytes.as_slice()); + self.register_cell(link); + Ok((link, link_left)) } @@ -259,6 +431,20 @@ impl Data { Ok(inner_data[link.offset as usize..(link.offset + link.length) as usize].to_vec()) } + /// Copies a wrapped row while clearing its runtime-only synchronization + /// byte in the copy. CDC and vacuum must never persist or publish an active + /// reader count into another cell. + pub(crate) fn get_raw_row_without_cell_state(&self, link: Link) -> Result, ExecutionError> + where + Row: Archive, + ::Archived: ArchivedRowWrapper, + { + let mut bytes = self.get_raw_row(link)?; + let state = Self::archived_cell_state_offset(bytes.as_mut_slice())?; + bytes[state] = 0; + Ok(bytes) + } + /// Moves data within the page from one location to another. /// Used for defragmentation - shifts data left to fill gaps. /// @@ -315,11 +501,13 @@ impl Data { let inner_data = unsafe { &mut *self.inner_data.get() }; inner_data[offset as usize..][..length as usize].copy_from_slice(data); - Ok(Link { + let link = Link { page_id: self.id, offset, length, - }) + }; + self.register_cell(link); + Ok(link) } pub fn free_space(&self) -> usize { @@ -328,6 +516,37 @@ impl Data { pub fn reset(&self) { self.free_offset.store(0, Ordering::Release); + self.live_cells.store(0, Ordering::Release); + } + + pub(crate) fn reset_cell_state(&self, link: Link) -> Result<(), ExecutionError> + where + Row: Archive, + ::Archived: ArchivedRowWrapper, + { + // Persisted pages can contain whatever synchronization byte happened + // to be present in the last in-memory image. A cold load has no live + // readers, so reset runtime state before publishing the table. + unsafe { &*self.cell_state_ptr(link)? }.store(0, Ordering::Release); + Ok(()) + } + + pub(crate) fn register_cell(&self, link: Link) { + debug_assert_eq!(link.page_id, self.id); + self.live_cells + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1)) + .expect("live cell count overflow"); + } + + pub(crate) fn remove_cell(&self, link: Link) { + debug_assert_eq!(link.page_id, self.id); + self.live_cells + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_sub(1)) + .expect("removing a cell from an empty page"); + } + + pub(crate) fn has_live_cells(&self) -> bool { + self.live_cells.load(Ordering::Acquire) != 0 } } diff --git a/src/in_memory/mod.rs b/src/in_memory/mod.rs index fefa6add..81633576 100644 --- a/src/in_memory/mod.rs +++ b/src/in_memory/mod.rs @@ -1,10 +1,9 @@ mod data; mod empty_link_registry; mod pages; -mod publication; mod row; pub use data::{DATA_INNER_LENGTH, Data, ExecutionError as DataExecutionError}; pub use empty_link_registry::EmptyLinkRegistry; pub use pages::{DataPages, ExecutionError as PagesExecutionError, ReadGuard as DataPagesReadGuard}; -pub use row::{ArchivedRowWrapper, PublicationSafe, Query, RowWrapper, StorableRow}; +pub use row::{ArchivedRowWrapper, CellState, PublicationSafe, Query, RowWrapper, StorableRow}; diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index f714eefa..7c1e3b3c 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -1,3 +1,4 @@ +use arc_swap::ArcSwap; use data_bucket::page::PageId; use derive_more::{Display, Error, From}; use parking_lot::Mutex; @@ -11,20 +12,17 @@ use rkyv::{ ser::{Serializer, allocator::ArenaHandle, sharing::Share}, util::AlignedVec, }; -use std::collections::{HashMap, HashSet, VecDeque}; -use std::hash::{BuildHasherDefault, Hasher}; +use std::collections::{HashSet, VecDeque}; use std::marker::PhantomData; -use std::sync::atomic::AtomicUsize; +use std::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize}; use std::{ fmt::Debug, sync::Arc, - sync::atomic::{AtomicU32, AtomicU64, Ordering}, + sync::atomic::{AtomicU64, Ordering}, }; use crate::in_memory::empty_link_registry::EmptyLinkRegistry; -use crate::in_memory::publication::{DELETED, GHOSTED, PublishedRow, VACUUMED}; use crate::prelude::ArchivedRowWrapper; -use crate::util::OffsetEqLink; use crate::util::epoch::EpochDomain; use crate::{ in_memory::{ @@ -38,7 +36,11 @@ fn page_id_mapper(page_id: usize) -> usize { page_id - 1usize } -const PUBLICATION_SHARD_COUNT: usize = 64; +const PAGE_DIRECTORY_CHUNK_SIZE: usize = 64; +const PAGE_DIRECTORY_ROOTS: usize = 64; +const GHOSTED: u8 = 1 << 0; +const DELETED: u8 = 1 << 1; +const VACUUMED: u8 = 1 << 2; const RETIREMENT_BACKLOG_WARN_AT: usize = 1_024; /// Most retired items one reclaim call may recycle inline. Bounds the latency @@ -60,58 +62,86 @@ const RECLAIM_BATCH_LIMIT: usize = 256; /// pays, and a sweep drains up to [`RECLAIM_BATCH_LIMIT`] at once, so matching /// them means a triggered sweep clears the backlog it was triggered by. /// -/// Deliberately not applied to `mark_page_empty` or `retire_published_link`. -/// Neither is a hot path, so deferring them would change when pages become -/// allocatable for no measurable gain. +/// Deliberately not applied to `mark_page_empty`. It is not a hot path, so +/// deferring it would change when pages become allocatable for no measurable +/// gain. const RECLAIM_BACKLOG_TRIGGER: usize = RECLAIM_BATCH_LIMIT; -fn mix_publication_offset(mut value: u64) -> u64 { - value ^= value >> 30; - value = value.wrapping_mul(0xbf58_476d_1ce4_e5b9); - value ^= value >> 27; - value = value.wrapping_mul(0x94d0_49bb_1331_11eb); - value ^ (value >> 31) +#[derive(Debug)] +struct PageDirectoryChunk { + pages: [AtomicPtr; PAGE_DIRECTORY_CHUNK_SIZE], } -/// `OffsetEqLink` already reduces publication keys to a trusted internal u64 -/// storage offset. Avalanche that offset so both hash-table bucket bits and -/// SIMD control bits remain distributed for aligned, monotonically allocated -/// row positions. -struct PublicationHasher(u64); - -impl Default for PublicationHasher { - fn default() -> Self { - Self(0xcbf2_9ce4_8422_2325) +impl PageDirectoryChunk { + fn new() -> Self { + Self { + pages: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + } } } -impl Hasher for PublicationHasher { - fn finish(&self) -> u64 { - self.0 - } +/// Non-owning, stable page pointers for the first 4,096 pages (64 MiB at the +/// default page size). `DataPages::pages` owns every allocation; this directory +/// exists solely to avoid shared ArcSwap snapshot accounting on point access. +#[derive(Debug)] +struct PageDirectory { + roots: [AtomicPtr>; PAGE_DIRECTORY_ROOTS], + chunks: Mutex>>>, +} - fn write(&mut self, bytes: &[u8]) { - let mut hash = self.0; - for byte in bytes { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x0000_0100_0000_01b3); +impl PageDirectory { + fn new(pages: &[Arc]) -> Self { + let directory = Self { + roots: std::array::from_fn(|_| AtomicPtr::new(std::ptr::null_mut())), + chunks: Mutex::new(Vec::new()), + }; + for (index, page) in pages.iter().enumerate() { + directory.publish(index, page); } - self.0 = mix_publication_offset(hash); + directory } - fn write_u64(&mut self, value: u64) { - self.0 = mix_publication_offset(self.0 ^ value); - } -} - -type PublicationMap = - HashMap, Arc>, BuildHasherDefault>; + fn publish(&self, index: usize, page: &Arc) { + let root_index = index / PAGE_DIRECTORY_CHUNK_SIZE; + let Some(root) = self.roots.get(root_index) else { + return; + }; + let mut chunk = root.load(Ordering::Acquire); + if chunk.is_null() { + let mut chunks = self.chunks.lock(); + chunk = root.load(Ordering::Acquire); + if chunk.is_null() { + let mut owned = Box::new(PageDirectoryChunk::new()); + chunk = (&mut *owned) as *mut PageDirectoryChunk; + chunks.push(owned); + root.store(chunk, Ordering::Release); + } + } -type PublicationShards = - [RwLock>; PUBLICATION_SHARD_COUNT]; + // SAFETY: `chunk` points into one of the Boxes retained in `chunks`. + // Boxes are never removed, so the allocation remains stable. + unsafe { &*chunk }.pages[index % PAGE_DIRECTORY_CHUNK_SIZE] + .store(Arc::as_ptr(page).cast_mut(), Ordering::Release); + } -fn publication_shard(key: &OffsetEqLink) -> usize { - mix_publication_offset(key.absolute_index()) as usize & (PUBLICATION_SHARD_COUNT - 1) + fn get(&self, index: usize) -> Option<&T> { + let root = self.roots.get(index / PAGE_DIRECTORY_CHUNK_SIZE)?; + let chunk = root.load(Ordering::Acquire); + if chunk.is_null() { + return None; + } + // SAFETY: `publish` retains this chunk's Box for this directory's + // lifetime and stores page pointers only after their owning Arc is in + // the immutable-snapshot directory. + let page = unsafe { &*chunk }.pages[index % PAGE_DIRECTORY_CHUNK_SIZE].load(Ordering::Acquire); + if page.is_null() { + None + } else { + // SAFETY: page entries are appended but never removed or replaced, + // and every future `pages` snapshot retains the same Arc. + Some(unsafe { &*page }) + } + } } /// A read-side grace-period guard: an epoch pin in the table's own @@ -132,25 +162,21 @@ pub struct ReadGuard<'a> { } /// One unit of retired state waiting out its grace period. Reclamation is -/// *recycling*, not just freeing: links return to `empty_links`, pages return -/// to `empty_pages`, and publication slots leave their shard map. +/// *recycling*, not just freeing: links return to `empty_links` and pages +/// return to `empty_pages`. #[derive(Debug, Clone, Copy)] -enum Retired { - /// A freed row slot: remove its publication, then hand the slot back to - /// the empty-link allocator (unless a whole-page retirement supersedes - /// it). +enum Retired { + /// A freed row slot: hand the slot back to the empty-link allocator unless + /// a whole-page retirement supersedes it. Link(Link), /// A wholly emptied page: purge any of its stale empty links, then hand /// the page back to the empty-page allocator. Page(PageId), - /// A publication whose row bytes moved elsewhere (vacuum): remove the - /// shard entry only, the physical slot stays owned by its page. - Publication(OffsetEqLink), } -/// Page storage with immutable row publication. +/// Page storage with row-granular read/write exclusion. /// -/// # Versioned-publication synchronization +/// # Read synchronization /// /// Generated readers enter the grace period before resolving an index link by /// pinning the table's epoch domain. Writers must remove or replace every @@ -187,33 +213,28 @@ enum Retired { /// /// 1. generated row/lock-manager locks (outside this type, always first); /// 2. `empty_pages` (only the insert page-switch path holds it into 3/4); -/// 3. `pages` (the vector lock; its write side is only taken for growth, -/// with no page lock held); +/// 3. `pages_write` (only for appending a page, with no page lock held); /// 4. one or two per-page `Data::access` locks — two only in the vacuum row /// move, always in ascending page-id order; -/// 5. one `published_rows` shard, or the empty-link registry's `op_lock`. +/// 5. one exact cell lock, or the empty-link registry's `op_lock`. /// -/// Reclamation holds the retirement queue, then briefly acquires individual -/// publication shards and the empty-link/page registries; nothing acquires -/// the retirement queue while holding any of those (or any page lock), so -/// the order is acyclic. Callers must not invoke reclamation while retaining -/// the retirement-queue guard. +/// Reclamation holds the retirement queue, then briefly acquires the +/// empty-link/page registries; nothing acquires the retirement queue while +/// holding either registry (or any page/row lock), so the order is acyclic. +/// Callers must not invoke reclamation while retaining the retirement-queue +/// guard. #[derive(Debug)] pub struct DataPages where Row: StorableRow, { - /// Immutable application-visible row versions. Published readers never - /// borrow the mutable archived page image. - published_rows: PublicationShards, - /// Read-side grace periods protecting the interval from index lookup /// until an immutable row version has been acquired. Owned by this table: /// a reader of another table never delays reclamation here. epoch: EpochDomain, /// Retired items in retirement order, awaiting grace expiry. - retired: Mutex>>, + retired: Mutex>, /// How many queued retirements' grace periods have expired. Incremented /// by deferred epoch markers; consumed (front-of-queue) by reclaimers. @@ -233,8 +254,13 @@ where /// one relaxed load in the case that matters. queued_page_retirements: AtomicUsize, - /// Pages vector. Currently, not lock free. - pages: RwLock::WrappedRow, DATA_LENGTH>>>>, + /// Immutable page-directory snapshots. Reads load one snapshot without a + /// shared read-modify-write; rare growth copies and swaps the short vector. + pages: ArcSwap::WrappedRow, DATA_LENGTH>>>>, + /// Stable pointers for point access without ArcSwap's shared snapshot + /// accounting. The corresponding `Arc`s remain owned by `pages`. + page_directory: PageDirectory::WrappedRow, DATA_LENGTH>>, + pages_write: Mutex<()>, empty_links: EmptyLinkRegistry, @@ -263,6 +289,31 @@ where Row: StorableRow, ::WrappedRow: RowWrapper, { + fn page_ref( + &self, + page_id: PageId, + ) -> Result<&Data<::WrappedRow, DATA_LENGTH>, ExecutionError> { + let index = page_id_mapper(page_id.into()); + if let Some(page) = self.page_directory.get(index) { + return Ok(page); + } + + let page = { + let pages = self.pages.load(); + pages.get(index).map(Arc::as_ptr) + } + .ok_or(ExecutionError::PageNotFound(page_id))?; + + // SAFETY: as above, the current directory retains this allocation and + // all future directory snapshots clone its Arc. + Ok(unsafe { &*page }) + } + + fn publish_page(&self, page: &Arc::WrappedRow, DATA_LENGTH>>) { + let index = page_id_mapper(page.id.into()); + self.page_directory.publish(index, page); + } + fn publication_flags(row: &::WrappedRow) -> u8 { let mut flags = 0; if row.is_ghosted() { @@ -277,56 +328,16 @@ where flags } - fn publish_wrapped_row(&self, link: Link, wrapped: ::WrappedRow) { - let flags = Self::publication_flags(&wrapped); - let row = wrapped.get_inner(); - let key = OffsetEqLink(link); - - let mut published_rows = self.published_rows[publication_shard(&key)].write(); - if let Some(slot) = published_rows.get(&key).cloned() { - drop(published_rows); - slot.replace(row, flags); - } else { - published_rows.insert(key, Arc::new(PublishedRow::new(row, flags))); - } - } - - fn stage_published_row(&self, link: Link, row: Row) { - let wrapped = ::WrappedRow::from_inner(row); - self.publish_wrapped_row(link, wrapped); - } - - fn published_slot(&self, link: Link) -> Option>> { - let key = OffsetEqLink(link); - self.published_rows[publication_shard(&key)].read().get(&key).cloned() - } - - fn published_slot_or_hydrate(&self, link: Link) -> Result>, ExecutionError> + fn page_row(&self, link: Link) -> Result<(Row, u8), ExecutionError> where <::WrappedRow as Archive>::Archived: - Deserialize<::WrappedRow, HighDeserializer>, + Portable + Deserialize<::WrappedRow, HighDeserializer>, { - if let Some(slot) = self.published_slot(link) { - return Ok(slot); - } - - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let _page_guard = page.access.read(); - // Re-check under the page barrier: a writer publishing this row holds - // the exclusive side, so once we hold the shared side the publication - // map is current for this page's rows. - if let Some(slot) = self.published_slot(link) { - return Ok(slot); - } + let page = self.page_ref(link.page_id)?; + let _cell_guard = page.read_cell(link).map_err(ExecutionError::DataPageError)?; let wrapped = page.get_row(link).map_err(ExecutionError::DataPageError)?; let flags = Self::publication_flags(&wrapped); - let slot = Arc::new(PublishedRow::new(wrapped.get_inner(), flags)); - let key = OffsetEqLink(link); - let mut published_rows = self.published_rows[publication_shard(&key)].write(); - Ok(published_rows.entry(key).or_insert(slot).clone()) + Ok((wrapped.get_inner(), flags)) } pub fn read_guard(&self) -> ReadGuard<'_> { @@ -341,7 +352,7 @@ where /// The marker is flushed to the domain's global queue immediately so any /// thread can later collect it; it executes only after every reader /// pinned right now has unpinned. - fn retire(&self, item: Retired) { + fn retire(&self, item: Retired) { self.retire_many(std::iter::once(item)); } @@ -356,7 +367,7 @@ where /// The lock is taken once for the batch rather than once per item, which /// also stops a bulk delete interleaving its queue pushes with concurrent /// mutations for no reason. - fn retire_many(&self, items: impl IntoIterator>) { + fn retire_many(&self, items: impl IntoIterator) { let mut queued = 0usize; let mut queued_pages = 0usize; let len = { @@ -394,7 +405,7 @@ where return; } if len >= RETIREMENT_BACKLOG_WARN_AT && len.is_power_of_two() { - tracing::warn!(len, "versioned publication retirement backlog is growing"); + tracing::warn!(len, "row retirement backlog is growing"); } let reclaimable = Arc::clone(&self.reclaimable); let guard = self.epoch.pin(); @@ -524,15 +535,10 @@ where }; match item { Retired::Link(link) => { - let key = OffsetEqLink(link); - self.published_rows[publication_shard(&key)].write().remove(&key); if !queued_pages.contains(&link.page_id) { freed.push(link); } } - Retired::Publication(key) => { - self.published_rows[publication_shard(&key)].write().remove(&key); - } Retired::Page(page_id) => { // `queued_pages` has already kept this page's links out of // the buffer, so flushing here is not what upholds the @@ -557,15 +563,18 @@ where } pub fn new() -> Self { + let page = Arc::new(Data::new(1.into())); + let pages = vec![page]; Self { - published_rows: std::array::from_fn(|_| RwLock::new(PublicationMap::default())), epoch: EpochDomain::new(), retired: Mutex::new(VecDeque::new()), reclaimable: Arc::new(AtomicUsize::new(0)), pending_retirements: AtomicUsize::new(0), queued_page_retirements: AtomicUsize::new(0), // We are starting ID's from `1` because `0`'s page in file is info page. - pages: RwLock::new(vec![Arc::new(Data::new(1.into()))]), + page_directory: PageDirectory::new(&pages), + pages: ArcSwap::from_pointee(pages), + pages_write: Mutex::new(()), empty_links: EmptyLinkRegistry::::default(), empty_pages: Default::default(), row_count: AtomicU64::new(0), @@ -580,14 +589,16 @@ where Self::new() } else { let last_page_id = vec.len(); + let page_directory = PageDirectory::new(&vec); Self { - published_rows: std::array::from_fn(|_| RwLock::new(PublicationMap::default())), epoch: EpochDomain::new(), retired: Mutex::new(VecDeque::new()), reclaimable: Arc::new(AtomicUsize::new(0)), pending_retirements: AtomicUsize::new(0), queued_page_retirements: AtomicUsize::new(0), - pages: RwLock::new(vec), + page_directory, + pages: ArcSwap::from_pointee(vec), + pages_write: Mutex::new(()), empty_links: EmptyLinkRegistry::default(), empty_pages: Default::default(), row_count: AtomicU64::new(0), @@ -614,9 +625,7 @@ where // until the write through the link below has completed. Hold it // for the whole block. let _vacuum_guard = vacuum_guard; - let pages = self.pages.read(); - let current_page: usize = page_id_mapper(link.page_id.into()); - let page = &pages[current_page]; + let page = self.page_ref(link.page_id)?; let _page_guard = page.access.write(); match unsafe { page.try_save_row_by_link(&general_row, link) } { @@ -624,7 +633,6 @@ where if let Some(l) = left_link { self.empty_links.push(l); } - self.stage_published_row(link, row); self.row_count.fetch_add(1, Ordering::Relaxed); return Ok(link); } @@ -642,9 +650,9 @@ where loop { let (link, tried_page) = { - let pages = self.pages.read(); - let current_page = page_id_mapper(self.current_page_id.load(Ordering::Acquire) as usize); - let page = &pages[current_page]; + let current_page_id = self.current_page_id.load(Ordering::Acquire); + let current_page = page_id_mapper(current_page_id as usize); + let page = self.page_ref(current_page_id.into())?; let _page_guard = page.access.write(); // Re-check under the page barrier. A switch may have completed // between the load above and the lock; in the worst case the @@ -664,7 +672,6 @@ where }; match link { Ok(link) => { - self.stage_published_row(link, row); self.row_count.fetch_add(1, Ordering::Relaxed); return Ok(link); } @@ -685,8 +692,7 @@ where // the read-side grace period completes. Reset // only after reclamation made the page // available for reuse. - let pages = self.pages.read(); - let page = &pages[page_id_mapper(page_id.into())]; + let page = self.page_ref(page_id)?; let _page_guard = page.access.write(); page.reset(); self.current_page_id.store(page_id.into(), Ordering::Release); @@ -725,11 +731,15 @@ where } fn add_next_page(&self, tried_page: usize) { - let mut pages = self.pages.write(); + let _write = self.pages_write.lock(); if tried_page == page_id_mapper(self.current_page_id.load(Ordering::Acquire) as usize) { let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; - - pages.push(Arc::new(Data::new(index.into()))); + let pages = self.pages.load_full(); + let mut next = (*pages).clone(); + let page = Arc::new(Data::new(index.into())); + next.push(page.clone()); + self.pages.store(Arc::new(next)); + self.publish_page(&page); self.current_page_id.store(index, Ordering::Release); } } @@ -745,7 +755,7 @@ where }; if let Some(page_id) = page_id { - let pages = self.pages.read(); + let pages = self.pages.load(); let index = page_id_mapper(page_id.into()); let page = pages[index].clone(); { @@ -756,10 +766,14 @@ where return page; } - let mut pages = self.pages.write(); + let _write = self.pages_write.lock(); let index = self.last_page_id.fetch_add(1, Ordering::AcqRel) + 1; let page = Arc::new(Data::new(index.into())); - pages.push(page.clone()); + let pages = self.pages.load_full(); + let mut next = (*pages).clone(); + next.push(page.clone()); + self.pages.store(Arc::new(next)); + self.publish_page(&page); page } @@ -774,8 +788,7 @@ where Portable + Deserialize<::WrappedRow, HighDeserializer>, { let link = link.into(); - let slot = self.published_slot_or_hydrate(link)?; - Ok(slot.snapshot().as_ref().clone()) + self.page_row(link).map(|(row, _)| row) } pub fn select_non_ghosted(&self, link: Link) -> Result @@ -786,15 +799,14 @@ where <::WrappedRow as Archive>::Archived: Portable + Deserialize<::WrappedRow, HighDeserializer>, { - let slot = self.published_slot_or_hydrate(link)?; - let (row, flags) = slot.load(); + let (row, flags) = self.page_row(link)?; if flags & GHOSTED != 0 { return Err(ExecutionError::Ghosted); } if flags & DELETED != 0 { return Err(ExecutionError::Deleted); } - Ok(row.as_ref().clone()) + Ok(row) } /// Loads one persisted row through rkyv validation without publishing it. @@ -807,7 +819,7 @@ where + Deserialize<::WrappedRow, HighDeserializer> + for<'a> rkyv::bytecheck::CheckBytes>, { - let pages = self.pages.read(); + let pages = self.pages.load(); let page_id: usize = link.page_id.into(); let page_index = page_id .checked_sub(1) @@ -833,8 +845,7 @@ where <::WrappedRow as Archive>::Archived: Portable + Deserialize<::WrappedRow, HighDeserializer>, { - let slot = self.published_slot_or_hydrate(link)?; - let (row, flags) = slot.load(); + let (row, flags) = self.page_row(link)?; if flags & GHOSTED != 0 { return Err(ExecutionError::Ghosted); } @@ -844,7 +855,7 @@ where if flags & DELETED != 0 { return Err(ExecutionError::Deleted); } - Ok(row.as_ref().clone()) + Ok(row) } #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "DataPages"))] @@ -853,11 +864,8 @@ where Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, Op: Fn(&<::WrappedRow as Archive>::Archived) -> Res, { - let pages = self.pages.read(); - let page = pages - .get::(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let _page_guard = page.access.read(); + let page = self.page_ref(link.page_id)?; + let _cell_guard = page.read_cell(link).map_err(ExecutionError::DataPageError)?; let gen_row = page.get_row_ref(link).map_err(ExecutionError::DataPageError)?; let res = op(gen_row); Ok(res) @@ -873,11 +881,8 @@ where Deserialize<::WrappedRow, HighDeserializer>, Op: FnMut(&mut <::WrappedRow as Archive>::Archived) -> Res, { - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let _page_guard = page.access.write(); + let page = self.page_ref(link.page_id)?; + let _cell_guard = page.write_cell(link).map_err(ExecutionError::DataPageError)?; let res = { let gen_row = unsafe { page.get_mut_row_ref(link) @@ -887,11 +892,6 @@ where op(gen_row) }; - { - let wrapped = page.get_row(link).map_err(ExecutionError::DataPageError)?; - self.publish_wrapped_row(link, wrapped); - } - Ok(res) } @@ -907,24 +907,19 @@ where ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, { - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let _page_guard = page.access.write(); + let page = self.page_ref(link.page_id)?; + let _cell_guard = page.write_cell(link).map_err(ExecutionError::DataPageError)?; let gen_row = ::WrappedRow::from_inner(row.clone()); let result = unsafe { - page.save_row_by_link(&gen_row, link) + page.save_row_by_link_preserving_cell_state(&gen_row, link) .map_err(ExecutionError::DataPageError) }?; - self.stage_published_row(link, row); Ok(result) } /// In-place update of an already-live row at `link`: re-serialize the full - /// row into the SAME slot and republish it as LIVE (unghosted). Unlike - /// [`Self::update`], this does not stage the row as a new (ghosted) - /// publication — a live row that is edited must stay visible to readers. + /// row into the SAME slot and leave it LIVE (unghosted). A live row that is + /// edited must stay visible to readers. /// The caller must guarantee the new row serializes to the same length as /// the current slot (so it fits exactly). /// @@ -938,10 +933,8 @@ where /// this reason. /// /// Serialization and the exact-length check finish before any page byte is - /// changed. The page's write barrier excludes low-level archived-page - /// readers during the copy, while generated reads continue from the old immutable - /// publication until [`Self::publish_wrapped_row`] replaces the complete - /// owned row and flags together. + /// changed. The exact cell guard excludes readers of this cell during the + /// copy while unrelated cells proceed independently. /// /// # Safety /// Same contract as [`Self::update`]: `link` must be valid and no other @@ -955,12 +948,9 @@ where <::WrappedRow as Archive>::Archived: Deserialize<::WrappedRow, HighDeserializer>, { - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let _page_guard = page.access.write(); - // Write the new bytes into the slot. `save_row_by_link` requires the + let page = self.page_ref(link.page_id)?; + let _cell_guard = page.write_cell(link).map_err(ExecutionError::DataPageError)?; + // Write the new bytes into the slot. The preserving variant requires the // serialized wrapped row to be EXACTLY the slot length; the caller only // guaranteed equal *field* sizes, which need not imply equal total // serialized length (alignment/padding). If it does not fit, report it @@ -968,20 +958,17 @@ where // `row` is consumed by the wrapper here (no clone): it is not used again. let gen_row = ::WrappedRow::from_inner(row); unsafe { - page.save_row_by_link(&gen_row, link) + page.save_row_by_link_preserving_cell_state(&gen_row, link) .map_err(ExecutionError::DataPageError)?; } - // Clear the ghost bit on the stored row and republish the LIVE image - // from the page, exactly like `with_mut_ref` — so the publication cache - // is not left ghosted (a fresh `from_inner` wrapper is ghosted). + // Clear the ghost bit on the stored row. A fresh `from_inner` wrapper + // is ghosted, but this is an update of an already-live cell. unsafe { page.get_mut_row_ref(link) .map_err(ExecutionError::DataPageError)? .unseal_unchecked() .unghost(); } - let wrapped = page.get_row(link).map_err(ExecutionError::DataPageError)?; - self.publish_wrapped_row(link, wrapped); Ok(()) } @@ -995,6 +982,7 @@ where + Deserialize<::WrappedRow, HighDeserializer>, { unsafe { self.with_mut_ref(link, |r| r.delete())? } + self.remove_cell(link)?; self.row_count.fetch_sub(1, Ordering::Relaxed); self.retire(Retired::Link(link)); @@ -1036,7 +1024,10 @@ where let mut failure = None; for link in links { match unsafe { self.with_mut_ref(*link, |r| r.delete()) } { - Ok(()) => ghosted += 1, + Ok(()) => { + self.remove_cell(*link)?; + ghosted += 1; + } Err(error) => { failure = Some(error); break; @@ -1057,12 +1048,10 @@ where } pub fn select_raw(&self, link: Link) -> Result, ExecutionError> { - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let _page_guard = page.access.read(); - page.get_raw_row(link).map_err(ExecutionError::DataPageError) + let page = self.page_ref(link.page_id)?; + let _cell_guard = page.read_cell(link).map_err(ExecutionError::DataPageError)?; + page.get_raw_row_without_cell_state(link) + .map_err(ExecutionError::DataPageError) } pub fn mark_page_empty(&self, page_id: PageId) { @@ -1082,10 +1071,7 @@ where return; } - let pages = self.pages.read(); - let index = page_id_mapper(page_id.into()); - - if let Some(page) = pages.get(index) { + if let Ok(page) = self.page_ref(page_id) { let free_offset = page.free_offset.load(Ordering::Acquire); let remaining = DATA_LENGTH.saturating_sub(free_offset as usize); @@ -1108,11 +1094,37 @@ where } pub fn get_page(&self, page_id: PageId) -> Option::WrappedRow, DATA_LENGTH>>> { - let pages = self.pages.read(); + let pages = self.pages.load(); let page = pages.get(page_id_mapper(page_id.into()))?; Some(page.clone()) } + /// Registers an already-indexed cell while rebuilding runtime metadata + /// for a persisted table. + pub fn register_cell(&self, link: Link) -> Result<(), ExecutionError> { + let page = self.page_ref(link.page_id)?; + let _page_guard = page.access.read(); + page.reset_cell_state(link).map_err(ExecutionError::DataPageError)?; + page.register_cell(link); + Ok(()) + } + + fn remove_cell(&self, link: Link) -> Result<(), ExecutionError> { + let page = self.page_ref(link.page_id)?; + let _page_guard = page.access.read(); + page.remove_cell(link); + Ok(()) + } + + pub(crate) fn page_has_cells(&self, page_id: PageId) -> Result { + let page = self.page_ref(page_id)?; + Ok(page.has_live_cells()) + } + + pub(crate) fn remove_moved_cell(&self, link: Link) -> Result<(), ExecutionError> { + self.remove_cell(link) + } + /// Bytes actually occupied across every page. /// /// The sum of each page's `free_offset`, which is what `get_bytes` was @@ -1123,7 +1135,7 @@ where /// Approximate under concurrency: a failing `save_row`'s transient /// reservation may be counted before its rollback. Metrics only. pub fn used_bytes(&self) -> u64 { - let pages = self.pages.read(); + let pages = self.pages.load(); pages .iter() .map(|p| u64::from(p.free_offset.load(Ordering::Relaxed))) @@ -1152,28 +1164,16 @@ where + Portable + Deserialize<::WrappedRow, HighDeserializer>, { - let pages = self.pages.read(); - let from_page = pages - .get(page_id_mapper(from_link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(from_link.page_id))?; - let to_page = pages - .get(page_id_mapper(to_page_id.into())) - .ok_or(ExecutionError::PageNotFound(to_page_id))?; - // The one genuinely multi-page mutation: both barriers are needed, - // taken in ascending page-id order so no lock cycle can form with a - // concurrent pair. - let _page_guards = if from_link.page_id == to_page_id { - (from_page.access.write(), None) - } else if u32::from(from_link.page_id) < u32::from(to_page_id) { - let first = from_page.access.write(); - (first, Some(to_page.access.write())) - } else { - let first = to_page.access.write(); - (first, Some(from_page.access.write())) - }; + let from_page = self.page_ref(from_link.page_id)?; + let to_page = self.page_ref(to_page_id)?; + // Only the source is reachable from an index while this copy runs. + // Its exact-cell guard prevents a reader from borrowing the bytes + // while the vacuum flag is changed. Destination bytes are published + // only after the complete copy and index swing. + let _cell_guard = from_page.write_cell(from_link).map_err(ExecutionError::DataPageError)?; let raw_data = from_page - .get_raw_row(from_link) + .get_raw_row_without_cell_state(from_link) .map_err(ExecutionError::DataPageError)?; // Copy to the destination BEFORE flagging the source. The vacuumed // flag used to be set first, so a failing destination save returned @@ -1190,23 +1190,11 @@ where }; archived.set_in_vacuum_process(); - { - let old_wrapped = from_page.get_row(from_link).map_err(ExecutionError::DataPageError)?; - self.publish_wrapped_row(from_link, old_wrapped); - let new_wrapped = to_page.get_row(new_link).map_err(ExecutionError::DataPageError)?; - self.publish_wrapped_row(new_link, new_wrapped); - } - Ok((raw_data, new_link)) } - pub(crate) fn retire_published_link(&self, link: Link) { - self.retire(Retired::Publication(OffsetEqLink(link))); - self.reclaim_retired(); - } - pub fn get_page_count(&self) -> usize { - self.pages.read().len() + self.pages.load().len() } pub fn get_empty_links(&self) -> Vec { @@ -1229,7 +1217,12 @@ where /// figure without it cannot be checked, because a sweep that never runs /// looks exactly like a sweep that is free. pub fn allocated_pages(&self) -> usize { - self.pages.read().len() + self.pages.load().len() + } + + /// Heap bytes reserved by the fixed-size data-page allocations. + pub fn allocated_bytes(&self) -> usize { + self.pages.load().len() * std::mem::size_of::::WrappedRow, DATA_LENGTH>>() } /// Pages allocated but currently on the empty list, so reusable without @@ -1316,21 +1309,21 @@ impl ExecutionError { #[cfg(test)] mod tests { + use super::{DELETED, GHOSTED, VACUUMED}; use std::collections::HashSet; use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::atomic::Ordering; use std::sync::mpsc; use std::thread; use std::time::Duration; use std::time::Instant; use parking_lot::RwLock; - use rkyv::with::{AtomicLoad, Relaxed}; use rkyv::{Archive, Deserialize, Serialize}; use crate::in_memory::data::Data; use crate::in_memory::pages::{DataPages, ExecutionError}; - use crate::in_memory::{DATA_INNER_LENGTH, PagesExecutionError, RowWrapper, StorableRow}; + use crate::in_memory::{CellState, DATA_INNER_LENGTH, PagesExecutionError, RowWrapper, StorableRow}; use crate::prelude::ArchivedRowWrapper; use data_bucket::Link; @@ -1343,21 +1336,14 @@ mod tests { /// General `Row` wrapper that is used to append general data for every `Inner` /// `Row`. #[derive(Archive, Deserialize, Debug, Serialize)] + #[rkyv(attr(repr(C)))] pub struct GeneralRow { /// Inner generic `Row`. pub inner: Inner, - /// Indicator for ghosted rows. - #[rkyv(with = AtomicLoad)] - pub is_ghosted: AtomicBool, - - /// Indicator for vacuumed rows. - #[rkyv(with = AtomicLoad)] - pub is_vacuumed: AtomicBool, + pub publication_flags: u8, - /// Indicator for deleted rows. - #[rkyv(with = AtomicLoad)] - pub deleted: AtomicBool, + pub cell_state: CellState, } impl RowWrapper for GeneralRow { @@ -1366,24 +1352,23 @@ mod tests { } fn is_ghosted(&self) -> bool { - self.is_ghosted.load(Ordering::Relaxed) + self.publication_flags & GHOSTED != 0 } fn is_vacuumed(&self) -> bool { - self.is_vacuumed.load(Ordering::Relaxed) + self.publication_flags & VACUUMED != 0 } fn is_deleted(&self) -> bool { - self.deleted.load(Ordering::Relaxed) + self.publication_flags & DELETED != 0 } /// Creates new [`GeneralRow`] from `Inner`. fn from_inner(inner: Inner) -> Self { Self { inner, - is_ghosted: AtomicBool::new(true), - is_vacuumed: AtomicBool::new(false), - deleted: AtomicBool::new(false), + publication_flags: GHOSTED, + cell_state: CellState, } } } @@ -1396,17 +1381,20 @@ mod tests { where T: Archive, { + unsafe fn cell_state_ptr(this: *mut Self) -> *mut std::sync::atomic::AtomicU8 { + unsafe { std::ptr::addr_of_mut!((*this).cell_state).cast() } + } fn unghost(&mut self) { - self.is_ghosted = false + self.publication_flags &= !GHOSTED } fn set_in_vacuum_process(&mut self) { - self.is_vacuumed = true + self.publication_flags |= VACUUMED } fn delete(&mut self) { - self.deleted = true + self.publication_flags |= DELETED } fn is_deleted(&self) -> bool { - self.deleted + self.publication_flags & DELETED != 0 } } @@ -1473,11 +1461,13 @@ mod tests { } #[test] - fn versioned_reader_observes_old_row_while_page_update_is_incomplete() { + fn same_row_reader_waits_while_update_is_incomplete() { let pages = Arc::new(DataPages::::new()); let link = pages.insert(TestRow { a: 0, b: 0 }).unwrap(); + let other_link = pages.insert(TestRow { a: 9, b: 9 }).unwrap(); unsafe { pages.with_mut_ref(link, |row| row.unghost()).unwrap(); + pages.with_mut_ref(other_link, |row| row.unghost()).unwrap(); } let (first_field_written_tx, first_field_written_rx) = mpsc::channel(); @@ -1501,20 +1491,35 @@ mod tests { read_tx.send(reader_pages.select_non_ghosted(link)).unwrap(); }); + assert!( + matches!( + read_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "same-cell reader must wait instead of observing a torn row" + ); + + let (other_tx, other_rx) = mpsc::channel(); + let other_pages = pages.clone(); + let other_reader = thread::spawn(move || { + other_tx.send(other_pages.select_non_ghosted(other_link)).unwrap(); + }); assert_eq!( - read_rx.recv_timeout(Duration::from_secs(1)).unwrap(), - Ok(TestRow { a: 0, b: 0 }), - "reader must use the old immutable version instead of page bytes" + other_rx.recv_timeout(Duration::from_millis(50)).unwrap(), + Ok(TestRow { a: 9, b: 9 }), + "a writer on one cell must not block a different cell on the same page" ); + other_reader.join().unwrap(); finish_update_tx.send(()).unwrap(); writer.join().unwrap(); + assert_eq!(read_rx.recv().unwrap(), Ok(TestRow { a: 1, b: 1 })); reader.join().unwrap(); assert_eq!(pages.select_non_ghosted(link), Ok(TestRow { a: 1, b: 1 })); } #[test] - fn failed_exact_length_update_preserves_page_bytes_and_publication() { + fn failed_exact_length_update_preserves_page_bytes() { let pages = DataPages::::new(); let old_row = TestRow { a: 10, b: 20 }; let link = pages.insert(old_row).unwrap(); @@ -1539,28 +1544,6 @@ mod tests { assert_eq!(pages.select_non_ghosted(link), Ok(old_row)); } - #[test] - fn retired_version_survives_link_reuse_for_in_flight_reader() { - let pages = DataPages::::new(); - let link = pages.insert(TestRow { a: 1, b: 1 }).unwrap(); - unsafe { - pages.with_mut_ref(link, |row| row.unghost()).unwrap(); - } - let old_slot = pages.published_slot(link).unwrap(); - let old_version = old_slot.snapshot(); - - pages.delete(link).unwrap(); - let reused_link = pages.insert(TestRow { a: 2, b: 2 }).unwrap(); - assert_eq!(reused_link, link); - assert_eq!(pages.select_non_ghosted(reused_link), Err(ExecutionError::Ghosted)); - unsafe { - pages.with_mut_ref(reused_link, |row| row.unghost()).unwrap(); - } - - assert_eq!(old_version.as_ref(), &TestRow { a: 1, b: 1 }); - assert_eq!(pages.select_non_ghosted(reused_link), Ok(TestRow { a: 2, b: 2 })); - } - /// A helper thread that holds (or releases) one `ReadGuard` on command, /// so a test can interleave reader intervals across threads. One thread /// cannot model overlapping readers: nested epoch pins keep the thread's @@ -1698,7 +1681,6 @@ mod tests { pages.current_page_id.store(3, Ordering::Release); let read_guard = pages.read_guard(); - pages.retire_published_link(old_link); pages.mark_page_empty(old_link.page_id); let temporary_page = pages.allocate_new_or_pop_free(); @@ -1709,7 +1691,7 @@ mod tests { assert_eq!( pages.select_non_ghosted(old_link), Ok(TestRow { a: 1, b: 1 }), - "the old publication must survive until the reader leaves" + "the old page must survive until the reader leaves" ); drop(read_guard); @@ -1719,7 +1701,6 @@ mod tests { let reused_page = pages.allocate_new_or_pop_free(); assert_eq!(reused_page.id, old_link.page_id); assert_eq!(reused_page.free_offset.load(Ordering::Acquire), 0); - assert!(pages.published_slot(old_link).is_none()); } #[test] @@ -1929,7 +1910,7 @@ mod tests { // The source row must NOT be left flagged as in-vacuum-process: that // flag is written into the persisted page image, and with no copy on // the destination it would mean durable row loss after a restart. - let vacuumed = pages.with_ref(link, |r| r.is_vacuumed).unwrap(); + let vacuumed = pages.with_ref(link, |r| r.publication_flags & VACUUMED != 0).unwrap(); assert!(!vacuumed, "failed move must not leave the source marked vacuumed"); assert_eq!(pages.select_non_vacuumed(link), Ok(row)); } diff --git a/src/in_memory/publication.rs b/src/in_memory/publication.rs deleted file mode 100644 index 849b33fa..00000000 --- a/src/in_memory/publication.rs +++ /dev/null @@ -1,93 +0,0 @@ -use parking_lot::RwLock; -use std::fmt::{Debug, Formatter}; -use std::sync::Arc; - -pub(super) const GHOSTED: u8 = 1 << 0; -pub(super) const DELETED: u8 = 1 << 1; -pub(super) const VACUUMED: u8 = 1 << 2; - -/// One immutable application-visible row version plus atomic lifecycle bits. -/// -/// Readers hold an `Arc` to a complete version, so replacing or retiring a -/// version cannot invalidate an in-flight read. The short per-row lock keeps -/// the `Arc` and its lifecycle flags in one coherent publication; readers -/// never access mutable archived bytes. -struct PublishedVersion { - row: Arc, - flags: u8, -} - -pub(super) struct PublishedRow { - version: RwLock>, -} - -impl PublishedRow { - pub(super) fn new(row: Row, flags: u8) -> Self { - Self { - version: RwLock::new(PublishedVersion { - row: Arc::new(row), - flags, - }), - } - } - - pub(super) fn replace(&self, row: Row, flags: u8) { - *self.version.write() = PublishedVersion { - row: Arc::new(row), - flags, - }; - } - - pub(super) fn load(&self) -> (Arc, u8) { - let version = self.version.read(); - (version.row.clone(), version.flags) - } - - pub(super) fn snapshot(&self) -> Arc { - self.version.read().row.clone() - } -} - -impl Debug for PublishedRow { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let version = self.version.read(); - f.debug_struct("PublishedRow") - .field("flags", &version.flags) - .finish_non_exhaustive() - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; - use std::thread; - - use super::PublishedRow; - - #[test] - fn row_and_flags_are_loaded_from_one_version() { - const ITERATIONS: usize = 100_000; - - let published = Arc::new(PublishedRow::new(0_u8, 0)); - let done = Arc::new(AtomicBool::new(false)); - let writer = { - let published = published.clone(); - let done = done.clone(); - thread::spawn(move || { - for value in 0..ITERATIONS { - let state = (value & 1) as u8; - published.replace(state, state); - } - done.store(true, Ordering::Release); - }) - }; - - while !done.load(Ordering::Acquire) { - let (row, flags) = published.load(); - assert_eq!(*row, flags); - } - - writer.join().unwrap(); - } -} diff --git a/src/in_memory/row.rs b/src/in_memory/row.rs index 89eedaa1..b730049a 100644 --- a/src/in_memory/row.rs +++ b/src/in_memory/row.rs @@ -1,6 +1,38 @@ use std::fmt::Debug; +use std::sync::atomic::AtomicU8; -use rkyv::Archive; +use rkyv::rancor::Fallible; +use rkyv::{Archive, Deserialize, Place, Serialize}; + +/// Runtime synchronization state embedded as the first byte of every archived +/// cell wrapper. +/// +/// The source value is zero-sized; its archived representation is one byte. +/// Deserialization deliberately ignores that byte because active readers +/// modify it atomically. It is synchronization state, never row data. +#[derive(Clone, Copy, Debug, Default)] +pub struct CellState; + +impl Archive for CellState { + type Archived = u8; + type Resolver = (); + + fn resolve(&self, _: Self::Resolver, out: Place) { + out.write(0); + } +} + +impl Serialize for CellState { + fn serialize(&self, _: &mut S) -> Result { + Ok(()) + } +} + +impl Deserialize for u8 { + fn deserialize(&self, _: &mut D) -> Result { + Ok(CellState) + } +} pub trait PublicationSafe: Send + Sync + 'static {} @@ -10,7 +42,7 @@ impl PublicationSafe for T {} /// /// [`Data`]: crate::in_memory::data::Data pub trait StorableRow: PublicationSafe { - type WrappedRow: Archive + Debug; + type WrappedRow: Archive + Debug; } pub trait RowWrapper { @@ -22,6 +54,14 @@ pub trait RowWrapper { } pub trait ArchivedRowWrapper { + /// Returns the atomic synchronization byte for this archived cell without + /// first creating a reference to the rest of the row. Implementations must + /// place `cell_state` in a stable position in a `repr(C)` archived wrapper. + /// + /// # Safety + /// + /// `this` must point to a valid archived wrapper in writable page memory. + unsafe fn cell_state_ptr(this: *mut Self) -> *mut AtomicU8; fn unghost(&mut self); fn set_in_vacuum_process(&mut self); fn delete(&mut self); diff --git a/src/index/arctic.rs b/src/index/arctic.rs index 329a74dd..2e13c132 100644 --- a/src/index/arctic.rs +++ b/src/index/arctic.rs @@ -2,6 +2,7 @@ use std::borrow::Borrow; use std::fmt::{self, Debug}; +use std::marker::PhantomData; use std::ops::{Bound, RangeBounds}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -9,64 +10,63 @@ use arctic::{ConcurrentMap, Key, Order}; use super::UniqueIndex; +type ArcticSmr = arctic::concurrent::smr::PsReclaim; + /// Lossless conversion between a WorkTable key and a native Arctic key. /// /// Keeping this trait local lets generated primary-key newtypes delegate to /// their underlying integer without implementing Arctic's low-level key API. pub trait ArcticKey: Clone + Debug + Ord + Send + Sync + 'static { - type Raw: ArcticRawKey; + type Raw: Key + Clone + Debug + Ord + Send + Sync + 'static; fn to_arctic(&self) -> Self::Raw; fn from_arctic(value: Self::Raw) -> Self; } -/// Integer operations needed to translate Rust's inclusive/exclusive bounds -/// into the native range forms accepted by Arctic 0.1. -#[doc(hidden)] -pub trait ArcticRawKey: Key + Copy + Debug + Ord + Send + Sync + 'static { - fn next(self) -> Option; - fn previous(self) -> Option; +/// Arctic key used for arbitrary UTF-8 strings. +/// +/// Arctic's variable-sized keys require a byte sequence with no zero byte so +/// the tree can append its own logical terminator. Valid UTF-8 never contains +/// `0xff`, therefore adding one to every encoded byte is a lossless, +/// order-preserving mapping into `1..=0xf5`. This includes Rust strings that +/// contain `\0`, without reserving a value or changing their ordering. +pub type ArcticStringKey = arctic::key::BoxedSlice; + +fn encode_string(value: &str) -> ArcticStringKey { + let encoded = value + .as_bytes() + .iter() + .map(|byte| byte.checked_add(1).expect("UTF-8 bytes never reach 0xff")) + .collect::>() + .into_boxed_slice(); + ArcticStringKey::new(encoded).expect("the shifted UTF-8 encoding contains no zero byte") } -macro_rules! impl_arctic_raw_key { - ($($ty:ty),* $(,)?) => { - $( - impl ArcticRawKey for $ty { - #[inline] - fn next(self) -> Option { self.checked_add(1) } - - #[inline] - fn previous(self) -> Option { self.checked_sub(1) } - } - )* - }; +fn decode_string(value: ArcticStringKey) -> String { + let decoded = value + .into_boxed_slice() + .into_vec() + .into_iter() + .map(|byte| { + byte.checked_sub(1) + .expect("Arctic string encoding contains no zero byte") + }) + .collect::>(); + String::from_utf8(decoded).expect("Arctic string key was not encoded from UTF-8") } -impl_arctic_raw_key!(u16, u32, u64, u128); +impl ArcticKey for String { + type Raw = ArcticStringKey; -/// Inclusive native bounds: `None` when the range is provably empty, and -/// `None` on a side for an unbounded side. -pub(crate) type RawInclusiveBounds = Option<(Option, Option)>; + #[inline] + fn to_arctic(&self) -> Self::Raw { + encode_string(self) + } -/// Translates Rust range bounds over `K` into the inclusive native bounds -/// Arctic accepts. -/// -/// An `Excluded` bound whose neighbour does not exist (`next()` on the -/// maximum key, `previous()` on zero) makes the range empty, which is -/// signalled as `None`. It must not fall through to an unbounded side, which -/// would return the whole table for an empty range. -pub(crate) fn raw_inclusive_bounds(range: &impl RangeBounds) -> RawInclusiveBounds { - let lower = match range.start_bound() { - Bound::Included(key) => Some(key.to_arctic()), - Bound::Excluded(key) => Some(key.to_arctic().next()?), - Bound::Unbounded => None, - }; - let upper = match range.end_bound() { - Bound::Included(key) => Some(key.to_arctic()), - Bound::Excluded(key) => Some(key.to_arctic().previous()?), - Bound::Unbounded => None, - }; - Some((lower, upper)) + #[inline] + fn from_arctic(value: Self::Raw) -> Self { + decode_string(value) + } } macro_rules! impl_arctic_key { @@ -96,9 +96,8 @@ impl_arctic_key!(u16, u32, u64, u128); /// property the tree needs, so `i64::MIN` lands at `0`, `-1` at `0x7fff_..._ffff`, /// `0` at `0x8000_..._0000` and `i64::MAX` at `u64::MAX`. /// -/// Being a bijection over the *whole* width is also what makes the exclusive -/// bounds in `raw_inclusive_bounds` correct: `next`/`previous` run in the raw -/// space, and adjacency is preserved because no raw value is unreachable. +/// Being a bijection over the *whole* width also preserves adjacency, which +/// keeps excluded range bounds exact in the raw key space. /// /// There is no `i8`, because Arctic's narrowest raw key is `u16`. macro_rules! impl_arctic_signed_key { @@ -123,19 +122,76 @@ macro_rules! impl_arctic_signed_key { impl_arctic_signed_key!(i16 => u16, i32 => u32, i64 => u64, i128 => u128); +/// Lossless codec for values held inline by Arctic. +#[doc(hidden)] +pub trait ArcticValue: Clone + Debug + Send + Sync + 'static { + fn into_arctic(self) -> u64; + fn from_arctic(value: u64) -> Self; +} + +impl ArcticValue for u64 { + #[inline] + fn into_arctic(self) -> u64 { + self + } + + #[inline] + fn from_arctic(value: u64) -> Self { + value + } +} + +fn pack_link(link: data_bucket::Link) -> u64 { + assert!(link.offset <= u16::MAX.into(), "link offset exceeds Arctic encoding"); + assert!(link.length <= u16::MAX.into(), "link length exceeds Arctic encoding"); + (u64::from(u32::from(link.page_id)) << 32) | (u64::from(link.offset) << 16) | u64::from(link.length) +} + +fn unpack_link(value: u64) -> data_bucket::Link { + data_bucket::Link { + page_id: ((value >> 32) as u32).into(), + offset: ((value >> 16) as u32) & u32::from(u16::MAX), + length: value as u32 & u32::from(u16::MAX), + } +} + +impl ArcticValue for data_bucket::Link { + #[inline] + fn into_arctic(self) -> u64 { + pack_link(self) + } + + #[inline] + fn from_arctic(value: u64) -> Self { + unpack_link(value) + } +} + +impl ArcticValue for crate::util::OffsetEqLink { + #[inline] + fn into_arctic(self) -> u64 { + pack_link(self.0) + } + + #[inline] + fn from_arctic(value: u64) -> Self { + Self(unpack_link(value)) + } +} + /// Arctic's lock-free adaptive radix tree with WorkTable's unique-index /// contract. /// -/// WorkTable links are stored as boxed values because Arctic's inline value -/// representation is limited to 64 bits. Point operations remain directly -/// backed by Arctic; ordered scans are collected into a stable snapshot to -/// satisfy WorkTable's double-ended query interface. -pub struct ArcticIndex { - inner: ConcurrentMap>, +/// WorkTable links are packed into Arctic's inline 64-bit value. Point +/// operations remain directly backed by Arctic; ordered scans are collected +/// into a stable snapshot to satisfy WorkTable's double-ended query interface. +pub struct ArcticIndex { + inner: ConcurrentMap, len: AtomicUsize, + marker: PhantomData V>, } -impl Debug for ArcticIndex { +impl Debug for ArcticIndex { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ArcticIndex") .field("len", &self.len.load(Ordering::Relaxed)) @@ -146,11 +202,13 @@ impl Debug for ArcticIndex { impl Default for ArcticIndex where K: ArcticKey, + V: ArcticValue, { fn default() -> Self { Self { inner: ConcurrentMap::default(), len: AtomicUsize::new(0), + marker: PhantomData, } } } @@ -159,7 +217,7 @@ impl ArcticIndex where K: ArcticKey, K::Raw: arctic::topology::Key, - V: Clone + Debug + Send + Sync + 'static, + V: ArcticValue, { /// Every entry, ascending. /// @@ -181,18 +239,19 @@ where &mut self, mut encode: impl FnMut(&V) -> T, ) -> Result, arctic::topology::Error> { - self.inner.export_topology(|value| encode(value)) + self.inner.export_topology(|value| encode(&V::from_arctic(*value))) } pub(crate) fn from_topology( topology: arctic::topology::Topology, mut decode: impl FnMut(T) -> V, ) -> Result { - let inner = ConcurrentMap::from_topology(topology, |value| Box::new(decode(value)))?; + let inner = ConcurrentMap::from_topology(topology, |value| decode(value).into_arctic())?; let len = inner.all().entries(Order::Ascend).count(); Ok(Self { inner, len: AtomicUsize::new(len), + marker: PhantomData, }) } } @@ -200,7 +259,7 @@ where impl UniqueIndex for ArcticIndex where K: ArcticKey, - V: Clone + Debug + Send + Sync + 'static, + V: ArcticValue, { #[inline] fn get_value(&self, key: &K) -> Option { @@ -210,7 +269,10 @@ where #[inline] fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { let key = key.to_arctic(); - self.inner.get(key.borrow()).map(|value| read(&value)) + self.inner.get(key.borrow()).map(|value| { + let decoded = V::from_arctic(*value); + read(&decoded) + }) } #[inline] @@ -222,8 +284,8 @@ where #[inline] fn insert_value(&self, key: K, value: V) -> Option { let key = key.to_arctic(); - let updated = self.inner.upsert(key.as_insert(), Box::new(value)); - let old = updated.old().cloned(); + let updated = self.inner.upsert(key.as_insert(), value.into_arctic()); + let old = updated.old().copied().map(V::from_arctic); if old.is_none() { self.len.fetch_add(1, Ordering::Relaxed); } @@ -233,13 +295,13 @@ where #[inline] fn insert_value_checked(&self, key: K, value: V) -> Option<()> { let key = key.to_arctic(); - match self.inner.insert(key.as_insert(), Box::new(value)) { + match self.inner.insert(key.as_insert(), value.into_arctic()) { Ok(_) => { self.len.fetch_add(1, Ordering::Relaxed); Some(()) } Err((_old, new)) => { - drop(new); + let _ = new; None } } @@ -250,7 +312,7 @@ where let raw_key = key.to_arctic(); let old = self.inner.remove(raw_key.borrow())?; self.len.fetch_sub(1, Ordering::Relaxed); - Some((key.clone(), (*old).clone())) + Some((key.clone(), V::from_arctic(*old))) } #[inline] @@ -262,7 +324,7 @@ where let shard = self.inner.all(); shard .entries(Order::Ascend) - .map(|(key, value)| (K::from_arctic(key), value.clone())) + .map(|(key, value)| (K::from_arctic(key), V::from_arctic(value))) .collect::>() .into_iter() } @@ -275,8 +337,13 @@ where where R: RangeBounds + 'a, { - let Some((lower, upper)) = raw_inclusive_bounds(&range) else { - return Vec::new().into_iter(); + let lower = match range.start_bound() { + Bound::Included(key) | Bound::Excluded(key) => Some(key.to_arctic()), + Bound::Unbounded => None, + }; + let upper = match range.end_bound() { + Bound::Included(key) | Bound::Excluded(key) => Some(key.to_arctic()), + Bound::Unbounded => None, }; macro_rules! collect_range { @@ -284,7 +351,7 @@ where self.inner .range($native_range) .entries(Order::Ascend) - .map(|(key, value)| (K::from_arctic(key), value.clone())) + .map(|(key, value)| (K::from_arctic(key), V::from_arctic(value))) .collect::>() }}; } @@ -300,9 +367,12 @@ where .inner .all() .entries(Order::Ascend) - .map(|(key, value)| (K::from_arctic(key), value.clone())) + .map(|(key, value)| (K::from_arctic(key), V::from_arctic(value))) .collect(), - }; + } + .into_iter() + .filter(move |(key, _)| range.contains(key)) + .collect::>(); values.into_iter() } @@ -422,6 +492,30 @@ mod tests { assert_eq!(index.range_values(10..).collect::>(), Vec::new()); } + #[test] + fn arbitrary_strings_round_trip_and_scan_in_utf8_order() { + let index = ArcticIndex::::default(); + let keys = ["", "\0", "a", "a\0", "aa", "é", "🦀"]; + for (value, key) in keys.iter().enumerate().rev() { + assert_eq!(index.insert_value_checked((*key).to_owned(), value as u64), Some(())); + } + + for (value, key) in keys.iter().enumerate() { + assert_eq!(index.get_value(&(*key).to_owned()), Some(value as u64)); + } + assert_eq!( + index.iter_values().map(|(key, _)| key).collect::>(), + keys.map(str::to_owned) + ); + assert_eq!( + index + .range_values((Bound::Excluded("a".to_owned()), Bound::Included("é".to_owned()))) + .map(|(key, _)| key) + .collect::>(), + vec!["a\0".to_owned(), "aa".to_owned(), "é".to_owned()] + ); + } + #[test] fn excluded_bounds_without_neighbours_yield_empty_ranges() { let index = ArcticIndex::::default(); diff --git a/src/index/arctic_multi.rs b/src/index/arctic_multi.rs index 37cc8776..c0b9664e 100644 --- a/src/index/arctic_multi.rs +++ b/src/index/arctic_multi.rs @@ -47,13 +47,13 @@ use std::borrow::Borrow; use std::fmt::{self, Debug}; -use std::ops::{ControlFlow, RangeBounds}; +use std::ops::{Bound, ControlFlow, RangeBounds}; use std::sync::atomic::{AtomicUsize, Ordering}; use arctic::{ConcurrentMap, Key as ArcticNativeKey, Order}; use parking_lot::RwLock; -use super::arctic::{ArcticKey, raw_inclusive_bounds}; +use super::arctic::ArcticKey; /// Links of a single key, guarded by the slot's `RwLock`. struct LinkSlot { @@ -219,8 +219,13 @@ where where R: RangeBounds + 'a, { - let Some((lower, upper)) = raw_inclusive_bounds(&range) else { - return Vec::new().into_iter(); + let lower = match range.start_bound() { + Bound::Included(key) | Bound::Excluded(key) => Some(key.to_arctic()), + Bound::Unbounded => None, + }; + let upper = match range.end_bound() { + Bound::Included(key) | Bound::Excluded(key) => Some(key.to_arctic()), + Bound::Unbounded => None, }; // `EntryIter` only implements `Iterator` for cloneable payloads, so @@ -238,7 +243,7 @@ where links .links .iter() - .map(|value| (K::from_arctic(raw), value.clone())), + .map(|value| (K::from_arctic(raw.clone()), value.clone())), ); } pairs @@ -253,7 +258,10 @@ where (Some(lower), None) => collect_range!(self.inner.range(lower.borrow()..)), (None, Some(upper)) => collect_range!(self.inner.range(..=upper.borrow())), (None, None) => collect_range!(self.inner.all()), - }; + } + .into_iter() + .filter(move |(key, _)| range.contains(key)) + .collect::>(); values.into_iter() } diff --git a/src/index/mod.rs b/src/index/mod.rs index afeec301..558cff14 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -11,7 +11,7 @@ mod table_secondary_index; mod unique; mod unsized_node; -pub use arctic::{ArcticIndex, ArcticKey}; +pub use arctic::{ArcticIndex, ArcticKey, ArcticStringKey, ArcticValue}; pub use arctic_multi::ArcticMultiIndex; pub use available_index::AvailableIndex; pub use congee::{CongeeIndex, CongeeKey}; diff --git a/src/index/primary_index.rs b/src/index/primary_index.rs index 7e1adaec..36268131 100644 --- a/src/index/primary_index.rs +++ b/src/index/primary_index.rs @@ -1,10 +1,8 @@ -//! Combined storage for primary and reverse indexes. -//! -//! [`PrimaryIndex`] keeps both the primary key index (PK → [`OffsetEqLink`]) -//! and the reverse index ([`OffsetEqLink`] → PK) in sync. +//! Primary-key to row-location index. use std::fmt::Debug; use std::hash::Hash; +use std::marker::PhantomData; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; @@ -13,34 +11,44 @@ use indexset::core::pair::Pair; use crate::util::OffsetEqLink; use crate::{IndexMap, TableIndex, TableIndexCdc, UniqueIndex}; -/// Combined storage for primary and reverse indexes. +/// Primary-key to physical-row mapping. /// -/// Maintains bidirectional mapping between primary keys and their data locations: -/// - **Forward index**: `PrimaryKey` → [`OffsetEqLink`] (primary lookups) -/// - **Reverse index**: [`OffsetEqLink`] → `PrimaryKey` (vacuum, position queries) +/// Vacuum enumerates a compact per-page cell directory and reads the primary +/// key already stored in each row. Keeping a second link-to-key index here +/// would duplicate every key solely for maintenance. #[derive(Debug)] pub struct PrimaryIndex>> where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + Hash, PkMap: UniqueIndex>, { pub pk_map: PkMap, - pub reverse_pk_map: IndexMap, PrimaryKey>, + marker: PhantomData PrimaryKey>, } -impl Default for PrimaryIndex +impl PrimaryIndex where - PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + PrimaryKey: Clone + Ord + Send + 'static + Hash, PkMap: UniqueIndex>, { - fn default() -> Self { + pub fn from_map(pk_map: PkMap) -> Self { Self { - pk_map: PkMap::default(), - reverse_pk_map: IndexMap::default(), + pk_map, + marker: PhantomData, } } } +impl Default for PrimaryIndex +where + PrimaryKey: Clone + Ord + Send + 'static + Hash, + PkMap: UniqueIndex>, +{ + fn default() -> Self { + Self::from_map(PkMap::default()) + } +} + impl TableIndex for PrimaryIndex where @@ -48,32 +56,15 @@ where PkMap: UniqueIndex>, { fn insert(&self, value: PrimaryKey, link: Link) -> Option { - let offset_link = OffsetEqLink(link); - let old = self.pk_map.insert_value(value.clone(), offset_link); - if let Some(old_link) = old { - // Update reverse index - self.reverse_pk_map.remove(&old_link); - } - self.reverse_pk_map.insert(offset_link, value); - old.map(|l| l.0) + self.pk_map.insert_value(value, OffsetEqLink(link)).map(|old| old.0) } fn insert_checked(&self, value: PrimaryKey, link: Link) -> Option<()> { - let offset_link = OffsetEqLink(link); - self.pk_map.insert_value_checked(value.clone(), offset_link)?; - if self.reverse_pk_map.checked_insert(offset_link, value.clone()).is_none() { - // The link is already owned by another key. Roll the forward - // insert back, or pk_map keeps pointing at a link the caller is - // about to retire. - self.pk_map.remove_value(&value); - return None; - } - Some(()) + self.pk_map.insert_value_checked(value, OffsetEqLink(link)) } fn remove(&self, value: &PrimaryKey, _: Link) -> Option<(PrimaryKey, Link)> { let (_, old_link) = self.pk_map.remove_value(value)?; - self.reverse_pk_map.remove(&old_link); Some((value.clone(), old_link.0)) } } @@ -85,28 +76,11 @@ where PkMap: UniqueIndex> + TableIndexCdc, { fn insert_cdc(&self, value: PrimaryKey, link: Link) -> (Option, Vec>>) { - let offset_link = OffsetEqLink(link); - let (old_link, events) = TableIndexCdc::insert_cdc(&self.pk_map, value.clone(), link); - if let Some(old_link) = old_link { - self.reverse_pk_map.remove(&OffsetEqLink(old_link)); - } - self.reverse_pk_map.insert(offset_link, value); - - (old_link, events) + TableIndexCdc::insert_cdc(&self.pk_map, value, link) } fn insert_checked_cdc(&self, value: PrimaryKey, link: Link) -> Option>>> { - let offset_link = OffsetEqLink(link); - let events = TableIndexCdc::insert_checked_cdc(&self.pk_map, value.clone(), link)?; - if self.reverse_pk_map.checked_insert(offset_link, value.clone()).is_none() { - // Same invariant as `insert_checked`: a link owned by another key - // must fail the whole insert instead of silently rebinding the - // reverse entry. The forward insert is rolled back; its events are - // dropped together with the rollback's, cancelling each other out. - let _ = TableIndexCdc::remove_cdc(&self.pk_map, value, link); - return None; - } - Some(events) + TableIndexCdc::insert_checked_cdc(&self.pk_map, value, link) } fn remove_cdc( @@ -114,13 +88,7 @@ where value: PrimaryKey, link: Link, ) -> (Option<(PrimaryKey, Link)>, Vec>>) { - let (removed, events) = TableIndexCdc::remove_cdc(&self.pk_map, value, link); - if let Some((key, old_link)) = removed { - self.reverse_pk_map.remove(&OffsetEqLink(old_link)); - (Some((key, old_link)), events) - } else { - (None, events) - } + TableIndexCdc::remove_cdc(&self.pk_map, value, link) } } @@ -130,344 +98,49 @@ mod tests { use data_bucket::page::PageId; const TEST_DATA_LENGTH: usize = 4096; + type TestPrimaryIndex = PrimaryIndex; - type TestPrimaryIndex = PrimaryIndex; - - #[test] - fn test_default_creates_empty_indexes() { - let index = TestPrimaryIndex::default(); - assert_eq!(index.pk_map.len(), 0); - assert_eq!(index.reverse_pk_map.len(), 0); - } - - #[test] - fn test_insert_creates_bidirectional_mapping() { - let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - index.insert(42, link); - - assert_eq!(index.pk_map.get(&42).map(|v| v.get().value.0), Some(link)); - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link)).map(|v| v.get().value), - Some(42) - ); - } - - #[test] - fn test_insert_returns_old_link_on_duplicate() { - let index = TestPrimaryIndex::default(); - let link1 = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - let link2 = Link { - page_id: PageId::from(2), - offset: 200, - length: 50, - }; - - index.insert(42, link1); - let old = index.insert(42, link2); - - assert_eq!(old, Some(link1)); - assert_eq!(index.pk_map.get(&42).map(|v| v.get().value.0), Some(link2)); - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link2)).map(|v| v.get().value), - Some(42) - ); - assert!(index.reverse_pk_map.get(&OffsetEqLink(link1)).is_none()); - } - - #[test] - fn test_insert_checked_succeeds_on_new_key() { - let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - let result = index.insert_checked(42, link); - assert_eq!(result, Some(())); - - assert_eq!(index.pk_map.get(&42).map(|v| v.get().value.0), Some(link)); - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link)).map(|v| v.get().value), - Some(42) - ); - } - - #[test] - fn test_insert_checked_fails_on_duplicate() { - let index = TestPrimaryIndex::default(); - let link1 = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - let link2 = Link { - page_id: PageId::from(2), - offset: 200, - length: 50, - }; - - index.insert_checked(42, link1).unwrap(); - let result = index.insert_checked(42, link2); - - assert_eq!(result, None); - assert_eq!(index.pk_map.get(&42).map(|v| v.get().value.0), Some(link1)); - } - - #[test] - fn test_insert_checked_rolls_back_forward_entry_when_link_is_taken() { - let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - index.insert_checked(1, link).unwrap(); - // A second key claiming the same physical link must fail atomically. - let result = index.insert_checked(2, link); - - assert_eq!(result, None); - assert!( - index.pk_map.get(&2).is_none(), - "forward entry must be rolled back when the reverse insert fails" - ); - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link)).map(|v| v.get().value), - Some(1), - "reverse entry must keep its original owner" - ); - } - - #[test] - fn test_insert_checked_cdc_rolls_back_forward_entry_when_link_is_taken() { - let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - index.insert_checked_cdc(1, link).unwrap(); - let events = index.insert_checked_cdc(2, link); - - assert!(events.is_none()); - assert!( - index.pk_map.get(&2).is_none(), - "forward entry must be rolled back when the reverse insert fails" - ); - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link)).map(|v| v.get().value), - Some(1), - "reverse entry must not be silently rebound" - ); - } - - #[test] - fn test_removing_existing_key() { - let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - index.insert(42, link); - let removed = index.remove(&42, link); - - assert_eq!(removed, Some((42, link))); - assert!(index.pk_map.get(&42).is_none()); - assert!(index.reverse_pk_map.get(&OffsetEqLink(link)).is_none()); - } - - #[test] - fn test_removing_nonexistent_key_returns_none() { - let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - let removed = index.remove(&42, link); - assert_eq!(removed, None); - } - - #[test] - fn test_insert_cdc_new_key() { - let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - let (old_link, _events) = index.insert_cdc(42, link); - - assert_eq!(old_link, None); - assert_eq!(index.pk_map.get(&42).map(|v| v.get().value.0), Some(link)); - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link)).map(|v| v.get().value), - Some(42) - ); - } - - #[test] - fn test_insert_cdc_existing_key() { - let index = TestPrimaryIndex::default(); - let link1 = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - let link2 = Link { - page_id: PageId::from(2), - offset: 200, - length: 50, - }; - - index.insert_cdc(42, link1); - let (old_link, _events) = index.insert_cdc(42, link2); - - assert_eq!(old_link, Some(link1)); - assert_eq!(index.pk_map.get(&42).map(|v| v.get().value.0), Some(link2)); - assert!(index.reverse_pk_map.get(&OffsetEqLink(link1)).is_none()); - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link2)).map(|v| v.get().value), - Some(42) - ); - } - - #[test] - fn test_insert_checked_cdc_new_key() { - let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - let events = index.insert_checked_cdc(42, link); - - assert!(events.is_some()); - assert_eq!(index.pk_map.get(&42).map(|v| v.get().value.0), Some(link)); - } - - #[test] - fn test_insert_checked_cdc_existing_key() { - let index = TestPrimaryIndex::default(); - let link1 = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - let link2 = Link { - page_id: PageId::from(2), - offset: 200, - length: 50, - }; - - index.insert_checked_cdc(42, link1).unwrap(); - let events = index.insert_checked_cdc(42, link2); - - assert!(events.is_none()); - assert_eq!(index.pk_map.get(&42).map(|v| v.get().value.0), Some(link1)); + fn link(page: u32, offset: u32) -> Link { + Link { + page_id: PageId::from(page), + offset, + length: 16, + } } #[test] - fn test_remove_cdc_existing_key() { - let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - index.insert_cdc(42, link); - let (removed, _events) = index.remove_cdc(42, link); - - assert_eq!(removed, Some((42, link))); - assert!(index.pk_map.get(&42).is_none()); - assert!(index.reverse_pk_map.get(&OffsetEqLink(link)).is_none()); + fn default_is_empty() { + assert_eq!(TestPrimaryIndex::default().pk_map.len(), 0); } #[test] - fn test_remove_cdc_nonexistent_key() { + fn insert_replaces_the_forward_location() { let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - let (removed, _events) = index.remove_cdc(42, link); + let first = link(1, 0); + let second = link(2, 16); - assert_eq!(removed, None); + assert_eq!(index.insert(42, first), None); + assert_eq!(index.insert(42, second), Some(first)); + assert_eq!(index.pk_map.get_value(&42), Some(OffsetEqLink(second))); } #[test] - fn test_multiple_keys_maintain_separate_mappings() { + fn checked_insert_rejects_an_existing_key() { let index = TestPrimaryIndex::default(); - let link1 = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - let link2 = Link { - page_id: PageId::from(2), - offset: 200, - length: 50, - }; - let link3 = Link { - page_id: PageId::from(3), - offset: 300, - length: 50, - }; + let first = link(1, 0); - index.insert(1, link1); - index.insert(2, link2); - index.insert(3, link3); - - assert_eq!(index.pk_map.get(&1).map(|v| v.get().value.0), Some(link1)); - assert_eq!(index.pk_map.get(&2).map(|v| v.get().value.0), Some(link2)); - assert_eq!(index.pk_map.get(&3).map(|v| v.get().value.0), Some(link3)); - - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link1)).map(|v| v.get().value), - Some(1) - ); - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link2)).map(|v| v.get().value), - Some(2) - ); - assert_eq!( - index.reverse_pk_map.get(&OffsetEqLink(link3)).map(|v| v.get().value), - Some(3) - ); + assert_eq!(index.insert_checked(42, first), Some(())); + assert_eq!(index.insert_checked(42, link(2, 0)), None); + assert_eq!(index.pk_map.get_value(&42), Some(OffsetEqLink(first))); } #[test] - fn test_reverse_lookup_by_link() { + fn remove_returns_the_indexed_location() { let index = TestPrimaryIndex::default(); - let link = Link { - page_id: PageId::from(1), - offset: 100, - length: 50, - }; - - index.insert(42, link); + let row_link = link(1, 0); + index.insert(42, row_link); - let pk = index.reverse_pk_map.get(&OffsetEqLink(link)).map(|v| v.get().value); - assert_eq!(pk, Some(42)); + assert_eq!(index.remove(&42, row_link), Some((42, row_link))); + assert_eq!(index.pk_map.get_value(&42), None); } } diff --git a/src/lib.rs b/src/lib.rs index 6e1c06de..b8fb16f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,7 @@ mod util; pub mod features; pub use index::*; -pub use persistence::{LoadMode, PersistedWorkTable, PersistenceConfig, PersistenceLoadError}; +pub use persistence::{LoadMode, PersistedWorkTable, PersistenceConfig, PersistenceLoadError, UnloadReport}; pub use row::*; pub use table::*; @@ -32,7 +32,7 @@ pub use worktable_dsl; pub use worktable_codegen::s3_sync_persistence; pub mod prelude { - pub use crate::in_memory::{ArchivedRowWrapper, Data, DataPages, Query, RowWrapper, StorableRow}; + pub use crate::in_memory::{ArchivedRowWrapper, CellState, Data, DataPages, Query, RowWrapper, StorableRow}; pub use crate::lock::FullRowLock; pub use crate::lock::{Lock, RowLock}; pub use crate::lock::{LockAcquirer, LockGuard, LockMap, PendingLock}; @@ -43,10 +43,11 @@ pub mod prelude { IndexTableOfContents, InsertOperation, LoadMode, Operation, OperationId, PersistedWorkTable, PersistenceConfig, PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, PersistenceMonitor, PersistenceResult, PersistenceState, PersistenceTask, ReadOnlyPersistenceEngine, SpaceArcticIndex, - SpaceArcticMultiIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, - SpaceLogicalIndex, SpaceLogicalIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, UpdateOperation, - load_persisted_state, map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general, - reconstruct_multi_index_nodes, validate_events, + SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, + SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, SpaceSecondaryIndexOps, + TocEntryOversizedError, UnloadReport, UpdateOperation, load_persisted_state, + map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, + validate_events, }; pub use crate::primary_key::{ PrimaryKeyGenerator, PrimaryKeyGeneratorRange, PrimaryKeyGeneratorState, TablePrimaryKey, @@ -55,12 +56,13 @@ pub mod prelude { pub use crate::table::system_info::{IndexInfo, IndexKind, SystemInfo}; pub use crate::util::{OffsetEqLink, OrderedF32Def, OrderedF64Def}; pub use crate::{ - ArcticIndex, ArcticKey, ArcticMultiIndex, AvailableIndex, BatchDeleteError, BatchInsertError, CongeeIndex, - CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, PersistentArcticIndex, - PersistentArcticMultiIndex, PersistentArtIndex, PersistentCongeeIndex, PersistentWtiIndex, PrimaryIndex, - TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, - TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, UpstreamIndexMap, UpstreamIndexPair, WorkTable, - WorkTableError, vacuum::EmptyDataVacuum, vacuum::VacuumPersistence, vacuum::WorkTableVacuum, + ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticStringKey, AvailableIndex, BatchDeleteError, BatchInsertError, + CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, + PersistentArcticIndex, PersistentArcticMultiIndex, PersistentArtIndex, PersistentCongeeIndex, + PersistentWtiIndex, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, + TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, + UpstreamIndexMap, UpstreamIndexPair, WorkTable, WorkTableError, vacuum::EmptyDataVacuum, + vacuum::VacuumPersistence, vacuum::WorkTableVacuum, }; pub use data_bucket::{ DATA_VERSION, DataPage, GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, INNER_PAGE_SIZE, IndexPage, Interval, diff --git a/src/mem_stat/mod.rs b/src/mem_stat/mod.rs index adc641a4..0b760647 100644 --- a/src/mem_stat/mod.rs +++ b/src/mem_stat/mod.rs @@ -17,12 +17,13 @@ use uuid::Uuid; use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; use vanilla_indexset::core::pair::Pair as VanillaPair; +use crate::in_memory::{RowWrapper, StorableRow}; use crate::persistence::OperationType; use crate::prelude::OperationId; use crate::util::OffsetEqLink; use crate::{ - ArcticIndex, ArcticKey, ArcticMultiIndex, CongeeIndex, CongeeKey, IndexMultiMap, PersistentArtIndex, - PersistentWtiIndex, UniqueIndex, UpstreamIndexMap, + ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticValue, CongeeIndex, CongeeKey, IndexMultiMap, PersistentArtIndex, + PersistentWtiIndex, UniqueIndex, UpstreamIndexMap, WorkTable, }; use crate::{IndexMap, impl_memstat_zero}; @@ -31,6 +32,44 @@ pub trait MemStat { fn used_size(&self) -> usize; } +impl< + Row, + PrimaryKey, + AvailableTypes, + AvailableIndexes, + SecondaryIndexes, + LockType, + PkGen, + const DATA_LENGTH: usize, + PkMap, +> MemStat + for WorkTable< + Row, + PrimaryKey, + AvailableTypes, + AvailableIndexes, + SecondaryIndexes, + LockType, + PkGen, + DATA_LENGTH, + PkMap, + > +where + PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, + Row: StorableRow + Send + Clone + 'static, + ::WrappedRow: RowWrapper, + PkMap: UniqueIndex> + MemStat, + SecondaryIndexes: MemStat, +{ + fn heap_size(&self) -> usize { + self.data.allocated_bytes() + self.primary_index.pk_map.heap_size() + self.indexes.heap_size() + } + + fn used_size(&self) -> usize { + self.data.used_bytes() as usize + self.primary_index.pk_map.used_size() + self.indexes.used_size() + } +} + impl MemStat for Option { fn heap_size(&self) -> usize { self.as_ref().map_or(0, |v| v.heap_size()) @@ -121,7 +160,7 @@ where impl MemStat for ArcticIndex where K: ArcticKey, - V: Clone + Debug + Send + Sync + 'static, + V: ArcticValue, { fn heap_size(&self) -> usize { self.len() * std::mem::size_of::<(K, V)>() diff --git a/src/persistence/mod.rs b/src/persistence/mod.rs index acfe1cf7..6a0d000e 100644 --- a/src/persistence/mod.rs +++ b/src/persistence/mod.rs @@ -16,13 +16,20 @@ pub use operation::{ }; pub use readonly_engine::ReadOnlyPersistenceEngine; pub use space::{ - ArtPersistenceKey, IndexTableOfContents, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceCongeeIndex, SpaceData, - SpaceDataOps, SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, - SpaceSecondaryIndexOps, TocEntryOversizedError, map_index_pages_to_toc_and_general, + ArtPersistenceKey, IndexTableOfContents, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, + SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, + SpaceLogicalIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, }; pub use task::{PersistenceMonitor, PersistenceTask}; +/// Result of retiring one Arc-owned persisted table generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct UnloadReport { + /// Memory attributed to the generation immediately before it was dropped. + pub released_bytes: usize, +} + mod engine; mod error; pub mod operation; diff --git a/src/persistence/space/art_index.rs b/src/persistence/space/art_index.rs index 15bfd276..08c29c39 100644 --- a/src/persistence/space/art_index.rs +++ b/src/persistence/space/art_index.rs @@ -40,6 +40,8 @@ enum Backend { Congee = 2, /// Non-unique Arctic: one record per `(key, link)` pair. ArcticMulti = 3, + /// Unique Arctic with a flat key/link checkpoint for variable-width keys. + ArcticVariable = 4, } impl Backend { @@ -48,6 +50,7 @@ impl Backend { 1 => Ok(Self::Arctic), 2 => Ok(Self::Congee), 3 => Ok(Self::ArcticMulti), + 4 => Ok(Self::ArcticVariable), _ => bail!("unknown ART backend tag {byte}"), } } @@ -58,7 +61,8 @@ impl Backend { /// Generated single-column primary-key newtypes delegate this contract to /// their supported unsigned integer field. pub trait ArtPersistenceKey: Clone + Debug + Eq + Hash + Ord + Send + Sync + 'static { - /// Number of key bytes written to the WAL. + /// Number of key bytes written to the WAL, or zero for a variable-width + /// key whose records carry a `u32` byte length. const WIDTH: u8; /// Appends exactly [`Self::WIDTH`] bytes in big-endian order. @@ -129,6 +133,18 @@ macro_rules! impl_art_persistence_key_signed { impl_art_persistence_key_signed!(i8 => u8, i16 => u16, i32 => u32, i64 => u64, i128 => u128, isize => usize); +impl ArtPersistenceKey for String { + const WIDTH: u8 = 0; + + fn encode_art_key(&self, output: &mut Vec) { + output.extend_from_slice(self.as_bytes()) + } + + fn decode_art_key(bytes: &[u8]) -> eyre::Result { + String::from_utf8(bytes.to_vec()).map_err(|error| eyre!("invalid UTF-8 ART key: {error}")) + } +} + #[derive(Clone, Debug, Eq, PartialEq)] enum WalOp { /// Unique files: associate the key with this link. Multi files: add one @@ -260,8 +276,12 @@ impl ArtFile { bail!("ART WAL frame at byte {position} has an invalid magic"); } let payload_len = u32::from_le_bytes(bytes[position + 4..position + 8].try_into().unwrap()) as usize; - let expected_len = 9usize + K::WIDTH as usize + 12; - if payload_len != expected_len { + let valid_payload_len = if K::WIDTH == 0 { + payload_len >= 9 + 4 + 12 + } else { + payload_len == 9 + K::WIDTH as usize + 12 + }; + if !valid_payload_len { bail!("ART WAL frame at byte {position} has invalid payload length {payload_len}"); } let payload_crc = u32::from_le_bytes(bytes[position + 8..position + 12].try_into().unwrap()); @@ -356,14 +376,22 @@ impl ArtFile { } fn encode_wal_record(record: &WalRecord) -> Vec { - let mut bytes = Vec::with_capacity(9 + K::WIDTH as usize + 12); + let mut key = Vec::new(); + record.key.encode_art_key(&mut key); + let variable_prefix = usize::from(K::WIDTH == 0) * std::mem::size_of::(); + let mut bytes = Vec::with_capacity(9 + variable_prefix + key.len() + 12); bytes.extend_from_slice(&record.event_id.to_le_bytes()); match record.op { WalOp::Set(_) => bytes.push(1), WalOp::Remove => bytes.push(2), WalOp::RemovePair(_) => bytes.push(3), } - record.key.encode_art_key(&mut bytes); + if K::WIDTH == 0 { + bytes.extend_from_slice(&(key.len() as u32).to_le_bytes()); + } else { + debug_assert_eq!(key.len(), K::WIDTH as usize); + } + bytes.extend_from_slice(&key); let link = match record.op { WalOp::Set(link) | WalOp::RemovePair(link) => link, WalOp::Remove => Link::default(), @@ -376,14 +404,26 @@ fn encode_wal_record(record: &WalRecord) -> Vec { } fn decode_wal_record(bytes: &[u8]) -> eyre::Result> { - let expected_len = 9 + K::WIDTH as usize + 12; - if bytes.len() != expected_len { + let fixed_prefix = 9usize; + let link_len = 12usize; + let (key_start, key_len) = if K::WIDTH == 0 { + if bytes.len() < fixed_prefix + 4 + link_len { + bail!("invalid variable-key ART WAL payload length {}", bytes.len()); + } + let key_len = u32::from_le_bytes(bytes[9..13].try_into().unwrap()) as usize; + (13usize, key_len) + } else { + (9usize, K::WIDTH as usize) + }; + let key_end = key_start + .checked_add(key_len) + .ok_or_else(|| eyre!("ART WAL key length overflow"))?; + if key_end.checked_add(link_len) != Some(bytes.len()) { bail!("invalid ART WAL payload length {}", bytes.len()); } let event_id = u64::from_le_bytes(bytes[..8].try_into().unwrap()); let operation = bytes[8]; - let key_end = 9 + K::WIDTH as usize; - let key = K::decode_art_key(&bytes[9..key_end])?; + let key = K::decode_art_key(&bytes[key_start..key_end])?; let page_id = u32::from_le_bytes(bytes[key_end..key_end + 4].try_into().unwrap()); let offset = u32::from_le_bytes(bytes[key_end + 4..key_end + 8].try_into().unwrap()); let length = u32::from_le_bytes(bytes[key_end + 8..key_end + 12].try_into().unwrap()); @@ -503,7 +543,14 @@ fn encode_multi_pairs(pairs: impl Iterator(bytes: &[u8]) -> eyre::Result { + file: ArtFile, +} + +impl SpaceArcticStringIndex +where + K: ArtPersistenceKey + ArcticKey, +{ + async fn new(path: PathBuf, table_version: u32) -> eyre::Result { + Ok(Self { + file: ArtFile::open( + path, + Backend::ArcticVariable, + table_version, + encode_multi_pairs(std::iter::empty::<(K, Link)>()), + ) + .await?, + }) + } + + pub async fn load_index( + path: impl AsRef, + table_version: u32, + ) -> eyre::Result>> { + let image = ArtFile::::read_image(path.as_ref(), Backend::ArcticVariable, table_version).await?; + let index = PersistentArcticIndex::>::default(); + for (key, link) in decode_multi_pairs(&image.snapshot)? { + if index.insert_value_checked(key, OffsetEqLink(link)).is_none() { + bail!("variable-key Arctic checkpoint contains a duplicate key"); + } + } + apply_wal(&index, &image.wal, OffsetEqLink)?; + Ok(index) + } + + pub async fn write_checkpoint( + path: impl AsRef, + table_version: u32, + index: &PersistentArcticIndex>, + ) -> eyre::Result<()> { + let snapshot = encode_multi_pairs(index.iter_values().map(|(key, link)| (key, link.0))); + ArtFile::::write_file_atomically(path.as_ref(), Backend::ArcticVariable, table_version, &snapshot).await + } + + async fn compact(&mut self) -> eyre::Result<()> { + let image = ArtFile::::read_image(&self.file.path, Backend::ArcticVariable, self.file.table_version).await?; + let index = ArcticIndex::::default(); + for (key, link) in decode_multi_pairs(&image.snapshot)? { + if index.insert_value_checked(key, link).is_none() { + bail!("variable-key Arctic checkpoint contains a duplicate key"); + } + } + apply_wal(&index, &image.wal, |link| link)?; + let snapshot = encode_multi_pairs(index.iter_values()); + self.file.rewrite(&snapshot).await + } +} + +impl SpaceIndexOps for SpaceArcticStringIndex +where + K: ArtPersistenceKey + ArcticKey, +{ + async fn primary_from_table_files_path + Send>(path: S, version: u32) -> eyre::Result { + Self::new( + PathBuf::from(format!("{}/primary{}", path.as_ref(), WT_INDEX_EXTENSION)), + version, + ) + .await + } + + async fn secondary_from_table_files_path + Send, S2: AsRef + Send>( + path: S1, + name: S2, + version: u32, + ) -> eyre::Result { + Self::new( + PathBuf::from(format!("{}/{}{}", path.as_ref(), name.as_ref(), WT_INDEX_EXTENSION)), + version, + ) + .await + } + + async fn bootstrap(_: &mut File, _: String, _: u32) -> eyre::Result<()> { + Ok(()) + } + + async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { + self.file.append(&[logical_record(event)?]).await?; + if self.file.should_compact() { + self.compact().await?; + } + Ok(()) + } + + async fn process_change_event_batch(&mut self, events: BatchChangeEvent) -> eyre::Result<()> { + let records = events + .into_iter() + .map(logical_record) + .collect::>>()?; + self.file.append(&records).await?; + if self.file.should_compact() { + self.compact().await?; + } + Ok(()) + } +} + /// Disk-side Congee checkpoint and WAL state. #[derive(Debug)] pub struct SpaceCongeeIndex { @@ -874,6 +1040,10 @@ impl<'a> Decoder<'a> { Ok(u16::from_le_bytes(self.take(2)?.try_into().unwrap())) } + fn u32(&mut self) -> eyre::Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().unwrap())) + } + fn u64(&mut self) -> eyre::Result { Ok(u64::from_le_bytes(self.take(8)?.try_into().unwrap())) } @@ -1104,7 +1274,7 @@ mod tests { } } - fn set_event(id: u64, key: u64, value: Link) -> ChangeEvent> { + fn set_event(id: u64, key: K, value: Link) -> ChangeEvent> { let pair = Pair { key, value }; ChangeEvent::InsertAt { event_id: id.into(), @@ -1340,6 +1510,35 @@ mod tests { tokio::fs::remove_file(path).await.unwrap(); } + #[tokio::test] + async fn variable_string_keys_survive_wal_and_checkpoint() { + let path = std::env::temp_dir().join(format!("worktable-art-string-{}.wt.idx", uuid::Uuid::new_v4())); + let mut space = SpaceArcticStringIndex::::new(path.clone(), 3) + .await + .unwrap(); + for (event_id, key) in ["", "nul\0inside", "é", "🦀"].into_iter().enumerate() { + space + .process_change_event(set_event(event_id as u64, key.to_owned(), link(event_id as u32 + 1))) + .await + .unwrap(); + } + drop(space); + + let index = SpaceArcticStringIndex::::load_index::<4096>(&path, 3) + .await + .unwrap(); + assert_eq!(index.get_value(&"nul\0inside".to_owned()).unwrap().0, link(2)); + SpaceArcticStringIndex::::write_checkpoint::<4096>(&path, 3, &index) + .await + .unwrap(); + let reloaded = SpaceArcticStringIndex::::load_index::<4096>(&path, 3) + .await + .unwrap(); + assert_eq!(reloaded.len(), 4); + assert_eq!(reloaded.get_value(&"🦀".to_owned()).unwrap().0, link(4)); + tokio::fs::remove_file(path).await.unwrap(); + } + #[test] fn temporary_path_appends_to_the_full_file_name() { assert_eq!( diff --git a/src/persistence/space/mod.rs b/src/persistence/space/mod.rs index 28a205b5..34b22b39 100644 --- a/src/persistence/space/mod.rs +++ b/src/persistence/space/mod.rs @@ -13,7 +13,9 @@ use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; use tokio::fs::{File, OpenOptions}; -pub use art_index::{ArtPersistenceKey, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceCongeeIndex}; +pub use art_index::{ + ArtPersistenceKey, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, +}; pub use data::SpaceData; pub use index::{ IndexTableOfContents, SpaceIndex, SpaceIndexUnsized, TocEntryOversizedError, map_index_pages_to_toc_and_general, diff --git a/src/table/mod.rs b/src/table/mod.rs index 80439b16..de1e6a64 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -9,7 +9,7 @@ use crate::primary_key::{PrimaryKeyGenerator, TablePrimaryKey}; use crate::util::OffsetEqLink; use crate::{ AvailableIndex, IndexError, IndexMap, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, - TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, UniqueIndex, convert_change_events, in_memory, + TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, convert_change_events, in_memory, }; use data_bucket::INNER_PAGE_SIZE; use derive_more::{Display, Error, From}; @@ -167,30 +167,12 @@ where format!("row at {:?} does not match primary key {primary_key:?}", offset_link.0), )); } - - let reverse_key = self.primary_index.reverse_pk_map.get_value(&offset_link); - if reverse_key.as_ref() != Some(&primary_key) { - return Err(PersistenceLoadError::corrupt( - path, - format!("reverse primary index does not match link {:?}", offset_link.0), - )); - } - } - - if self.primary_index.reverse_pk_map.len() != links.len() { - return Err(PersistenceLoadError::corrupt( - path, - "forward and reverse primary indexes contain different numbers of entries", - )); - } - - for (offset_link, primary_key) in self.primary_index.reverse_pk_map.iter_values() { - if self.primary_index.pk_map.get_value(&primary_key) != Some(offset_link) { - return Err(PersistenceLoadError::corrupt( + self.data.register_cell(offset_link.0).map_err(|error| { + PersistenceLoadError::corrupt( path, - format!("forward primary index does not match link {:?}", offset_link.0), - )); - } + format!("primary key {primary_key:?} has an invalid cell: {error}"), + ) + })?; } Ok(()) diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index 5d16f214..77212485 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -264,6 +264,23 @@ where let pages_processed = per_page_info.len(); + // The primary index is the authoritative position-to-key mapping. + // Group one weakly-consistent snapshot by page for this sweep instead + // of retaining a reverse entry beside every row for the table's whole + // lifetime. Each candidate is revalidated under its row mutation lock + // before moving, and the per-page live counter below is the final + // defense against reclaiming a row missed by the snapshot. + let mut candidates_by_page: Vec> = (0..=self.data_pages.allocated_pages()) + .map(|_| VecDeque::new()) + .collect(); + for (pk, offset_link) in self.primary_index.pk_map.iter_values() { + let link = offset_link.0; + let page_index = usize::from(link.page_id); + if let Some(candidates) = candidates_by_page.get_mut(page_index) { + candidates.push_back((link, pk)); + } + } + let info_iter = per_page_info.into_iter(); // The exclusion is released every `batch_pages` sources. It is held @@ -314,7 +331,11 @@ where page_from, page_to, "vacuum destination must differ from the source being reclaimed" ); - let move_result = match self.move_data_from(page_from, page_to).await { + let page_index = usize::from(page_from); + let candidates = candidates_by_page + .get_mut(page_index) + .expect("an allocated page id should have a candidate bucket"); + let move_result = match self.move_data_from_candidates(page_from, page_to, candidates).await { Ok(result) => result, Err(error) => { // Register every staged page before propagating, or @@ -423,38 +444,35 @@ where Ok(()) } + #[cfg(test)] async fn move_data_from(&self, from: PageId, to: PageId) -> eyre::Result<(bool, bool)> { + let mut candidates = self + .primary_index + .pk_map + .iter_values() + .filter_map(|(pk, offset_link)| { + let link = offset_link.0; + (link.page_id == from).then_some((link, pk)) + }) + .collect(); + self.move_data_from_candidates(from, to, &mut candidates).await + } + + async fn move_data_from_candidates( + &self, + from: PageId, + to: PageId, + candidates: &mut VecDeque<(Link, PrimaryKey)>, + ) -> eyre::Result<(bool, bool)> { let to_page = self.data_pages.get_page(to).expect("should exist as link exists"); let to_free_space = to_page.free_space(); - let page_start = OffsetEqLink::<_>(Link { - page_id: from, - offset: 0, - length: 0, - }); - - let page_end = OffsetEqLink::<_>(Link { - page_id: from.next(), - offset: 0, - length: 0, - }); - - let mut range = self.primary_index.reverse_pk_map.range(page_start..page_end); let mut sum_links_len = 0; let mut links = vec![]; let mut from_page_will_be_moved = false; let mut to_page_will_be_filled = false; - loop { - let Some((next, pk)) = range.next() else { - from_page_will_be_moved = true; - break; - }; - - if next.page_id != from { - continue; - } - + while let Some((next, _)) = candidates.front() { if sum_links_len + next.length > to_free_space as u32 { // This candidate stays on the source page, so the page must // never be reported fully moved in this pass — even when the @@ -465,14 +483,15 @@ where break; } sum_links_len += next.length; - links.push((next, pk)); + links.push(candidates.pop_front().expect("front candidate exists")); + } + if !to_page_will_be_filled { + from_page_will_be_moved = true; } - - drop(range); let mut any_move_failed = false; for (from_link, pk) in links { - if self.move_candidate_if_current(from_link.0, pk, to).await? == CandidateMove::Failed { + if self.move_candidate_if_current(from_link, pk, to).await? == CandidateMove::Failed { any_move_failed = true; } } @@ -501,12 +520,7 @@ where // needed a delete to free the space, an insert to take it, and // vacuum to reclaim underneath, which is why it only ever appeared // with all three running. - let occupied = self - .primary_index - .reverse_pk_map - .range(page_start..page_end) - .any(|(link, _)| link.0.page_id == from); - if occupied { + if self.data_pages.page_has_cells(from)? { from_page_will_be_moved = false; } } @@ -562,7 +576,7 @@ where } }; self.update_index_after_move(pk, from_link, new_link, raw_data)?; - self.data_pages.retire_published_link(from_link); + self.data_pages.remove_moved_cell(from_link)?; Ok(CandidateMove::Moved) } diff --git a/src/util/epoch.rs b/src/util/epoch.rs index b0b31967..03456fb9 100644 --- a/src/util/epoch.rs +++ b/src/util/epoch.rs @@ -21,13 +21,13 @@ //! //! Cheaper than `crossbeam` and flat, but it reclaims only when no reader at //! all is live. `select` holds a read guard, so under continuous read traffic -//! that instant never arrives and retired links, pages and publications queue +//! that instant never arrives and retired links and pages queue //! forever. That is the property `reclamation_progresses_under_continuous_reader_overlap` //! in `in_memory::pages` asserts, and it is the reason the global reader //! counter was replaced in the first place. //! -//! `arctic` reclaims through `seize` and is right to: a trie with short reads -//! reaches quiescence constantly. This does not. +//! WorkTable's Arctic adapter now also selects `ps-reclaim`; Arctic supports +//! other SMRs for its general users, but WorkTable keeps one progress model. //! //! # crossbeam is still in the tree //! diff --git a/tests/generation_swap_requirement.rs b/tests/generation_swap_requirement.rs new file mode 100644 index 00000000..e1d66c70 --- /dev/null +++ b/tests/generation_swap_requirement.rs @@ -0,0 +1,167 @@ +//! Serving-process requirements for swapping one generation for the next. +//! +//! # Where this comes from +//! +//! `moe-pgo` serves model clusters and swaps them without restarting: attach +//! generation N+1, let generation N's readers drain, release N's memory, keep +//! answering throughout. It cannot do that with WorkTable, so it does it with a +//! private crate that hand-writes `data_bucket` pages, a `GeneralHeader` at a +//! time, with its own manifest and its own lease registry. That crate is 1,548 +//! lines reimplementing a layer this one already owns, and the reason it exists +//! is the gap below rather than a preference. +//! +//! # What is missing, in order of how much it matters +//! +//! 1. **A lease-aware unload.** `close(self)` takes ownership. A server holds a +//! generation behind an `Arc`, so the unload receiver owns the retiring Arc, +//! waits for readers to drain under a caller-supplied barrier and timeout, +//! then unwraps and drops the allocation while reporting its measured size: +//! +//! ```ignore +//! async fn unload_gracefully(self: Arc, timeout: Duration, quiesce: F) +//! -> eyre::Result; +//! ``` +//! +//! `wait_for_ops` is not this. It drains queued *writes*; a swap has to +//! drain *readers*, and there is no lease count to observe. +//! +//! 2. **`MemStat` on a generated persisted table.** `heap_size()` supplies the +//! pre-drop measurement returned by unload. +//! +//! Two things that would help and are not blocking: a registry keyed by content +//! digest so generations are addressed by id rather than by path, and digest +//! verification on attach so a file whose bytes do not match its expected id is +//! refused. +//! +//! # What already works, so nobody spends a day on it +//! +//! Attaching two handles to one on-disk generation at the same time. That was +//! the first thing tested here on the assumption it was the gap, and it passed. +//! `attaching_a_second_handle_already_works` keeps that result so it stays true. + +use std::sync::Arc; +use std::time::Duration; + +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: GenerationSwap, + version: 1, + persist: true, + columns: { + id: u64 primary_key autoincrement, + blob: String, + }, +); + +const DIR: &str = "tests/data/generation_swap/persisted"; + +/// A generation big enough that releasing it is worth reporting. +const ROWS: u64 = 2_000; + +/// What a caller would pass as the drain barrier. +const DRAIN_TIMEOUT: Duration = Duration::from_secs(5); + +async fn attach(dir: &str) -> GenerationSwapWorkTable { + let engine = GenerationSwapPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + GenerationSwapWorkTable::name_snake_case(), + GenerationSwapWorkTable::version(), + )) + .await + .expect("an engine"); + GenerationSwapWorkTable::load(engine).await.expect("a generation") +} + +async fn fill(table: &GenerationSwapWorkTable) { + for n in 0..ROWS { + table + .insert(GenerationSwapRow { + id: table.get_next_pk().into(), + blob: format!("row {n} with enough text to occupy real pages"), + }) + .await + .expect("a row"); + } + table.wait_for_ops().await.expect("the queue drains"); +} + +#[tokio::test] +async fn a_retired_generation_releases_its_memory() { + let _ = std::fs::remove_dir_all(DIR); + std::fs::create_dir_all(DIR).expect("a directory"); + + let generation = Arc::new(attach(DIR).await); + fill(&generation).await; + + // A reader in flight, exactly as during a swap. + let held = generation.heap_size(); + assert!(held > 0, "a filled generation reports memory"); + + let reader = Arc::clone(&generation); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let reader_task = tokio::spawn(async move { + assert_eq!( + reader.select_all().execute().expect("a read").len(), + ROWS as usize, + "the outgoing generation is still answering" + ); + let _ = release_rx.await; + drop(reader); + }); + + let report = generation + .unload_gracefully(DRAIN_TIMEOUT, move || async move { + let _ = release_tx.send(()); + reader_task.await.expect("reader drains"); + }) + .await + .expect("the generation retires"); + assert_eq!(report.released_bytes, held); + assert!(report.released_bytes > 0, "the memory came back"); + + let _ = std::fs::remove_dir_all(DIR); +} + +#[tokio::test] +async fn a_generation_can_report_what_it_holds() { + let _ = std::fs::remove_dir_all(DIR); + std::fs::create_dir_all(DIR).expect("a directory"); + + let generation = attach(DIR).await; + fill(&generation).await; + + let held = generation.heap_size(); + assert!(held > 0, "a filled generation holds memory: {held}"); + generation.close().await.expect("generation closes"); + let _ = std::fs::remove_dir_all(DIR); +} + +/// **Already works.** Kept so nobody spends a day building it: two handles on +/// one on-disk generation coexist, and the outgoing one keeps answering. +#[tokio::test] +async fn attaching_a_second_handle_already_works() { + let dir = "tests/data/generation_swap/second_handle"; + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir).expect("a directory"); + + let live = attach(dir).await; + live.insert(GenerationSwapRow { + id: live.get_next_pk().into(), + blob: "N".to_owned(), + }) + .await + .expect("a row"); + live.wait_for_ops().await.expect("the queue drains"); + + let next = attach(dir).await; + assert_eq!(next.select_all().execute().expect("a read").len(), 1); + assert_eq!( + live.select_all().execute().expect("a read").len(), + 1, + "the outgoing generation still answers while the next is attached" + ); + + let _ = std::fs::remove_dir_all(dir); +} diff --git a/tests/worktable/upsert.rs b/tests/worktable/upsert.rs index 6011fe9d..c00f0c83 100644 --- a/tests/worktable/upsert.rs +++ b/tests/worktable/upsert.rs @@ -100,8 +100,8 @@ async fn raw_insert_delete_churn_never_panics_or_stalls() { } // Force a deterministic final state, then audit every layer used to reach - // the row. A liveness-only test would miss a stale reverse entry, a ghost - // publication, or a primary link that points at unrelated data. + // the row. A liveness-only test would miss a ghosted cell or a primary + // link that points at unrelated data. let expected = UpsertChurnRow { id: KEY, val: 424_242 }; table.upsert(expected.clone()).await.unwrap(); @@ -113,17 +113,6 @@ async fn raw_insert_delete_churn_never_panics_or_stalls() { .get_value(&pk) .expect("final row must have one primary-index entry"); assert_eq!(table.0.primary_index.pk_map.len(), 1); - assert_eq!(table.0.primary_index.reverse_pk_map.len(), 1); - assert_eq!( - table - .0 - .primary_index - .reverse_pk_map - .get(&link) - .map(|entry| entry.get().value.clone()), - Some(pk), - "reverse index must point back to the final primary key" - ); assert_eq!( table.0.data.select_non_ghosted(link.0), Ok(expected.clone()), @@ -132,7 +121,7 @@ async fn raw_insert_delete_churn_never_panics_or_stalls() { assert_eq!(table.select(KEY), Some(expected)); } -/// Pins the exact publication schedule that used to let delete unwrap a +/// Pins the exact row schedule that used to let delete unwrap a /// ghosted row: data and primary-index reachability exist, but insert has not /// yet cleared the lifecycle bit. Delete must linearize before publication and /// leave the staged insert intact. diff --git a/tests/worktable/vacuum_invariants.rs b/tests/worktable/vacuum_invariants.rs index c80bd584..0c4c3a33 100644 --- a/tests/worktable/vacuum_invariants.rs +++ b/tests/worktable/vacuum_invariants.rs @@ -82,42 +82,22 @@ macro_rules! vacuum_invariant_suite { /// would hide exactly the failure being looked for, because it /// revalidates and retries. fn assert_indexes_resolve_to_their_own_rows(table: &VacInvWorkTable, phase: &str) { - // Enumerated through the reverse index, which is always the - // general-purpose map whatever the primary backend is. Arctic - // and congee expose no iterator, which is the three-backend - // difference in miniature: a check written against one of them - // does not compile against the others. let mut seen_links: HashMap = HashMap::new(); - for (link, pk) in table.0.primary_index.reverse_pk_map.iter() { - let link: Link = link.into(); + for (pk, offset_link) in table.0.primary_index.pk_map.iter_values() { + let link: Link = offset_link.into(); let key: u64 = pk.clone().into(); let row = table .0 .data .select_non_ghosted(link) - .unwrap_or_else(|e| panic!("{phase}: reverse entry {key} points at unreadable storage: {e:?}")); + .unwrap_or_else(|e| panic!("{phase}: primary entry {key} points at unreadable storage: {e:?}")); assert_eq!( row.id, key, - "{phase}: reverse entry {key} resolves to a row whose id is {}", + "{phase}: primary entry {key} resolves to a row whose id is {}", row.id ); - // Forward and reverse must agree, and must agree on the - // same storage. Vacuum plans from the reverse map, so a - // disagreement is a page drained against a stale picture. - let forward: Option = table - .0 - .primary_index - .pk_map - .get_value(&pk) - .map(Into::into); - assert_eq!( - forward, - Some(link), - "{phase}: key {key} is at {link:?} in the reverse index and {forward:?} in the forward one" - ); - let packed = (u32::from(link.page_id) as u64) << 32 | link.offset as u64; if let Some(other) = seen_links.insert(packed, key) { panic!("{phase}: keys {other} and {key} name the same storage"); From 47e9c1cbf8b4271ce0a7de8d4aecc368e72d25c3 Mon Sep 17 00:00:00 2001 From: meh Date: Sat, 5 Sep 2026 06:54:43 +0700 Subject: [PATCH 3/5] Keep local dependency overrides out of published manifests --- Cargo.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1108852a..206c08b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,16 +39,16 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"] [dependencies] arc-swap = "1" async-trait = "0.1" -arctic = { package = "arctic-wt", version = "^0.1", path = "../arctic-wt", default-features = false, features = ["smr-ps-reclaim"] } -congee = { package = "congee-wt", version = "^0.4", path = "../congee-wt" } +arctic = { package = "arctic-wt", version = "^0.1", default-features = false, features = ["smr-ps-reclaim"] } +congee = { package = "congee-wt", version = "^0.4" } convert_case = "0.6" crc32fast = "1" -data_bucket = { version = "^0.5", path = "../DataBucket" } +data_bucket = { version = "^0.5" } derive_more = { version = "2", features = ["from", "error", "display", "debug", "into"] } eyre = "0.6" fastrand = "2" futures = "0.3" -indexset = { package = "WorkTablesIndex", version = "^0.0", path = "../WorkTablesIndex", default-features = false, features = ["concurrent", "cdc", "multimap"] } +indexset = { package = "WorkTablesIndex", version = "^0.0", default-features = false, features = ["concurrent", "cdc", "multimap"] } vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"] } # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "^0.12", features = ["concurrent", "cdc", "multimap"] } @@ -61,7 +61,7 @@ prettytable-rs = "0.10" psc-nanoid = { version = "3", features = ["rkyv", "packed"] } rkyv = { version = "0.8", features = ["uuid-1"] } reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] } -ps-reclaim = { version = "^0.1", path = "../ps-reclaim" } +ps-reclaim = { version = "^0.1" } rustc-hash = "2" rusty-s3 = { version = "0.10", optional = true } smart-default = "0.7" From c2b711830c5994ac41b14cd5cdf367459cfb95f9 Mon Sep 17 00:00:00 2001 From: meh Date: Sat, 5 Sep 2026 08:59:42 +0700 Subject: [PATCH 4/5] Finish beta18 cell locking, Arctic persistence, and unload --- CHANGELOG.md | 13 +- Cargo.toml | 8 +- README.md | 2 +- codegen/src/generators/in_memory/wrapper.rs | 28 +- codegen/src/generators/persist/index/cdc.rs | 2 +- codegen/src/generators/persist/wrapper.rs | 28 +- codegen/src/generators/read_only/wrapper.rs | 28 +- codegen/src/persist_index/generator.rs | 136 ++++++-- codegen/src/persist_index/space/index.rs | 20 +- codegen/src/persist_table/generator/space.rs | 4 +- .../persist_table/generator/space_file/mod.rs | 57 ++- .../generator/space_file/worktable_impls.rs | 75 +++- codegen/src/persist_table/parser.rs | 1 + codegen/src/worktable/mod.rs | 25 +- codegen/src/worktable_version/mod.rs | 4 +- docs/index-backend-dsl-proposal.md | 63 ++-- dsl/src/model/column.rs | 33 +- dsl/src/model/index.rs | 10 +- dsl/src/parser/index.rs | 4 +- dsl/src/validate.rs | 33 +- dsl/tests/check.rs | 36 +- dsl/tests/schema.rs | 2 +- src/in_memory/data.rs | 324 +++++++++-------- src/in_memory/mod.rs | 2 +- src/in_memory/pages.rs | 106 ++++-- src/in_memory/row.rs | 43 +-- src/index/arctic.rs | 119 ++++++- src/index/arctic_multi.rs | 14 + src/index/congee.rs | 4 + src/index/mod.rs | 6 +- src/index/primary_index.rs | 6 +- src/index/table_index/mod.rs | 2 +- src/index/table_index/util.rs | 62 ++++ src/lib.rs | 20 +- src/mem_stat/mod.rs | 23 +- src/persistence/mod.rs | 67 +++- src/persistence/operation/batch.rs | 8 +- src/persistence/space/art_index.rs | 19 +- src/persistence/space/index/mod.rs | 24 ++ src/persistence/space/index/unsized_.rs | 28 +- src/persistence/space/logical_index.rs | 326 ++++++++++++++++++ src/persistence/space/mod.rs | 4 +- src/persistence/task.rs | 6 +- src/table/mod.rs | 20 ++ src/table/vacuum/manager.rs | 19 +- src/table/vacuum/mod.rs | 15 + src/table/vacuum/vacuum.rs | 41 ++- src/util/epoch.rs | 13 +- tests/generation_swap_requirement.rs | 39 ++- tests/persistence/sync/option.rs | 2 +- tests/persistence/sync/uuid_.rs | 4 +- tests/persistence/tuple_primary_key.rs | 4 +- tests/worktable/borrowed_primary_key.rs | 4 +- tests/worktable/custom_pk.rs | 2 +- tests/worktable/float.rs | 8 +- tests/worktable/key_widths.rs | 9 +- tests/worktable/nid.rs | 2 +- tests/worktable/option.rs | 2 +- tests/worktable/tuple_primary_key.rs | 4 +- tests/worktable/uuid.rs | 2 +- 60 files changed, 1489 insertions(+), 526 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1672a46b..cce68b0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,15 +16,16 @@ Change Log ### Changed -- WorkTable row/page reclamation uses the local `ps-reclaim` domain for every - index backend; the WorkTable Arctic adapter also selects Arctic's - `ps-reclaim` SMR exclusively. +- WorkTable row/page reclamation uses the local `ps-reclaim` domain regardless + of the selected index backend. Arctic and Congee also select their local + `ps-reclaim` SMR implementations; WorkTablesIndex retains its structural + skip-list reclamation internally. - Readers now synchronize on the exact physical cell. Unrelated rows cannot block because of a hashed lock collision. - Vacuum discovers move candidates from a transient primary-index snapshot and keeps only one live-cell counter per page, removing the previous four-byte per-row directory. -- Vacuum waits for two quiet observations after mutation activity and yields +- Vacuum waits for three quiet observations after mutation activity and yields throughout a bulk mutation instead of competing with foreground work. - The archived wrapper retains the beta.17 inner-row position so legacy stores without bundled schema metadata remain readable. @@ -33,8 +34,8 @@ Change Log - Torn reads and premature physical-link reuse during concurrent update, delete, and vacuum activity. -- In-place replacement now preserves the embedded cell lock byte while copying - the rest of the archived row. +- In-place replacement synchronizes through the runtime side-table cell lock; + the beta.17 archived row bytes remain unchanged. - Whole-map Arctic destruction uses an unordered physical drain instead of repeatedly searching for the next logical key. - Persisted primary/secondary index reconstruction and validation failures that diff --git a/Cargo.toml b/Cargo.toml index 206c08b0..8f165031 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,11 +15,7 @@ keywords = ["database", "embedded", "in-memory", "index", "storage"] categories = ["database-implementations", "data-structures", "caching"] [features] -default = ["wti-predictable-search", "arctic-ps-reclaim"] -# Compatibility name: WorkTable's Arctic adapter always uses ps-reclaim. Keep -# this feature so existing manifests do not break, but do not allow a -# `--no-default-features` build to silently select Arctic's no-op SMR. -arctic-ps-reclaim = [] +default = ["wti-predictable-search"] perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"] s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktable_codegen/s3-support"] # Moves unique WorkTablesIndex structural CDC work out of the table mutation @@ -37,6 +33,8 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +# Read-mostly snapshots own page Arcs while the fixed directory supplies a +# pointer-only fast path. Publication is append-only and asserted at each swap. arc-swap = "1" async-trait = "0.1" arctic = { package = "arctic-wt", version = "^0.1", default-features = false, features = ["smr-ps-reclaim"] } diff --git a/README.md b/README.md index 3f7bb89e..e13a8cde 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ cargo add worktable@1.0.0-beta.5 | **Concurrency** | Lock-free concurrent indexes with change-data-capture, plus a row-level `LockMap` for ordered access. | | **Optional persistence** | `PersistedWorkTable` writes to local disk; the `s3-support` feature syncs that to S3. Both opt-in, so a purely in-memory table pays for neither. | | **Schema migration** | `worktable_version!` and `migration_engine!` version a table's schema and generate migrations between versions. See [docs/migration.md](docs/migration.md). | -| **Memory accounting** | `MemStat` reports actual memory held. | +| **Memory accounting** | `MemStat` estimates live heap; resident benchmarks measure allocator and SMR overhead. | ## Persistence diff --git a/codegen/src/generators/in_memory/wrapper.rs b/codegen/src/generators/in_memory/wrapper.rs index 1c40617a..48c31da7 100644 --- a/codegen/src/generators/in_memory/wrapper.rs +++ b/codegen/src/generators/in_memory/wrapper.rs @@ -25,12 +25,12 @@ impl InMemoryGenerator { quote! { #[derive(rkyv::Archive, Debug, rkyv::Deserialize, rkyv::Serialize)] - #[rkyv(attr(repr(C)))] #[repr(C)] pub struct #wrapper_ident { inner: #row_ident, - publication_flags: u8, - cell_state: CellState, + is_ghosted: bool, + is_deleted: bool, + is_in_vacuum_process: bool, } } } @@ -48,22 +48,23 @@ impl InMemoryGenerator { } fn is_ghosted(&self) -> bool { - self.publication_flags & 1 != 0 + self.is_ghosted } fn is_vacuumed(&self) -> bool { - self.publication_flags & 4 != 0 + self.is_in_vacuum_process } fn is_deleted(&self) -> bool { - self.publication_flags & 2 != 0 + self.is_deleted } fn from_inner(inner: #row_ident) -> Self { Self { inner, - publication_flags: 1, - cell_state: CellState, + is_ghosted: true, + is_deleted: false, + is_in_vacuum_process: false, } } } @@ -88,20 +89,17 @@ impl InMemoryGenerator { quote! { impl ArchivedRowWrapper for #row_ident { - unsafe fn cell_state_ptr(this: *mut Self) -> *mut std::sync::atomic::AtomicU8 { - unsafe { std::ptr::addr_of_mut!((*this).cell_state).cast() } - } fn unghost(&mut self) { - self.publication_flags &= !1; + self.is_ghosted = false; } fn set_in_vacuum_process(&mut self) { - self.publication_flags |= 4; + self.is_in_vacuum_process = true; } fn delete(&mut self) { - self.publication_flags |= 2; + self.is_deleted = true; } fn is_deleted(&self) -> bool { - self.publication_flags & 2 != 0 + self.is_deleted } } } diff --git a/codegen/src/generators/persist/index/cdc.rs b/codegen/src/generators/persist/index/cdc.rs index ccc01965..8aee8d6a 100644 --- a/codegen/src/generators/persist/index/cdc.rs +++ b/codegen/src/generators/persist/index/cdc.rs @@ -446,7 +446,7 @@ mod tests { let mut columns = parser.parse_columns().unwrap(); let mut parser = Parser::new(quote! { indexes: { - price_idx: price, + price_idx: price using worktables_index, } }); columns.indexes = parser.parse_indexes().unwrap(); diff --git a/codegen/src/generators/persist/wrapper.rs b/codegen/src/generators/persist/wrapper.rs index 222c2dd1..8308495a 100644 --- a/codegen/src/generators/persist/wrapper.rs +++ b/codegen/src/generators/persist/wrapper.rs @@ -25,12 +25,12 @@ impl PersistGenerator { quote! { #[derive(rkyv::Archive, Debug, rkyv::Deserialize, rkyv::Serialize)] - #[rkyv(attr(repr(C)))] #[repr(C)] pub struct #wrapper_ident { inner: #row_ident, - publication_flags: u8, - cell_state: CellState, + is_ghosted: bool, + is_deleted: bool, + is_in_vacuum_process: bool, } } } @@ -48,22 +48,23 @@ impl PersistGenerator { } fn is_ghosted(&self) -> bool { - self.publication_flags & 1 != 0 + self.is_ghosted } fn is_vacuumed(&self) -> bool { - self.publication_flags & 4 != 0 + self.is_in_vacuum_process } fn is_deleted(&self) -> bool { - self.publication_flags & 2 != 0 + self.is_deleted } fn from_inner(inner: #row_ident) -> Self { Self { inner, - publication_flags: 1, - cell_state: CellState, + is_ghosted: true, + is_deleted: false, + is_in_vacuum_process: false, } } } @@ -88,20 +89,17 @@ impl PersistGenerator { quote! { impl ArchivedRowWrapper for #row_ident { - unsafe fn cell_state_ptr(this: *mut Self) -> *mut std::sync::atomic::AtomicU8 { - unsafe { std::ptr::addr_of_mut!((*this).cell_state).cast() } - } fn unghost(&mut self) { - self.publication_flags &= !1; + self.is_ghosted = false; } fn set_in_vacuum_process(&mut self) { - self.publication_flags |= 4; + self.is_in_vacuum_process = true; } fn delete(&mut self) { - self.publication_flags |= 2; + self.is_deleted = true; } fn is_deleted(&self) -> bool { - self.publication_flags & 2 != 0 + self.is_deleted } } } diff --git a/codegen/src/generators/read_only/wrapper.rs b/codegen/src/generators/read_only/wrapper.rs index 236b0cff..9f69d5b4 100644 --- a/codegen/src/generators/read_only/wrapper.rs +++ b/codegen/src/generators/read_only/wrapper.rs @@ -25,12 +25,12 @@ impl ReadOnlyGenerator { quote! { #[derive(rkyv::Archive, Debug, rkyv::Deserialize, rkyv::Serialize)] - #[rkyv(attr(repr(C)))] #[repr(C)] pub struct #wrapper_ident { inner: #row_ident, - publication_flags: u8, - cell_state: CellState, + is_ghosted: bool, + is_deleted: bool, + is_in_vacuum_process: bool, } } } @@ -48,22 +48,23 @@ impl ReadOnlyGenerator { } fn is_ghosted(&self) -> bool { - self.publication_flags & 1 != 0 + self.is_ghosted } fn is_vacuumed(&self) -> bool { - self.publication_flags & 4 != 0 + self.is_in_vacuum_process } fn is_deleted(&self) -> bool { - self.publication_flags & 2 != 0 + self.is_deleted } fn from_inner(inner: #row_ident) -> Self { Self { inner, - publication_flags: 1, - cell_state: CellState, + is_ghosted: true, + is_deleted: false, + is_in_vacuum_process: false, } } } @@ -88,20 +89,17 @@ impl ReadOnlyGenerator { quote! { impl ArchivedRowWrapper for #row_ident { - unsafe fn cell_state_ptr(this: *mut Self) -> *mut std::sync::atomic::AtomicU8 { - unsafe { std::ptr::addr_of_mut!((*this).cell_state).cast() } - } fn unghost(&mut self) { - self.publication_flags &= !1; + self.is_ghosted = false; } fn set_in_vacuum_process(&mut self) { - self.publication_flags |= 4; + self.is_in_vacuum_process = true; } fn delete(&mut self) { - self.publication_flags |= 2; + self.is_deleted = true; } fn is_deleted(&self) -> bool { - self.publication_flags & 2 != 0 + self.is_deleted } } } diff --git a/codegen/src/persist_index/generator.rs b/codegen/src/persist_index/generator.rs index 8e903092..10e98749 100644 --- a/codegen/src/persist_index/generator.rs +++ b/codegen/src/persist_index/generator.rs @@ -146,7 +146,7 @@ impl Generator { let layout = index_layout(field)?; let i = field.ident.as_ref().expect("index fields should be named"); let t = self.field_types.get(i).expect("field type was collected"); - if layout.art_backend.is_some() { + if layout.art_backend == Some(ArtBackend::Congee) { let field_type = &field.ty; Ok(quote! { #i: #field_type, }) } else if is_unsized(&t.to_string()) { @@ -207,20 +207,6 @@ impl Generator { let ty = self.field_types.get(i).expect("field type was collected"); let index_name_literal = Literal::string(i.to_string().as_str()); Ok(match layout.art_backend { - Some(ArtBackend::Arctic) => quote! { - SpaceArcticIndex::<#ty, { #inner_const_name as u32 }>::write_checkpoint( - format!("{}/{}{}", path, #index_name_literal, #index_extension), - #version_const_name, - &mut self.#i, - ).await?; - }, - Some(ArtBackend::ArcticMulti) => quote! { - SpaceArcticMultiIndex::<#ty, { #inner_const_name as u32 }>::write_checkpoint( - format!("{}/{}{}", path, #index_name_literal, #index_extension), - #version_const_name, - &mut self.#i, - ).await?; - }, Some(ArtBackend::Congee) => quote! { SpaceCongeeIndex::<#ty, { #inner_const_name as u32 }>::write_checkpoint( format!("{}/{}{}", path, #index_name_literal, #index_extension), @@ -228,7 +214,7 @@ impl Generator { &mut self.#i, ).await?; }, - None => quote! { + _ => quote! { { let mut file = tokio::fs::File::create(format!("{}/{}{}", path, #index_name_literal, #index_extension)).await?; let mut info = #ident::space_info_default(); @@ -274,27 +260,40 @@ impl Generator { let i = field.ident.as_ref().expect("index fields should be named"); let ty = self.field_types.get(i).expect("field type was collected"); let literal = Literal::string(i.to_string().as_str()); + let parsed_type = if is_unsized(&ty.to_string()) { + quote! { + (Vec>>, + Vec>>) + } + } else { + quote! { + (Vec>>, + Vec>>) + } + }; + let validate_arctic_links = if matches!( + layout.art_backend, + Some(ArtBackend::Arctic | ArtBackend::ArcticMulti) + ) { + quote! { + for page in &#i.1 { + for pair in page.inner.get_node() { + validate_arctic_link(pair.value)?; + } + } + } + } else { + quote! {} + }; Ok(match layout.art_backend { - Some(ArtBackend::Arctic) => quote! { - let #i = SpaceArcticIndex::<#ty, { #inner_const_name as u32 }>::load_index( - format!("{}/{}{}", path, #literal, #index_extension), - #version_const_name, - ).await?; - }, - Some(ArtBackend::ArcticMulti) => quote! { - let #i = SpaceArcticMultiIndex::<#ty, { #inner_const_name as u32 }>::load_index( - format!("{}/{}{}", path, #literal, #index_extension), - #version_const_name, - ).await?; - }, Some(ArtBackend::Congee) => quote! { let #i = SpaceCongeeIndex::<#ty, { #inner_const_name as u32 }>::load_index( format!("{}/{}{}", path, #literal, #index_extension), #version_const_name, ).await?; }, - None => quote! { - let #i = { + _ => quote! { + let #i: #parsed_type = { let mut #i = vec![]; let mut file = tokio::fs::File::open(format!("{}/{}{}", path, #literal, #index_extension)).await?; let info = parse_page::, { #page_const_name as u32 }>(&mut file, 0).await?; @@ -312,6 +311,7 @@ impl Generator { } (toc.pages, #i) }; + #validate_arctic_links } }) }) @@ -391,13 +391,59 @@ impl Generator { .field_types .get(i) .expect("should be available as constructed from same values"); - if layout.art_backend == Some(ArtBackend::ArcticMulti) { - let field_type = &field.ty; + if layout.art_backend == Some(ArtBackend::ArcticMulti) && is_unsized(&ty.to_string()) { Ok(quote! { - let #i: #field_type = Default::default(); + let shadow = IndexMultiMap::<#ty, OffsetEqLink, UnsizedNode<_>>::with_maximum_node_size(#const_name); for (key, value) in self.#i.iter() { - #i.insert_pair(key, value); + shadow.insert(key, value); + } + let mut pages = vec![]; + for node in shadow.iter_nodes() { + pages.push(UnsizedIndexPage::from_node(node.lock_arc().as_ref())); } + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let #i = (toc.pages, pages); + }) + } else if layout.art_backend == Some(ArtBackend::ArcticMulti) { + Ok(quote! { + let size = get_index_page_size_from_data_length::<#ty>(#const_name); + let shadow = IndexMultiMap::<#ty, OffsetEqLink>::with_maximum_node_size(size); + for (key, value) in self.#i.iter() { + shadow.insert(key, value); + } + let mut pages = vec![]; + for node in shadow.iter_nodes() { + pages.push(IndexPage::from_node(node.lock_arc().as_ref(), size)); + } + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let #i = (toc.pages, pages); + }) + } else if layout.art_backend == Some(ArtBackend::Arctic) && is_unsized(&ty.to_string()) { + Ok(quote! { + let shadow = IndexMap::<#ty, OffsetEqLink, UnsizedNode<_>>::with_maximum_node_size(#const_name); + for (key, value) in self.#i.iter_values() { + shadow.insert(key, value); + } + let mut pages = vec![]; + for node in shadow.iter_nodes() { + pages.push(UnsizedIndexPage::from_node(node.lock_arc().as_ref())); + } + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let #i = (toc.pages, pages); + }) + } else if layout.art_backend == Some(ArtBackend::Arctic) { + Ok(quote! { + let size = get_index_page_size_from_data_length::<#ty>(#const_name); + let shadow = IndexMap::<#ty, OffsetEqLink>::with_maximum_node_size(size); + for (key, value) in self.#i.iter_values() { + shadow.insert(key, value); + } + let mut pages = vec![]; + for node in shadow.iter_nodes() { + pages.push(IndexPage::from_node(node.lock_arc().as_ref(), size)); + } + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let #i = (toc.pages, pages); }) } else if layout.art_backend.is_some() { let field_type = &field.ty; @@ -560,7 +606,27 @@ impl Generator { } }; - if layout.art_backend.is_some() { + if layout.art_backend == Some(ArtBackend::ArcticMulti) { + let field_type = &f.ty; + Ok(quote! { + let #i: #field_type = Default::default(); + for page in persisted.#i.1 { + for pair in page.inner.get_node() { + #i.insert_pair(pair.key, OffsetEqLink(pair.value)); + } + } + }) + } else if layout.art_backend == Some(ArtBackend::Arctic) { + let field_type = &f.ty; + Ok(quote! { + let #i: #field_type = Default::default(); + for page in persisted.#i.1 { + for pair in page.inner.get_node() { + #i.insert_value(pair.key, OffsetEqLink(pair.value)); + } + } + }) + } else if layout.art_backend.is_some() { Ok(quote! { let #i = persisted.#i; }) diff --git a/codegen/src/persist_index/space/index.rs b/codegen/src/persist_index/space/index.rs index 9d6f7c61..166b514e 100644 --- a/codegen/src/persist_index/space/index.rs +++ b/codegen/src/persist_index/space/index.rs @@ -19,11 +19,17 @@ impl Generator { let i = field.ident.as_ref().expect("index fields should be named"); let t = self.field_types.get(i).expect("field type was collected"); Ok(match layout.art_backend { + Some(ArtBackend::Arctic) if is_unsized(&t.to_string()) => quote! { + #i: SpaceLogicalIndexUnsized<#t, { #inner_const_name as u32}>, + }, Some(ArtBackend::Arctic) => quote! { - #i: SpaceArcticIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalIndex<#t, { #inner_const_name as u32}>, + }, + Some(ArtBackend::ArcticMulti) if is_unsized(&t.to_string()) => quote! { + #i: SpaceLogicalMultiIndexUnsized<#t, { #inner_const_name as u32}>, }, Some(ArtBackend::ArcticMulti) => quote! { - #i: SpaceArcticMultiIndex<#t, { #inner_const_name as u32}>, + #i: SpaceLogicalMultiIndex<#t, { #inner_const_name as u32}>, }, Some(ArtBackend::Congee) => quote! { #i: SpaceCongeeIndex<#t, { #inner_const_name as u32}>, @@ -82,11 +88,17 @@ impl Generator { let t = self.field_types.get(i).expect("field type was collected"); let literal_name = Literal::string(i.to_string().as_str()); Ok(match layout.art_backend { + Some(ArtBackend::Arctic) if is_unsized(&t.to_string()) => quote! { + #i: SpaceLogicalIndexUnsized::secondary_from_table_files_path(path, #literal_name, version).await?, + }, Some(ArtBackend::Arctic) => quote! { - #i: SpaceArcticIndex::secondary_from_table_files_path(path, #literal_name, version).await?, + #i: SpaceLogicalIndex::secondary_from_table_files_path(path, #literal_name, version).await?, + }, + Some(ArtBackend::ArcticMulti) if is_unsized(&t.to_string()) => quote! { + #i: SpaceLogicalMultiIndexUnsized::secondary_from_table_files_path(path, #literal_name, version).await?, }, Some(ArtBackend::ArcticMulti) => quote! { - #i: SpaceArcticMultiIndex::secondary_from_table_files_path(path, #literal_name, version).await?, + #i: SpaceLogicalMultiIndex::secondary_from_table_files_path(path, #literal_name, version).await?, }, Some(ArtBackend::Congee) => quote! { #i: SpaceCongeeIndex::secondary_from_table_files_path(path, #literal_name, version).await?, diff --git a/codegen/src/persist_table/generator/space.rs b/codegen/src/persist_table/generator/space.rs index 2a9e1718..01ed6376 100644 --- a/codegen/src/persist_table/generator/space.rs +++ b/codegen/src/persist_table/generator/space.rs @@ -38,7 +38,7 @@ impl Generator { let avt_index_ident = name_generator.get_available_indexes_ident(); let space_index_type = if self.attributes.pk_arctic_string { quote! { - SpaceArcticStringIndex<#primary_key_type, { #inner_const_name as u32 }>, + SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }>, } } else if self.attributes.pk_unsized && self.attributes.pk_wti_logical { quote! { @@ -54,7 +54,7 @@ impl Generator { } } else if self.attributes.pk_arctic { quote! { - SpaceArcticIndex<#primary_key_type, { #inner_const_name as u32 }>, + SpaceLogicalIndex<#primary_key_type, { #inner_const_name as u32 }>, } } else if self.attributes.pk_congee { quote! { diff --git a/codegen/src/persist_table/generator/space_file/mod.rs b/codegen/src/persist_table/generator/space_file/mod.rs index c1f99e5d..669373af 100644 --- a/codegen/src/persist_table/generator/space_file/mod.rs +++ b/codegen/src/persist_table/generator/space_file/mod.rs @@ -28,18 +28,10 @@ impl Generator { let inner_const_name = name_generator.get_page_inner_size_const_ident(); let pk_type = name_generator.get_primary_key_type_ident(); let space_file_ident = name_generator.get_space_file_ident(); - let primary_index = if self.attributes.pk_arctic_string { - quote! { - pub primary_index: PersistentArcticIndex<#pk_type, OffsetEqLink<#inner_const_name>>, - } - } else if self.attributes.pk_unsized { + let primary_index = if self.attributes.pk_unsized { quote! { pub primary_index: (Vec>>, Vec>>), } - } else if self.attributes.pk_arctic { - quote! { - pub primary_index: PersistentArcticIndex<#pk_type, OffsetEqLink<#inner_const_name>>, - } } else if self.attributes.pk_congee { quote! { pub primary_index: PersistentCongeeIndex<#pk_type, OffsetEqLink<#inner_const_name>>, @@ -65,12 +57,11 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let literal_name = name_generator.get_work_table_literal_name(); let version_const = name_generator.get_version_const_ident(); - let primary_page_count = - if self.attributes.pk_arctic || self.attributes.pk_arctic_string || self.attributes.pk_congee { - quote! { 1 } - } else { - quote! { self.primary_index.0.len() as u32 + self.primary_index.1.len() as u32 } - }; + let primary_page_count = if self.attributes.pk_congee { + quote! { 1 } + } else { + quote! { self.primary_index.0.len() as u32 + self.primary_index.1.len() as u32 } + }; let row_schema = self.attributes.row_schema.iter().map(|(name, type_name)| { quote! { (#name.to_string(), #type_name.to_string()) } }); @@ -146,9 +137,21 @@ impl Generator { let secondary_index_events = name_generator.get_space_secondary_index_events_ident(); let avt_index_ident = name_generator.get_available_indexes_ident(); - let primary_index_init = if self.attributes.pk_arctic_string { + let primary_index_init = if self.attributes.pk_arctic || self.attributes.pk_arctic_string { + let map_type = if self.attributes.read_only { + quote! { ArcticIndex } + } else { + quote! { PersistentArcticIndex } + }; quote! { - let pk_map = self.primary_index; + let pk_map = #map_type::<#pk_type, OffsetEqLink<#const_name>>::default(); + for page in self.primary_index.1 { + for pair in page.inner.get_node() { + validate_arctic_link(pair.value) + .map_err(|error| PersistenceLoadError::corrupt(path, error))?; + pk_map.insert_value(pair.key, OffsetEqLink(pair.value)); + } + } let primary_index = PrimaryIndex::from_map(pk_map); } } else if self.attributes.pk_unsized { @@ -174,7 +177,7 @@ impl Generator { } let primary_index = PrimaryIndex::from_map(pk_map); } - } else if self.attributes.pk_arctic || self.attributes.pk_congee { + } else if self.attributes.pk_congee { quote! { let pk_map = self.primary_index; let primary_index = PrimaryIndex::from_map(pk_map); @@ -338,7 +341,7 @@ impl Generator { let index_extension = Literal::string(WT_INDEX_EXTENSION); let data_extension = Literal::string(WT_DATA_EXTENSION); - let parse_pk_page = if self.attributes.pk_unsized && !self.attributes.pk_arctic_string { + let parse_pk_page = if self.attributes.pk_unsized { quote! { let index = parse_page::, { #page_const_name as u32 }>(&mut primary_file, (*page_id).into()).await?; } @@ -348,21 +351,7 @@ impl Generator { } }; - let parse_primary = if self.attributes.pk_arctic_string { - quote! { - SpaceArcticStringIndex::<#pk_type, { #inner_const_name as u32 }>::load_index::<#inner_const_name>( - format!("{}/primary{}", path, #index_extension), - #version_const_name, - ).await? - } - } else if self.attributes.pk_arctic { - quote! { - SpaceArcticIndex::<#pk_type, { #inner_const_name as u32 }>::load_index::<#inner_const_name>( - format!("{}/primary{}", path, #index_extension), - #version_const_name, - ).await? - } - } else if self.attributes.pk_congee { + let parse_primary = if self.attributes.pk_congee { quote! { SpaceCongeeIndex::<#pk_type, { #inner_const_name as u32 }>::load_index::<#inner_const_name>( format!("{}/primary{}", path, #index_extension), diff --git a/codegen/src/persist_table/generator/space_file/worktable_impls.rs b/codegen/src/persist_table/generator/space_file/worktable_impls.rs index 38ef6d2e..96d54f86 100644 --- a/codegen/src/persist_table/generator/space_file/worktable_impls.rs +++ b/codegen/src/persist_table/generator/space_file/worktable_impls.rs @@ -36,29 +36,38 @@ impl Generator { self: std::sync::Arc, timeout: std::time::Duration, quiesce: F, - ) -> eyre::Result + ) -> Result> where F: FnOnce() -> Fut, Fut: std::future::Future, { - let released_bytes = self.heap_size(); - tokio::time::timeout(timeout, quiesce()) - .await - .map_err(|_| eyre::eyre!("timed out waiting for generation leases to quiesce"))?; - - let outstanding = std::sync::Arc::strong_count(&self).saturating_sub(1); - if outstanding != 0 { - return Err(eyre::eyre!( - "cannot unload generation: {outstanding} Arc lease(s) remain after quiesce" + // Attribute the generation at the retirement request. The + // quiesce callback can give background maintenance time to + // shrink or rearrange live structures before the final drop; + // measuring afterwards would make the report depend on how + // long the reader barrier happened to take. + let estimated_released_bytes = self.heap_size(); + if tokio::time::timeout(timeout, quiesce()).await.is_err() { + return Err(UnloadFailure::retained( + self, + eyre::eyre!("timed out waiting for generation leases to quiesce"), )); } - let owned = std::sync::Arc::try_unwrap(self).map_err(|arc| { - let outstanding = std::sync::Arc::strong_count(&arc).saturating_sub(1); - eyre::eyre!("cannot unload generation: {outstanding} Arc lease(s) remain") + let owned = match std::sync::Arc::try_unwrap(self) { + Ok(owned) => owned, + Err(arc) => { + let outstanding = std::sync::Arc::strong_count(&arc).saturating_sub(1); + return Err(UnloadFailure::retained( + arc, + eyre::eyre!("cannot unload generation: {outstanding} Arc lease(s) remain"), + )); + } + }; + owned.close().await.map_err(|error| { + UnloadFailure::after_close(eyre::Report::new(error)) })?; - owned.close().await?; - Ok(UnloadReport { released_bytes }) + Ok(UnloadReport { estimated_released_bytes }) } } } @@ -175,10 +184,40 @@ impl Generator { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let pk_type = name_generator.get_primary_key_type_ident(); let const_name = name_generator.get_page_inner_size_const_ident(); - if self.attributes.pk_arctic || self.attributes.pk_arctic_string || self.attributes.pk_congee { - // ART durability is maintained incrementally by its native - // checkpoint/WAL file rather than materialized as DataBucket pages. + if self.attributes.pk_congee { + // Congee durability is maintained by its native checkpoint/WAL. quote! {} + } else if self.attributes.pk_arctic_string { + quote! { + pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { + let shadow = IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name); + for (key, value) in self.0.primary_index.pk_map.iter_values() { + shadow.insert(key, value); + } + let mut pages = vec![]; + for node in shadow.iter_nodes() { + pages.push(UnsizedIndexPage::from_node(node.lock_arc().as_ref())); + } + let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + (toc.pages, pages) + } + } + } else if self.attributes.pk_arctic { + quote! { + pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { + let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); + let shadow = IndexMap::<#pk_type, OffsetEqLink<#const_name>>::with_maximum_node_size(size); + for (key, value) in self.0.primary_index.pk_map.iter_values() { + shadow.insert(key, value); + } + let mut pages = vec![]; + for node in shadow.iter_nodes() { + pages.push(IndexPage::from_node(node.lock_arc().as_ref(), size)); + } + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + (toc.pages, pages) + } + } } else if self.attributes.pk_unsized { quote! { pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { diff --git a/codegen/src/persist_table/parser.rs b/codegen/src/persist_table/parser.rs index e19f2a43..4f84aad2 100644 --- a/codegen/src/persist_table/parser.rs +++ b/codegen/src/persist_table/parser.rs @@ -60,6 +60,7 @@ impl Parser { } if meta.path.is_ident("pk_arctic_string") { res.pk_arctic_string = true; + res.pk_unsized = true; return Ok(()); } if meta.path.is_ident("pk_congee") { diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index f17c45d2..fe5e3276 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -83,6 +83,7 @@ pub fn expand(input: TokenStream) -> syn::Result { worktable_dsl::validate::validate_index_backends(&columns, persistence)?; worktable_dsl::validate::validate_page_size(config.as_ref(), persistence)?; + worktable_dsl::validate::validate_arctic_page_size(&columns, config.as_ref())?; if let Some(q) = &queries { worktable_dsl::validate::validate_in_place_queries(&columns, q)?; } @@ -179,8 +180,8 @@ mod tests { name: CompositePrimaryKeyOrder, persist: #persist, columns: { - tenant_id: u64 primary_key, - record_id: u64 primary_key, + tenant_id: u64 primary_key using worktables_index, + record_id: u64 primary_key using worktables_index, value: i64, }, }) @@ -191,7 +192,7 @@ mod tests { } #[test] - fn absent_using_keeps_worktables_index_default() { + fn absent_using_selects_arctic_runtime_with_compatible_persistence() { let output = expand(quote! { name: DefaultBackend, persist: true, @@ -206,12 +207,8 @@ mod tests { .unwrap() .to_string(); - if cfg!(feature = "logical-index-persistence") { - assert!(output.contains("PersistentWtiIndex")); - } else { - assert!(output.contains("IndexMap")); - assert!(!output.contains("PersistentWtiIndex")); - } + assert!(output.contains("PersistentArcticIndex")); + assert!(output.contains("table (pk_arctic)")); } #[test] @@ -312,7 +309,7 @@ mod tests { } #[test] - fn art_backend_requires_explicit_persistence_choice() { + fn congee_backend_requires_explicit_persistence_choice() { let error = expand(quote! { name: MissingAcknowledgement, columns: { @@ -415,7 +412,7 @@ mod tests { assert!( error .to_string() - .contains("supported types: String, u16, u32, u64, u128, i16, i32, i64, i128") + .contains("supported types: String, u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128") ); } @@ -465,11 +462,11 @@ mod tests { #[test] fn arctic_rejects_unsupported_secondary_keys() { let error = expand(quote! { - name: ByteArctic, + name: BoolArctic, persist: false, columns: { id: u64 primary_key, - value: u8, + value: bool, }, indexes: { value_idx: value unique using arctic, @@ -480,7 +477,7 @@ mod tests { assert!( error .to_string() - .contains("supported types: String, u16, u32, u64, u128, i16, i32, i64, i128") + .contains("supported types: String, u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128") ); } diff --git a/codegen/src/worktable_version/mod.rs b/codegen/src/worktable_version/mod.rs index ea7c1ed2..496d0e7a 100644 --- a/codegen/src/worktable_version/mod.rs +++ b/codegen/src/worktable_version/mod.rs @@ -64,7 +64,7 @@ mod tests { "should generate read_only index attribute" ); assert!( - output.contains("table (read_only)"), + output.contains("table (read_only , pk_arctic)"), "should generate read_only table attribute" ); } @@ -153,7 +153,7 @@ mod tests { let output = res.to_string(); assert!( - output.contains("table (read_only , pk_unsized)"), + output.contains("table (read_only , pk_arctic_string)"), "an unsized primary key must keep read_only, not replace it with pk_unsized" ); } diff --git a/docs/index-backend-dsl-proposal.md b/docs/index-backend-dsl-proposal.md index b904dea5..19613026 100644 --- a/docs/index-backend-dsl-proposal.md +++ b/docs/index-backend-dsl-proposal.md @@ -1,8 +1,8 @@ # Per-index backends with `using` -**Status:** PR #187 is merged; native ART persistence is implemented on `feat/art-native-persistence` and remains experimental pending validation. +**Status:** Arctic is the default runtime backend for beta.18. Persisted Arctic-backed declarations retain the beta.17 WorkTablesIndex page format. -**Default:** `worktables_index` +**Default:** `arctic` **Backends in this change:** `worktables_index`, `indexset`, `congee`, `arctic` @@ -14,7 +14,7 @@ The generated table contains concrete map types. Selection is resolved by the ma This has two distinct uses: -- **Production migration:** persisted tables can select a backend per index. WorkTablesIndex and vanilla IndexSet share the existing DataBucket page representation; Congee and Arctic use backend-native topology checkpoints plus logical WAL records. +- **Production migration:** persisted tables can select a backend per index. WorkTablesIndex, vanilla IndexSet, and Arctic share the existing DataBucket page representation; Congee remains backend-native. - **Research and measurement:** the same schema can compare memory-only and persisted Congee/Arctic access paths without runtime backend dispatch. The useful paper claim is not that WorkTable bundles several maps. It is that a generated table can statically select a physical implementation per access path, keep a stable typed API, and reject incompatible persistence or key semantics at compile time. @@ -45,14 +45,14 @@ There is no separate `config` syntax for this feature. The physical choice stays ### The absent-`using` default -Omitting `using` always means `worktables_index`: +Omitting `using` means an Arctic runtime index: ```rust columns: { - id: u64 primary_key autoincrement, // WorkTablesIndex + id: u64 primary_key autoincrement, // Arctic runtime, WTI disk pages when persisted }, indexes: { - account_idx: account_id unique, // WorkTablesIndex + account_idx: account_id unique, // Arctic runtime, WTI disk pages when persisted } ``` @@ -60,14 +60,15 @@ The explicit equivalent is: ```rust columns: { - id: u64 primary_key autoincrement using worktables_index, + id: u64 primary_key autoincrement using arctic, }, indexes: { - account_idx: account_id unique using worktables_index, + account_idx: account_id unique using arctic, } ``` -This default is intentional. Vanilla `indexset` is an explicit fourth backend; it is not the default and does not silently replace WorkTablesIndex. +Use `using worktables_index` explicitly for composite, UUID, PackedNanoid, +floating-point, and other key shapes Arctic does not support. ## Persistence is controlled by the existing `persist` declaration @@ -75,12 +76,13 @@ No new persistence keyword is introduced. | Declaration | Meaning | Allowed backends | |---|---|---| -| `persist` omitted | Existing non-persisted table behavior | WorkTablesIndex or vanilla IndexSet; ART use requires an explicit persistence choice | +| `persist` omitted | Existing non-persisted table behavior | Arctic by default; all memory backends are selectable | | `persist: false` | Explicitly memory-only | All four backends, subject to key and uniqueness constraints | | `persist: true` | Local durable persistence plus in-memory indexes | All four; ART persistence is experimental | | S3 support | Existing S3 sync layered over local persistence | File paths are compatible; ART end-to-end S3 validation remains required | -Congee and Arctic require an explicit `persist: true` or `persist: false`; omitting `persist` is not sufficient acknowledgement. This makes the durability choice visible during review: +Congee requires an explicit `persist: true` or `persist: false`. Arctic does not, +because it is the ordinary default: ```rust worktable!( @@ -96,7 +98,8 @@ worktable!( ); ``` -The macro accepts the same schema with `persist: true` and selects native ART persistence. It rejects the schema when `persist` is omitted. +The macro accepts the same schema with `persist: true`. Arctic is also valid when +`persist` is omitted because it is the default runtime backend. ## Current capability matrix @@ -105,18 +108,18 @@ The macro accepts the same schema with `persist: true` and selects native ART pe | Primary index | Yes | Yes | Yes | Yes | | Unique secondary index | Yes | Yes | Yes | Yes | | Non-unique secondary index | Yes | No | No | Yes | -| Persisted local disk | Yes | Yes | Experimental | Experimental | -| Existing S3 persistence path | Yes | Yes | Files compatible; validation pending | Files compatible; validation pending | -| Variable-sized keys | Yes | Not in this change | No | No | +| Persisted local disk | Yes | Yes | Experimental native format | Yes, WTI-compatible format | +| Existing S3 persistence path | Yes | Yes | Files compatible; validation pending | WTI-compatible files; validation pending | +| Variable-sized keys | Yes | Not in this change | No | Yes (`String`) | | Ordered point/range API | Yes | Yes | Adapter snapshot for scans | Adapter snapshot for scans | -| Default when `using` is absent | Yes | No | No | No | +| Default when `using` is absent | No | No | No | Yes | -Arctic additionally supports non-unique secondary indexes: `value_idx: value using arctic` maps each key to a boxed link collection with multiset semantics, for memory-only and persisted tables alike (persisted through a pair-list checkpoint plus logical `(key, link)` WAL). Non-unique declarations on Congee or vanilla IndexSet still fail at macro expansion and tell the author to use `worktables_index` or `arctic`. +Arctic additionally supports non-unique secondary indexes: `value_idx: value using arctic` maps each key to a link collection with multiset semantics. Persisted Arctic tables keep the WTI page format and translate logical `(key, link)` mutations in the persistence worker, so beta.17 files remain directly readable. Non-unique declarations on Congee or vanilla IndexSet still fail at macro expansion and tell the author to use `worktables_index` or `arctic`. ### Key constraints - **Congee:** `u8`, `u16`, `u32`, `usize`, and `u64` on 64-bit targets. Its native key and payload are one machine word. Composite, NanoID, string, signed, and floating-point keys are rejected. -- **Arctic:** `u16`, `u32`, `u64`, and `u128` in this initial adapter. Its crate supports more representations, but WorkTable exposes only the shapes covered by the current contract tests. +- **Arctic:** `String`, `u8`, `u16`, `u32`, `u64`, `u128`, `usize`, `i8`, `i16`, `i32`, `i64`, and `i128`. - **Vanilla IndexSet:** sized ordered keys in this change. Variable-sized keys remain on WorkTablesIndex. - **WorkTablesIndex:** retains the existing generic and variable-sized key support. @@ -140,9 +143,12 @@ The selected provider is therefore an in-memory implementation detail, not a new It also separately covers vanilla IndexSet persist → reload → mutate → reload. This is the technical basis for deploying the two providers in parallel without a full data rebuild. -Congee and Arctic deliberately do **not** normalize into WorkTablesIndex pages. Their `*.wt.idx` files contain a checksummed pointer-free checkpoint of the selected ART's physical topology followed by logical Set/Remove WAL frames. Compaction reconstructs a temporary native ART, applies the WAL, and atomically replaces the checkpoint; it does not retain a duplicate authoritative tree during normal operation. See [Native ART index persistence](art-index-persistence-plan.md). - -Because those physical formats differ, switching an existing index between an ART and a B-tree requires an explicit rebuild or migration. WorkTablesIndex ↔ vanilla IndexSet remains the format-compatible provider switch. +Arctic deliberately normalizes into WorkTablesIndex pages. Its foreground map +emits logical Set/Remove events; the persistence worker applies them to a WTI +shadow reconstructed with the exact existing node boundaries, then persists the +resulting structural CDC. This lets a beta.18 Arctic runtime open and continue +mutating beta.17 files without a conversion pass. Congee's explicitly selected +backend still uses its native checkpoint/WAL format. This is still a sensitive storage path. Production rollout should retain backups, verify the exact downstream schema/version, and run crash/torn-write and sustained post-reload mutation tests before changing a live table. @@ -182,7 +188,8 @@ page bytes, ghost publication, and reclamation rather than index routing. ### Arctic - Point lookup and mutation call Arctic directly. -- WorkTable links are stored in `Box` values because Arctic's inline value is limited to 64 bits. Inserts allocate; reads copy the link from the box. +- WorkTable links are packed into Arctic's inline 64-bit value. The DSL rejects + page sizes above 65,535 bytes because offset and length each occupy 16 bits. - Ordered reads use Arctic's native bounded traversal and materialize the requested interval into a `Vec`. - Concurrent scan behavior inherits Arctic's non-linearizable traversal contract. - With `persist: true`, mutations use the same persistence-only sequencing wrapper as Congee. Point reads remain direct and lock-free. @@ -197,9 +204,10 @@ reporting one blended throughput number. ### Memory diagnostics -WorkTablesIndex and vanilla IndexSet expose node capacity and topology used by existing `system_info` reporting. Congee and Arctic do not expose equivalent stable allocator statistics. For those adapters: +WorkTablesIndex and vanilla IndexSet expose node capacity and topology used by existing `system_info` reporting. Arctic does not expose equivalent stable allocator statistics. For ART adapters: -- reported used/heap bytes are only a payload-size lower bound; +- `MemStat` includes logical payload and heap-backed key/value bytes and Congee + node allocations, but Arctic node overhead and retired SMR allocations remain estimates; - reported capacity equals logical length; - reported node count is zero/unknown. @@ -209,7 +217,7 @@ Use allocator/RSS measurements for comparative memory results; do not treat the This implementation pins two narrow forks for typed topology import/export: -- `WorkTablesIndex 0.0.5` as the default `indexset` dependency alias already used by WorkTable; +- the current WorkTablesIndex release as the compatible disk-format implementation; - vanilla `indexset 0.15.0` under the `vanilla_indexset` Cargo name; - `congee-wt` at commit `005bfb1968e781800176f2d7e465e6a1af630e1a`; - `arctic-wt` at commit `e13fc7df3c040f14ae66c1cb56b1bd0a3f6da3fc`. @@ -255,8 +263,9 @@ For the paper, the strongest controlled experiment keeps the WorkTable schema, g ## Production versus research classification -- **WorkTablesIndex:** production default. +- **Arctic:** production runtime default with WTI-compatible persistence. +- **WorkTablesIndex:** explicit fallback for unsupported key shapes and direct structural CDC. - **Vanilla IndexSet:** experimental provider. It preserves local/S3 persistence through the existing format boundary, but is excluded from concurrent correctness and published performance claims until upstream offers a stable structural-read primitive or the adapter gains a low-cost algorithm. -- **Congee and Arctic:** research/experimental backends with native local persistence. Promotion requires crash/S3 validation, relevant downstream evidence, allocation/reclamation review, and a workload that does not depend on the current allocating scan path. +- **Congee:** explicit experimental backend with native local persistence. That boundary is deliberate: `using` exposes optional physical specialization without quietly weakening WorkTable's in-memory/on-disk coordination contract. diff --git a/dsl/src/model/column.rs b/dsl/src/model/column.rs index 26f046ba..688d4f62 100644 --- a/dsl/src/model/column.rs +++ b/dsl/src/model/column.rs @@ -69,16 +69,17 @@ impl Columns { } else { gen_type = Some(row.gen_type) } - let backend = row.index_backend.unwrap_or_default(); - if let Some(existing) = primary_index_backend { - if existing != backend { - return Err(syn::Error::new( - row.name.span(), - "all columns in a composite primary key must use the same index backend", - )); + if let Some(backend) = row.index_backend { + if let Some(existing) = primary_index_backend { + if existing != backend { + return Err(syn::Error::new( + row.name.span(), + "all columns in a composite primary key must use the same index backend", + )); + } + } else { + primary_index_backend = Some(backend); } - } else { - primary_index_backend = Some(backend); } pk.push(row.name); } else if row.index_backend.is_some() { @@ -93,12 +94,24 @@ impl Columns { return Err(syn::Error::new(input.span(), "Primary key must be set")); } + // Arctic is the default for the common single-column key. Its native + // key contract cannot represent tuples, so a composite declaration + // with no explicit `using` retains WTI rather than becoming invalid + // merely because the global default changed. + let primary_index_backend = primary_index_backend.unwrap_or_else(|| { + if pk.len() > 1 { + IndexBackend::WorktablesIndex + } else { + IndexBackend::default() + } + }); + Ok(Self { is_sized: sized, columns_map, indexes: Default::default(), primary_keys: pk, - primary_index_backend: primary_index_backend.unwrap_or_default(), + primary_index_backend, generator_type: gen_type.expect("set"), field_positions, }) diff --git a/dsl/src/model/index.rs b/dsl/src/model/index.rs index 53c17be6..79fca62e 100644 --- a/dsl/src/model/index.rs +++ b/dsl/src/model/index.rs @@ -2,22 +2,22 @@ use proc_macro2::Ident; /// Physical implementation selected for a generated index. /// -/// `WorktablesIndex` is deliberately the default so existing declarations keep -/// their current implementation and persistence semantics when `using` is -/// absent. Vanilla upstream IndexSet is an explicit, parallel backend. +/// Arctic is the default runtime backend. Persisted tables retain the existing +/// WorkTablesIndex page format, so declarations without `using` can open files +/// created by earlier releases while getting Arctic for in-memory lookups. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum IndexBackend { - #[default] WorktablesIndex, Indexset, Congee, + #[default] Arctic, } impl IndexBackend { pub fn requires_explicit_persistence(self) -> bool { - matches!(self, Self::Congee | Self::Arctic) + matches!(self, Self::Congee) } pub fn name(self) -> &'static str { diff --git a/dsl/src/parser/index.rs b/dsl/src/parser/index.rs index 7dd4adcd..a495a8cd 100644 --- a/dsl/src/parser/index.rs +++ b/dsl/src/parser/index.rs @@ -145,10 +145,10 @@ mod tests { use crate::model::IndexBackend; #[test] - fn absent_using_defaults_to_worktables_index() { + fn absent_using_defaults_to_arctic() { let mut parser = Parser::new(quote! { value_idx: value unique, }); let (_, index) = parser.parse_index().unwrap(); - assert_eq!(index.backend, IndexBackend::WorktablesIndex); + assert_eq!(index.backend, IndexBackend::Arctic); } #[test] diff --git a/dsl/src/validate.rs b/dsl/src/validate.rs index 00885a37..36415291 100644 --- a/dsl/src/validate.rs +++ b/dsl/src/validate.rs @@ -50,6 +50,32 @@ pub fn validate_page_size(config: Option<&crate::model::Config>, persistence: Pe Ok(()) } +/// Arctic stores a row link in one inline `u64`: 32 bits for the page id and +/// 16 bits each for the offset and length. A larger page can produce a link +/// that cannot be represented, even for an in-memory table. +pub fn validate_arctic_page_size(columns: &Columns, config: Option<&crate::model::Config>) -> syn::Result<()> { + let Some(config) = config else { return Ok(()) }; + let Some(page_size) = config.page_size else { + return Ok(()); + }; + let uses_arctic = columns.primary_index_backend == IndexBackend::Arctic + || columns + .indexes + .values() + .any(|index| index.backend == IndexBackend::Arctic); + if uses_arctic && page_size > u32::from(u16::MAX) { + let span = config.page_size_span.unwrap_or_else(proc_macro2::Span::call_site); + return Err(syn::Error::new( + span, + format!( + "`page_size: {page_size}` is too large for an Arctic-backed table: Arctic packs each row link into 64 bits and its offset and length fields are 16 bits. Use a page size no larger than {} or select `using worktables_index` for every index", + u16::MAX, + ), + )); + } + Ok(()) +} + /// `in_place` queries hand the caller a mutable reference to the archived /// column bytes and bypass all index maintenance, so a column that any index /// is built over cannot be mutated in place: the index would keep resolving @@ -235,7 +261,9 @@ pub const AUTOINCREMENT_TYPES: &[&str] = &["u8", "u16", "u32", "u64", "i8", "i16 pub fn supported_key_types(backend: IndexBackend) -> Option<&'static [&'static str]> { match backend { IndexBackend::Congee => Some(&["u8", "u16", "u32", "u64", "usize"]), - IndexBackend::Arctic => Some(&["String", "u16", "u32", "u64", "u128", "i16", "i32", "i64", "i128"]), + IndexBackend::Arctic => Some(&[ + "String", "u8", "u16", "u32", "u64", "u128", "usize", "i8", "i16", "i32", "i64", "i128", + ]), IndexBackend::WorktablesIndex | IndexBackend::Indexset => None, } } @@ -265,6 +293,9 @@ pub fn all( if let Err(error) = validate_page_size(config, persistence) { errors.push(error); } + if let Err(error) = validate_arctic_page_size(columns, config) { + errors.push(error); + } if let Some(queries) = queries && let Err(error) = validate_in_place_queries(columns, queries) { diff --git a/dsl/tests/check.rs b/dsl/tests/check.rs index 593c44e6..f10dc2a0 100644 --- a/dsl/tests/check.rs +++ b/dsl/tests/check.rs @@ -194,6 +194,31 @@ fn a_plain_page_size_is_still_accepted() { assert!(checked.is_acceptable(), "unexpected: {:?}", checked.diagnostics); } +#[test] +fn arctic_rejects_links_wider_than_its_inline_encoding() { + let checked = worktable_dsl::check( + "name: T, persist: false, columns: { id: u64 primary_key }, config: { page_size: 65536 },", + ); + assert!(!checked.is_acceptable()); + assert!( + checked + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("16 bits")), + "the diagnostic explains the link-width limit: {:?}", + checked.diagnostics + ); + + let wti = worktable_dsl::check( + "name: T, persist: false, columns: { id: u64 primary_key using worktables_index }, config: { page_size: 65536 },", + ); + assert!( + wti.is_acceptable(), + "WTI does not use Arctic's packed link: {:?}", + wti.diagnostics + ); +} + /// Several indexes breaking the same rule must all be reported. /// /// `index_backends_into` used to pick one offender: the primary if it @@ -254,10 +279,15 @@ fn a_primary_key_backend_is_checked_against_its_key_type() { refused.diagnostics ); - // arctic has the same shape with a different list. + // Arctic accepts its compact integer key widths, including u8. + assert!( + worktable_dsl::check("name: P, persist: false, columns: { id: u8 primary_key using arctic },").is_acceptable(), + "arctic holds u8" + ); assert!( - !worktable_dsl::check("name: P, persist: false, columns: { id: u8 primary_key using arctic },").is_acceptable(), - "arctic does not hold u8" + !worktable_dsl::check("name: P, persist: false, columns: { id: bool primary_key using arctic },") + .is_acceptable(), + "arctic does not hold bool" ); // And a key type the backend does hold is still accepted. diff --git a/dsl/tests/schema.rs b/dsl/tests/schema.rs index c3f36d7f..fb0ea36a 100644 --- a/dsl/tests/schema.rs +++ b/dsl/tests/schema.rs @@ -77,7 +77,7 @@ fn an_omitted_persist_is_not_written_back() { #[test] fn only_a_deliberate_backend_choice_is_written_back() { // A primary key always carries a backend once parsed, because the model - // fills the default in. Emitting `using worktables_index` everywhere would + // fills the default in. Emitting `using arctic` everywhere would // round-trip correctly and read like noise. let default = parse("name: Default, columns: { id: u64 primary_key }"); assert!(!default.to_dsl().contains("using")); diff --git a/src/in_memory/data.rs b/src/in_memory/data.rs index 93a1ad1d..7b15d55e 100644 --- a/src/in_memory/data.rs +++ b/src/in_memory/data.rs @@ -2,7 +2,7 @@ use std::cell::UnsafeCell; use std::fmt::Debug; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; -use std::sync::atomic::{AtomicU8, AtomicU32, Ordering}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use data_bucket::page::INNER_PAGE_SIZE; use data_bucket::page::PageId; @@ -12,7 +12,7 @@ use derive_more::{Display, Error}; use performance_measurement_codegen::performance_measurement; use rkyv::{ Archive, Deserialize, Portable, Serialize, - api::{high::HighDeserializer, root_position}, + api::high::HighDeserializer, rancor::Strategy, seal::Seal, ser::{Serializer, allocator::ArenaHandle, sharing::Share}, @@ -23,34 +23,159 @@ use rkyv::{ use crate::in_memory::ArchivedRowWrapper; use crate::prelude::Link; -const CELL_WRITER: u8 = 1 << 7; -const CELL_READERS: u8 = !CELL_WRITER; +const CELL_LOCK_SLOTS: usize = 64; +const CELL_KEY_MASK: u64 = u32::MAX as u64; +const CELL_READER_ONE: u64 = 1 << 32; +const CELL_READER_MASK: u64 = ((1_u64 << 31) - 1) << 32; +const CELL_WRITER: u64 = 1 << 63; + +#[derive(Debug)] +struct CellLocks { + slots: [AtomicU64; CELL_LOCK_SLOTS], +} + +impl Default for CellLocks { + fn default() -> Self { + Self { + slots: std::array::from_fn(|_| AtomicU64::new(0)), + } + } +} + +impl CellLocks { + #[inline] + fn key(link: Link) -> Result { + u64::from(link.offset) + .checked_add(1) + .filter(|key| *key <= CELL_KEY_MASK) + .ok_or(ExecutionError::InvalidLink) + } + + #[inline] + fn start(key: u64) -> usize { + (key.wrapping_mul(0x9e37_79b9) as usize) & (CELL_LOCK_SLOTS - 1) + } + + #[inline] + fn wait(spins: &mut u32) { + if *spins < 64 { + std::hint::spin_loop(); + *spins += 1; + } else { + std::thread::yield_now(); + } + } + + fn read(&self, link: Link) -> Result, ExecutionError> { + let key = Self::key(link)?; + let start = Self::start(key); + let mut spins = 0; + 'retry: loop { + for distance in 0..CELL_LOCK_SLOTS { + let state = &self.slots[(start + distance) & (CELL_LOCK_SLOTS - 1)]; + let current = state.load(Ordering::Acquire); + let current_key = current & CELL_KEY_MASK; + if current_key == key { + if current & CELL_WRITER != 0 || current & CELL_READER_MASK == CELL_READER_MASK { + Self::wait(&mut spins); + continue 'retry; + } + if state + .compare_exchange_weak(current, current + CELL_READER_ONE, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return Ok(CellReadGuard { state }); + } + continue 'retry; + } + if current == 0 { + if state + .compare_exchange_weak(0, key | CELL_READER_ONE, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return Ok(CellReadGuard { state }); + } + continue 'retry; + } + } + Self::wait(&mut spins); + } + } + + fn write(&self, link: Link) -> Result, ExecutionError> { + let key = Self::key(link)?; + let start = Self::start(key); + let mut spins = 0; + 'retry: loop { + for distance in 0..CELL_LOCK_SLOTS { + let state = &self.slots[(start + distance) & (CELL_LOCK_SLOTS - 1)]; + let current = state.load(Ordering::Acquire); + let current_key = current & CELL_KEY_MASK; + if current_key == key { + if current & CELL_WRITER != 0 { + Self::wait(&mut spins); + continue 'retry; + } + if state + .compare_exchange_weak(current, current | CELL_WRITER, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + continue 'retry; + } + while state.load(Ordering::Acquire) & CELL_READER_MASK != 0 { + Self::wait(&mut spins); + } + return Ok(CellWriteGuard { state }); + } + if current == 0 { + if state + .compare_exchange_weak(0, key | CELL_WRITER, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + return Ok(CellWriteGuard { state }); + } + continue 'retry; + } + } + Self::wait(&mut spins); + } + } + + fn reset(&self) { + for slot in &self.slots { + slot.store(0, Ordering::Release); + } + } +} /// Shared access to one exact archived cell. pub(crate) struct CellReadGuard<'a> { - state: *mut AtomicU8, - marker: PhantomData<&'a AtomicU8>, + state: &'a AtomicU64, } impl Drop for CellReadGuard<'_> { #[inline] fn drop(&mut self) { - // SAFETY: the guard cannot outlive the page from which `state` came. - unsafe { &*self.state }.fetch_sub(1, Ordering::Release); + let previous = self.state.fetch_sub(CELL_READER_ONE, Ordering::Release); + debug_assert_ne!(previous & CELL_READER_MASK, 0, "cell reader count underflow"); + let remaining = previous - CELL_READER_ONE; + if remaining & (CELL_READER_MASK | CELL_WRITER) == 0 { + let _ = self + .state + .compare_exchange(remaining, 0, Ordering::Release, Ordering::Relaxed); + } } } /// Exclusive access to one exact archived cell. pub(crate) struct CellWriteGuard<'a> { - state: *mut AtomicU8, - marker: PhantomData<&'a AtomicU8>, + state: &'a AtomicU64, } impl Drop for CellWriteGuard<'_> { #[inline] fn drop(&mut self) { - // SAFETY: the guard cannot outlive the page from which `state` came. - unsafe { &*self.state }.store(0, Ordering::Release); + self.state.store(0, Ordering::Release); } } @@ -97,6 +222,12 @@ pub struct Data { #[rkyv(with = Skip)] pub(crate) access: parking_lot::RwLock<()>, + /// Runtime-only exact-cell reader/writer coordination. The fixed table is + /// outside the archived row image, so lock state can never reach disk and + /// the beta.17 wrapper layout remains unchanged. + #[rkyv(with = Skip)] + cell_locks: CellLocks, + /// Number of live cells currently published on this page. /// /// Vacuum gets move candidates from a transient snapshot of the primary @@ -119,28 +250,7 @@ pub struct Data { unsafe impl Sync for Data {} impl Data { - fn archived_cell_state_offset(bytes: &mut [u8]) -> Result - where - Row: Archive, - ::Archived: ArchivedRowWrapper, - { - let root_offset = root_position::<::Archived>(bytes.len()); - let base = bytes.as_mut_ptr(); - let root = unsafe { base.add(root_offset).cast::<::Archived>() }; - let state = unsafe { ::Archived::cell_state_ptr(root) }.cast::(); - let offset = unsafe { state.offset_from(base) }; - let offset = usize::try_from(offset).map_err(|_| ExecutionError::InvalidLink)?; - if offset >= bytes.len() { - return Err(ExecutionError::InvalidLink); - } - Ok(offset) - } - - fn cell_state_ptr(&self, link: Link) -> Result<*mut AtomicU8, ExecutionError> - where - Row: Archive, - ::Archived: ArchivedRowWrapper, - { + fn validate_link(&self, link: Link) -> Result<(), ExecutionError> { let start = link.offset as usize; let end = start .checked_add(link.length as usize) @@ -149,17 +259,7 @@ impl Data { if link.length == 0 || end > initialized || end > DATA_LENGTH { return Err(ExecutionError::InvalidLink); } - - let inner_data = unsafe { &mut *self.inner_data.get() }; - let root = unsafe { - inner_data - .as_mut_ptr() - .add(start + root_position::<::Archived>(link.length as usize)) - .cast::<::Archived>() - }; - // SAFETY: `root` points at this cell's archived wrapper. The wrapper - // contract places its atomic state at a stable archived offset. - Ok(unsafe { ::Archived::cell_state_ptr(root) }) + Ok(()) } pub(crate) fn read_cell(&self, link: Link) -> Result, ExecutionError> @@ -167,24 +267,8 @@ impl Data { Row: Archive, ::Archived: ArchivedRowWrapper, { - let state = self.cell_state_ptr(link)?; - let state_ref = unsafe { &*state }; - loop { - let current = state_ref.load(Ordering::Acquire); - if current & CELL_WRITER != 0 || current & CELL_READERS == CELL_READERS { - std::hint::spin_loop(); - continue; - } - if state_ref - .compare_exchange_weak(current, current + 1, Ordering::Acquire, Ordering::Relaxed) - .is_ok() - { - return Ok(CellReadGuard { - state, - marker: PhantomData, - }); - } - } + self.validate_link(link)?; + self.cell_locks.read(link) } pub(crate) fn write_cell(&self, link: Link) -> Result, ExecutionError> @@ -192,27 +276,8 @@ impl Data { Row: Archive, ::Archived: ArchivedRowWrapper, { - let state = self.cell_state_ptr(link)?; - let state_ref = unsafe { &*state }; - loop { - let current = state_ref.load(Ordering::Acquire); - if current & CELL_WRITER != 0 { - std::hint::spin_loop(); - continue; - } - if state_ref - .compare_exchange_weak(current, current | CELL_WRITER, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { - while state_ref.load(Ordering::Acquire) != CELL_WRITER { - std::hint::spin_loop(); - } - return Ok(CellWriteGuard { - state, - marker: PhantomData, - }); - } - } + self.validate_link(link)?; + self.cell_locks.write(link) } /// Creates new [`Data`] page. @@ -221,6 +286,7 @@ impl Data { id, free_offset: AtomicU32::default(), access: parking_lot::RwLock::new(()), + cell_locks: CellLocks::default(), live_cells: AtomicU32::new(0), inner_data: UnsafeCell::new(AlignedBytes::([0; DATA_LENGTH])), _phantom: PhantomData, @@ -232,6 +298,7 @@ impl Data { id: page.header.page_id, free_offset: AtomicU32::from(page.header.data_length), access: parking_lot::RwLock::new(()), + cell_locks: CellLocks::default(), live_cells: AtomicU32::new(0), inner_data: UnsafeCell::new(AlignedBytes::(page.inner.data)), _phantom: PhantomData, @@ -278,12 +345,18 @@ impl Data { length, }; - self.register_cell(link); + self.register_cell(link)?; Ok(link) } - #[allow(clippy::missing_safety_doc)] + /// Replaces the complete archived row at an existing link. + /// + /// # Safety + /// + /// The caller must hold this cell's write guard until the copy finishes. + /// The serialized replacement must have the same archived layout as the + /// existing row. #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "DataRow"))] pub unsafe fn save_row_by_link(&self, row: &Row, link: Link) -> Result where @@ -305,34 +378,6 @@ impl Data { Ok(link) } - /// Replaces an archived cell while preserving its active synchronization - /// byte. The caller must hold this cell's write guard for the entire call. - /// - /// The lock lives inside the archived wrapper, so copying the serialized - /// replacement wholesale would briefly publish a zero lock byte while the - /// surrounding row is only partially copied. A reader could then enter the - /// cell and observe a torn row. Copy the bytes on either side instead. - #[allow(clippy::missing_safety_doc)] - pub unsafe fn save_row_by_link_preserving_cell_state(&self, row: &Row, link: Link) -> Result - where - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - ::Archived: ArchivedRowWrapper, - { - let mut bytes = rkyv::to_bytes(row).map_err(|_| ExecutionError::SerializeError)?; - let length = bytes.len() as u32; - if length != link.length { - return Err(ExecutionError::InvalidLink); - } - - let state = Self::archived_cell_state_offset(bytes.as_mut_slice())?; - let inner_data = unsafe { &mut *self.inner_data.get() }; - let destination = &mut inner_data[link.offset as usize..][..link.length as usize]; - destination[..state].copy_from_slice(&bytes[..state]); - destination[state + 1..].copy_from_slice(&bytes[state + 1..]); - - Ok(link) - } - #[allow(clippy::missing_safety_doc)] pub unsafe fn try_save_row_by_link(&self, row: &Row, mut link: Link) -> Result<(Link, Option), ExecutionError> where @@ -359,7 +404,7 @@ impl Data { let inner_data = unsafe { &mut *self.inner_data.get() }; inner_data[link.offset as usize..][..link.length as usize].copy_from_slice(bytes.as_slice()); - self.register_cell(link); + self.register_cell(link)?; Ok((link, link_left)) } @@ -431,20 +476,6 @@ impl Data { Ok(inner_data[link.offset as usize..(link.offset + link.length) as usize].to_vec()) } - /// Copies a wrapped row while clearing its runtime-only synchronization - /// byte in the copy. CDC and vacuum must never persist or publish an active - /// reader count into another cell. - pub(crate) fn get_raw_row_without_cell_state(&self, link: Link) -> Result, ExecutionError> - where - Row: Archive, - ::Archived: ArchivedRowWrapper, - { - let mut bytes = self.get_raw_row(link)?; - let state = Self::archived_cell_state_offset(bytes.as_mut_slice())?; - bytes[state] = 0; - Ok(bytes) - } - /// Moves data within the page from one location to another. /// Used for defragmentation - shifts data left to fill gaps. /// @@ -506,7 +537,7 @@ impl Data { offset, length, }; - self.register_cell(link); + self.register_cell(link)?; Ok(link) } @@ -516,38 +547,33 @@ impl Data { pub fn reset(&self) { self.free_offset.store(0, Ordering::Release); + self.cell_locks.reset(); self.live_cells.store(0, Ordering::Release); } - pub(crate) fn reset_cell_state(&self, link: Link) -> Result<(), ExecutionError> - where - Row: Archive, - ::Archived: ArchivedRowWrapper, - { - // Persisted pages can contain whatever synchronization byte happened - // to be present in the last in-memory image. A cold load has no live - // readers, so reset runtime state before publishing the table. - unsafe { &*self.cell_state_ptr(link)? }.store(0, Ordering::Release); - Ok(()) - } - - pub(crate) fn register_cell(&self, link: Link) { + pub(crate) fn register_cell(&self, link: Link) -> Result<(), ExecutionError> { debug_assert_eq!(link.page_id, self.id); self.live_cells .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1)) - .expect("live cell count overflow"); + .map(|_| ()) + .map_err(|_| ExecutionError::LiveCellCountOverflow) } - pub(crate) fn remove_cell(&self, link: Link) { + pub(crate) fn remove_cell(&self, link: Link) -> Result<(), ExecutionError> { debug_assert_eq!(link.page_id, self.id); self.live_cells .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_sub(1)) - .expect("removing a cell from an empty page"); + .map(|_| ()) + .map_err(|_| ExecutionError::LiveCellCountUnderflow) } pub(crate) fn has_live_cells(&self) -> bool { self.live_cells.load(Ordering::Acquire) != 0 } + + pub(crate) fn live_cell_count(&self) -> u32 { + self.live_cells.load(Ordering::Acquire) + } } /// Error that can appear on [`Data`] page operations. @@ -569,6 +595,12 @@ pub enum ExecutionError { /// Link provided for saving `Row` is invalid. InvalidLink, + + /// A page's live-cell count cannot represent another row. + LiveCellCountOverflow, + + /// A row was removed from a page whose live-cell count was already zero. + LiveCellCountUnderflow, } #[cfg(test)] diff --git a/src/in_memory/mod.rs b/src/in_memory/mod.rs index 81633576..227cd8c7 100644 --- a/src/in_memory/mod.rs +++ b/src/in_memory/mod.rs @@ -6,4 +6,4 @@ mod row; pub use data::{DATA_INNER_LENGTH, Data, ExecutionError as DataExecutionError}; pub use empty_link_registry::EmptyLinkRegistry; pub use pages::{DataPages, ExecutionError as PagesExecutionError, ReadGuard as DataPagesReadGuard}; -pub use row::{ArchivedRowWrapper, CellState, PublicationSafe, Query, RowWrapper, StorableRow}; +pub use row::{ArchivedRowWrapper, PublicationSafe, Query, RowWrapper, StorableRow}; diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index 7c1e3b3c..77e2ab74 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -111,9 +111,11 @@ impl PageDirectory { let mut chunks = self.chunks.lock(); chunk = root.load(Ordering::Acquire); if chunk.is_null() { - let mut owned = Box::new(PageDirectoryChunk::new()); - chunk = (&mut *owned) as *mut PageDirectoryChunk; - chunks.push(owned); + chunks.push(Box::new(PageDirectoryChunk::new())); + chunk = std::ptr::from_ref::>( + chunks.last().expect("the chunk was just appended").as_ref(), + ) + .cast_mut(); root.store(chunk, Ordering::Release); } } @@ -643,7 +645,9 @@ where DataExecutionError::PageIsFull { .. } | DataExecutionError::PageTooSmall { .. } | DataExecutionError::SerializeError - | DataExecutionError::DeserializeError => return Err(e.into()), + | DataExecutionError::DeserializeError + | DataExecutionError::LiveCellCountOverflow + | DataExecutionError::LiveCellCountUnderflow => return Err(e.into()), }, } } @@ -705,7 +709,9 @@ where DataExecutionError::PageTooSmall { .. } | DataExecutionError::SerializeError | DataExecutionError::DeserializeError - | DataExecutionError::InvalidLink => return Err(e.into()), + | DataExecutionError::InvalidLink + | DataExecutionError::LiveCellCountOverflow + | DataExecutionError::LiveCellCountUnderflow => return Err(e.into()), }, }; } @@ -738,6 +744,13 @@ where let mut next = (*pages).clone(); let page = Arc::new(Data::new(index.into())); next.push(page.clone()); + debug_assert_eq!(next.len(), pages.len() + 1); + debug_assert!( + next[..pages.len()] + .iter() + .zip(pages.iter()) + .all(|(new, old)| Arc::ptr_eq(new, old)) + ); self.pages.store(Arc::new(next)); self.publish_page(&page); self.current_page_id.store(index, Ordering::Release); @@ -772,6 +785,13 @@ where let pages = self.pages.load_full(); let mut next = (*pages).clone(); next.push(page.clone()); + debug_assert_eq!(next.len(), pages.len() + 1); + debug_assert!( + next[..pages.len()] + .iter() + .zip(pages.iter()) + .all(|(new, old)| Arc::ptr_eq(new, old)) + ); self.pages.store(Arc::new(next)); self.publish_page(&page); @@ -911,7 +931,7 @@ where let _cell_guard = page.write_cell(link).map_err(ExecutionError::DataPageError)?; let gen_row = ::WrappedRow::from_inner(row.clone()); let result = unsafe { - page.save_row_by_link_preserving_cell_state(&gen_row, link) + page.save_row_by_link(&gen_row, link) .map_err(ExecutionError::DataPageError) }?; Ok(result) @@ -958,7 +978,7 @@ where // `row` is consumed by the wrapper here (no clone): it is not used again. let gen_row = ::WrappedRow::from_inner(row); unsafe { - page.save_row_by_link_preserving_cell_state(&gen_row, link) + page.save_row_by_link(&gen_row, link) .map_err(ExecutionError::DataPageError)?; } // Clear the ghost bit on the stored row. A fresh `from_inner` wrapper @@ -1025,8 +1045,11 @@ where for link in links { match unsafe { self.with_mut_ref(*link, |r| r.delete()) } { Ok(()) => { - self.remove_cell(*link)?; ghosted += 1; + if let Err(error) = self.remove_cell(*link) { + failure = Some(error); + break; + } } Err(error) => { failure = Some(error); @@ -1050,8 +1073,7 @@ where pub fn select_raw(&self, link: Link) -> Result, ExecutionError> { let page = self.page_ref(link.page_id)?; let _cell_guard = page.read_cell(link).map_err(ExecutionError::DataPageError)?; - page.get_raw_row_without_cell_state(link) - .map_err(ExecutionError::DataPageError) + page.get_raw_row(link).map_err(ExecutionError::DataPageError) } pub fn mark_page_empty(&self, page_id: PageId) { @@ -1103,16 +1125,13 @@ where /// for a persisted table. pub fn register_cell(&self, link: Link) -> Result<(), ExecutionError> { let page = self.page_ref(link.page_id)?; - let _page_guard = page.access.read(); - page.reset_cell_state(link).map_err(ExecutionError::DataPageError)?; - page.register_cell(link); + page.register_cell(link).map_err(ExecutionError::DataPageError)?; Ok(()) } fn remove_cell(&self, link: Link) -> Result<(), ExecutionError> { let page = self.page_ref(link.page_id)?; - let _page_guard = page.access.read(); - page.remove_cell(link); + page.remove_cell(link).map_err(ExecutionError::DataPageError)?; Ok(()) } @@ -1121,6 +1140,19 @@ where Ok(page.has_live_cells()) } + pub(crate) fn page_live_cell_count(&self, page_id: PageId) -> Result { + let page = self.page_ref(page_id)?; + Ok(page.live_cell_count()) + } + + pub(crate) fn set_loaded_row_count(&self, count: usize) -> Result<(), ExecutionError> { + let count = u64::try_from(count).map_err(|_| ExecutionError::RowCountOverflow)?; + self.row_count.store(count, Ordering::Release); + Ok(()) + } + + /// Completes the vacuum's source-side accounting after every index has + /// been swung to the destination link. pub(crate) fn remove_moved_cell(&self, link: Link) -> Result<(), ExecutionError> { self.remove_cell(link) } @@ -1173,14 +1205,15 @@ where let _cell_guard = from_page.write_cell(from_link).map_err(ExecutionError::DataPageError)?; let raw_data = from_page - .get_raw_row_without_cell_state(from_link) + .get_raw_row(from_link) .map_err(ExecutionError::DataPageError)?; // Copy to the destination BEFORE flagging the source. The vacuumed // flag used to be set first, so a failing destination save returned // with the flag durably set in the source page image: after a restart // the row would load as vacuumed with no copy anywhere (row loss). - // The whole method holds both pages' write barriers, so the order is - // invisible to concurrent readers. + // The source cell guard keeps its bytes private, and destination bytes + // are unreachable until the caller publishes `new_link`, so readers + // cannot observe this intermediate state. let new_link = to_page.save_raw_row(&raw_data).map_err(ExecutionError::DataPageError)?; let archived = unsafe { from_page @@ -1279,6 +1312,9 @@ where pub enum ExecutionError { DataPageError(DataExecutionError), + #[display("row count exceeds u64")] + RowCountOverflow, + PageNotFound(#[error(not(source))] PageId), Locked, @@ -1309,7 +1345,6 @@ impl ExecutionError { #[cfg(test)] mod tests { - use super::{DELETED, GHOSTED, VACUUMED}; use std::collections::HashSet; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -1323,7 +1358,7 @@ mod tests { use crate::in_memory::data::Data; use crate::in_memory::pages::{DataPages, ExecutionError}; - use crate::in_memory::{CellState, DATA_INNER_LENGTH, PagesExecutionError, RowWrapper, StorableRow}; + use crate::in_memory::{DATA_INNER_LENGTH, PagesExecutionError, RowWrapper, StorableRow}; use crate::prelude::ArchivedRowWrapper; use data_bucket::Link; @@ -1336,14 +1371,15 @@ mod tests { /// General `Row` wrapper that is used to append general data for every `Inner` /// `Row`. #[derive(Archive, Deserialize, Debug, Serialize)] - #[rkyv(attr(repr(C)))] pub struct GeneralRow { /// Inner generic `Row`. pub inner: Inner, - pub publication_flags: u8, + pub is_ghosted: bool, + + pub is_deleted: bool, - pub cell_state: CellState, + pub is_in_vacuum_process: bool, } impl RowWrapper for GeneralRow { @@ -1352,23 +1388,24 @@ mod tests { } fn is_ghosted(&self) -> bool { - self.publication_flags & GHOSTED != 0 + self.is_ghosted } fn is_vacuumed(&self) -> bool { - self.publication_flags & VACUUMED != 0 + self.is_in_vacuum_process } fn is_deleted(&self) -> bool { - self.publication_flags & DELETED != 0 + self.is_deleted } /// Creates new [`GeneralRow`] from `Inner`. fn from_inner(inner: Inner) -> Self { Self { inner, - publication_flags: GHOSTED, - cell_state: CellState, + is_ghosted: true, + is_deleted: false, + is_in_vacuum_process: false, } } } @@ -1381,20 +1418,17 @@ mod tests { where T: Archive, { - unsafe fn cell_state_ptr(this: *mut Self) -> *mut std::sync::atomic::AtomicU8 { - unsafe { std::ptr::addr_of_mut!((*this).cell_state).cast() } - } fn unghost(&mut self) { - self.publication_flags &= !GHOSTED + self.is_ghosted = false } fn set_in_vacuum_process(&mut self) { - self.publication_flags |= VACUUMED + self.is_in_vacuum_process = true } fn delete(&mut self) { - self.publication_flags |= DELETED + self.is_deleted = true } fn is_deleted(&self) -> bool { - self.publication_flags & DELETED != 0 + self.is_deleted } } @@ -1910,7 +1944,7 @@ mod tests { // The source row must NOT be left flagged as in-vacuum-process: that // flag is written into the persisted page image, and with no copy on // the destination it would mean durable row loss after a restart. - let vacuumed = pages.with_ref(link, |r| r.publication_flags & VACUUMED != 0).unwrap(); + let vacuumed = pages.with_ref(link, |r| r.is_in_vacuum_process).unwrap(); assert!(!vacuumed, "failed move must not leave the source marked vacuumed"); assert_eq!(pages.select_non_vacuumed(link), Ok(row)); } diff --git a/src/in_memory/row.rs b/src/in_memory/row.rs index b730049a..5b38803b 100644 --- a/src/in_memory/row.rs +++ b/src/in_memory/row.rs @@ -1,38 +1,5 @@ +use rkyv::Archive; use std::fmt::Debug; -use std::sync::atomic::AtomicU8; - -use rkyv::rancor::Fallible; -use rkyv::{Archive, Deserialize, Place, Serialize}; - -/// Runtime synchronization state embedded as the first byte of every archived -/// cell wrapper. -/// -/// The source value is zero-sized; its archived representation is one byte. -/// Deserialization deliberately ignores that byte because active readers -/// modify it atomically. It is synchronization state, never row data. -#[derive(Clone, Copy, Debug, Default)] -pub struct CellState; - -impl Archive for CellState { - type Archived = u8; - type Resolver = (); - - fn resolve(&self, _: Self::Resolver, out: Place) { - out.write(0); - } -} - -impl Serialize for CellState { - fn serialize(&self, _: &mut S) -> Result { - Ok(()) - } -} - -impl Deserialize for u8 { - fn deserialize(&self, _: &mut D) -> Result { - Ok(CellState) - } -} pub trait PublicationSafe: Send + Sync + 'static {} @@ -54,14 +21,6 @@ pub trait RowWrapper { } pub trait ArchivedRowWrapper { - /// Returns the atomic synchronization byte for this archived cell without - /// first creating a reference to the rest of the row. Implementations must - /// place `cell_state` in a stable position in a `repr(C)` archived wrapper. - /// - /// # Safety - /// - /// `this` must point to a valid archived wrapper in writable page memory. - unsafe fn cell_state_ptr(this: *mut Self) -> *mut AtomicU8; fn unghost(&mut self); fn set_in_vacuum_process(&mut self); fn delete(&mut self); diff --git a/src/index/arctic.rs b/src/index/arctic.rs index 2e13c132..42336285 100644 --- a/src/index/arctic.rs +++ b/src/index/arctic.rs @@ -28,7 +28,7 @@ pub trait ArcticKey: Clone + Debug + Ord + Send + Sync + 'static { /// Arctic's variable-sized keys require a byte sequence with no zero byte so /// the tree can append its own logical terminator. Valid UTF-8 never contains /// `0xff`, therefore adding one to every encoded byte is a lossless, -/// order-preserving mapping into `1..=0xf5`. This includes Rust strings that +/// order-preserving mapping into `1..=0xf8`. This includes Rust strings that /// contain `\0`, without reserving a value or changing their ordering. pub type ArcticStringKey = arctic::key::BoxedSlice; @@ -85,6 +85,48 @@ macro_rules! impl_arctic_key { }; } +impl ArcticKey for u8 { + type Raw = u16; + + #[inline] + fn to_arctic(&self) -> Self::Raw { + u16::from(*self) + } + + #[inline] + fn from_arctic(value: Self::Raw) -> Self { + value as Self + } +} + +impl ArcticKey for usize { + type Raw = u64; + + #[inline] + fn to_arctic(&self) -> Self::Raw { + *self as u64 + } + + #[inline] + fn from_arctic(value: Self::Raw) -> Self { + value as Self + } +} + +impl ArcticKey for i8 { + type Raw = u16; + + #[inline] + fn to_arctic(&self) -> Self::Raw { + u16::from((*self as u8) ^ (1 << 7)) + } + + #[inline] + fn from_arctic(value: Self::Raw) -> Self { + ((value as u8) ^ (1 << 7)) as Self + } +} + impl_arctic_key!(u16, u32, u64, u128); /// Signed keys, mapped onto the unsigned key space by flipping the sign bit. @@ -99,7 +141,7 @@ impl_arctic_key!(u16, u32, u64, u128); /// Being a bijection over the *whole* width also preserves adjacency, which /// keeps excluded range bounds exact in the raw key space. /// -/// There is no `i8`, because Arctic's narrowest raw key is `u16`. +/// `i8` is widened losslessly into Arctic's narrowest raw key, `u16`. macro_rules! impl_arctic_signed_key { ($($ty:ty => $raw:ty),* $(,)?) => { $( @@ -141,9 +183,21 @@ impl ArcticValue for u64 { } } +#[doc(hidden)] +pub fn validate_arctic_link(link: data_bucket::Link) -> eyre::Result<()> { + if link.offset > u32::from(u16::MAX) || link.length > u32::from(u16::MAX) { + eyre::bail!( + "link cannot be represented by Arctic: page {:?}, offset {}, length {}", + link.page_id, + link.offset, + link.length, + ); + } + Ok(()) +} + fn pack_link(link: data_bucket::Link) -> u64 { - assert!(link.offset <= u16::MAX.into(), "link offset exceeds Arctic encoding"); - assert!(link.length <= u16::MAX.into(), "link length exceeds Arctic encoding"); + validate_arctic_link(link).expect("WorkTable validated the link before publishing it to Arctic"); (u64::from(u32::from(link.page_id)) << 32) | (u64::from(link.offset) << 16) | u64::from(link.length) } @@ -191,6 +245,21 @@ pub struct ArcticIndex { marker: PhantomData V>, } +/// Owned compatibility view returned by [`ArcticIndex::get`]. +/// +/// WTI exposes a guarded entry with a `get()` accessor. Arctic values are +/// decoded inline, so this view owns the decoded pair while preserving that +/// small inspection API for backend-agnostic diagnostics and tests. +pub struct ArcticEntry { + pair: indexset::core::pair::Pair, +} + +impl ArcticEntry { + pub fn get(&self) -> &indexset::core::pair::Pair { + &self.pair + } +} + impl Debug for ArcticIndex { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ArcticIndex") @@ -216,7 +285,6 @@ where impl ArcticIndex where K: ArcticKey, - K::Raw: arctic::topology::Key, V: ArcticValue, { /// Every entry, ascending. @@ -235,17 +303,47 @@ where >::iter_values(self) } + /// WTI-compatible point inspection for code that examines generated + /// index internals. Normal table reads use [`UniqueIndex::get_value`]. + pub fn get(&self, key: &K) -> Option> { + self.get_value(key).map(|value| ArcticEntry { + pair: indexset::core::pair::Pair { + key: key.clone(), + value, + }, + }) + } + + /// WTI-compatible ordered pair range. + pub fn range<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a + where + R: RangeBounds + 'a, + { + self.range_values(range) + } + + /// Heap bytes occupied by Arctic's reachable adaptive nodes. + pub fn allocated_node_bytes(&self) -> usize { + self.inner.allocated_node_bytes() + } + pub(crate) fn export_topology( &mut self, mut encode: impl FnMut(&V) -> T, - ) -> Result, arctic::topology::Error> { + ) -> Result, arctic::topology::Error> + where + K::Raw: arctic::topology::Key, + { self.inner.export_topology(|value| encode(&V::from_arctic(*value))) } pub(crate) fn from_topology( topology: arctic::topology::Topology, mut decode: impl FnMut(T) -> V, - ) -> Result { + ) -> Result + where + K::Raw: arctic::topology::Key, + { let inner = ConcurrentMap::from_topology(topology, |value| decode(value).into_arctic())?; let len = inner.all().entries(Order::Ascend).count(); Ok(Self { @@ -369,11 +467,8 @@ where .entries(Order::Ascend) .map(|(key, value)| (K::from_arctic(key), V::from_arctic(value))) .collect(), - } - .into_iter() - .filter(move |(key, _)| range.contains(key)) - .collect::>(); - values.into_iter() + }; + values.into_iter().filter(move |(key, _)| range.contains(key)) } fn range_links<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a diff --git a/src/index/arctic_multi.rs b/src/index/arctic_multi.rs index c0b9664e..744eba81 100644 --- a/src/index/arctic_multi.rs +++ b/src/index/arctic_multi.rs @@ -213,6 +213,20 @@ where self.len() == 0 } + /// Reachable Arctic nodes plus each boxed per-key slot and its link + /// vector allocation. Values' own nested allocations are counted by + /// WorkTable's `MemStat` implementation. + pub fn allocated_index_bytes(&self) -> usize { + let shard = self.inner.all(); + let mut entries = shard.entries(Order::Ascend); + let mut slots = 0; + while let Some((_, slot)) = entries.lend() { + let links = slot.read(); + slots += std::mem::size_of::>>() + links.links.capacity() * std::mem::size_of::(); + } + self.inner.allocated_node_bytes() + slots + } + /// Returns the pairs whose keys fall in `range`, ascending by key and in /// insertion order within a key, as a stable snapshot. pub fn range<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a diff --git a/src/index/congee.rs b/src/index/congee.rs index 4d18ac67..e564c754 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -89,6 +89,10 @@ where K: CongeeKey, V: Clone + Debug + Send + Sync + 'static, { + pub(crate) fn allocated_node_bytes(&self) -> usize { + self.inner.stats().total_memory_bytes() + } + /// Every entry, ascending. /// /// An inherent alias for [`UniqueIndex::iter_values`], so this reads the diff --git a/src/index/mod.rs b/src/index/mod.rs index 558cff14..c36970cd 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -11,7 +11,7 @@ mod table_secondary_index; mod unique; mod unsized_node; -pub use arctic::{ArcticIndex, ArcticKey, ArcticStringKey, ArcticValue}; +pub use arctic::{ArcticEntry, ArcticIndex, ArcticKey, ArcticStringKey, ArcticValue, validate_arctic_link}; pub use arctic_multi::ArcticMultiIndex; pub use available_index::AvailableIndex; pub use congee::{CongeeIndex, CongeeKey}; @@ -23,7 +23,9 @@ pub use persistent_art::{ }; pub use persistent_wti::PersistentWtiIndex; pub use primary_index::PrimaryIndex; -pub use table_index::{TableIndex, TableIndexCdc, convert_change_events, convert_upstream_change_events}; +pub use table_index::{ + TableIndex, TableIndexCdc, convert_change_events, convert_multi_change_events, convert_upstream_change_events, +}; pub use table_secondary_index::{ IndexError, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, }; diff --git a/src/index/primary_index.rs b/src/index/primary_index.rs index 36268131..d8883498 100644 --- a/src/index/primary_index.rs +++ b/src/index/primary_index.rs @@ -13,9 +13,9 @@ use crate::{IndexMap, TableIndex, TableIndexCdc, UniqueIndex}; /// Primary-key to physical-row mapping. /// -/// Vacuum enumerates a compact per-page cell directory and reads the primary -/// key already stored in each row. Keeping a second link-to-key index here -/// would duplicate every key solely for maintenance. +/// Vacuum groups a transient snapshot of this map by page. Keeping a second +/// link-to-key index here would duplicate every key for the table's lifetime +/// solely to accelerate an occasional maintenance pass. #[derive(Debug)] pub struct PrimaryIndex>> where diff --git a/src/index/table_index/mod.rs b/src/index/table_index/mod.rs index f5afeec0..a405b5d3 100644 --- a/src/index/table_index/mod.rs +++ b/src/index/table_index/mod.rs @@ -18,7 +18,7 @@ mod cdc; pub mod util; pub use cdc::TableIndexCdc; -pub use util::{convert_change_events, convert_upstream_change_events}; +pub use util::{convert_change_events, convert_multi_change_events, convert_upstream_change_events}; pub trait TableIndex { fn insert(&self, value: T, link: Link) -> Option; diff --git a/src/index/table_index/util.rs b/src/index/table_index/util.rs index c254ae81..4ffc57cb 100644 --- a/src/index/table_index/util.rs +++ b/src/index/table_index/util.rs @@ -1,4 +1,5 @@ use indexset::cdc::change::ChangeEvent; +use indexset::core::multipair::MultiPair; use indexset::core::pair::Pair; use vanilla_indexset::cdc::change::ChangeEvent as VanillaChangeEvent; use vanilla_indexset::core::pair::Pair as VanillaPair; @@ -78,6 +79,67 @@ where evs.into_iter().map(convert_change_event).collect() } +/// Converts the ordered pair representation used by WTI multimaps into the +/// key-only `Pair` representation retained by WorkTable's persistence format. +pub fn convert_multi_change_events(evs: Vec>>) -> Vec>> +where + L1: Into, +{ + fn pair(value: MultiPair) -> Pair + where + L1: Into, + { + Pair { + key: value.key, + value: value.value.into(), + } + } + + evs.into_iter() + .map(|event| match event { + ChangeEvent::InsertAt { + event_id, + max_value, + value, + index, + } => ChangeEvent::InsertAt { + event_id, + max_value: pair(max_value), + value: pair(value), + index, + }, + ChangeEvent::RemoveAt { + event_id, + max_value, + value, + index, + } => ChangeEvent::RemoveAt { + event_id, + max_value: pair(max_value), + value: pair(value), + index, + }, + ChangeEvent::CreateNode { event_id, max_value } => ChangeEvent::CreateNode { + event_id, + max_value: pair(max_value), + }, + ChangeEvent::RemoveNode { event_id, max_value } => ChangeEvent::RemoveNode { + event_id, + max_value: pair(max_value), + }, + ChangeEvent::SplitNode { + event_id, + max_value, + split_index, + } => ChangeEvent::SplitNode { + event_id, + max_value: pair(max_value), + split_index, + }, + }) + .collect() +} + /// Normalizes upstream IndexSet CDC events into WorkTablesIndex's event type, /// which remains the stable persistence boundary used by DataBucket. pub fn convert_upstream_change_events( diff --git a/src/lib.rs b/src/lib.rs index b8fb16f3..173b172f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,9 @@ mod util; pub mod features; pub use index::*; -pub use persistence::{LoadMode, PersistedWorkTable, PersistenceConfig, PersistenceLoadError, UnloadReport}; +pub use persistence::{ + LoadMode, PersistedWorkTable, PersistenceConfig, PersistenceLoadError, UnloadFailure, UnloadReport, +}; pub use row::*; pub use table::*; @@ -32,7 +34,7 @@ pub use worktable_dsl; pub use worktable_codegen::s3_sync_persistence; pub mod prelude { - pub use crate::in_memory::{ArchivedRowWrapper, CellState, Data, DataPages, Query, RowWrapper, StorableRow}; + pub use crate::in_memory::{ArchivedRowWrapper, Data, DataPages, Query, RowWrapper, StorableRow}; pub use crate::lock::FullRowLock; pub use crate::lock::{Lock, RowLock}; pub use crate::lock::{LockAcquirer, LockGuard, LockMap, PendingLock}; @@ -44,10 +46,10 @@ pub mod prelude { PersistenceEngine, PersistenceError, PersistenceIndexCorruption, PersistenceLoadError, PersistenceMonitor, PersistenceResult, PersistenceState, PersistenceTask, ReadOnlyPersistenceEngine, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, - SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, SpaceSecondaryIndexOps, - TocEntryOversizedError, UnloadReport, UpdateOperation, load_persisted_state, - map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, - validate_events, + SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, SpaceLogicalIndexUnsized, SpaceLogicalMultiIndex, + SpaceLogicalMultiIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, UnloadFailure, UnloadReport, + UpdateOperation, load_persisted_state, map_index_pages_to_toc_and_general, + map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, validate_events, }; pub use crate::primary_key::{ PrimaryKeyGenerator, PrimaryKeyGeneratorRange, PrimaryKeyGeneratorState, TablePrimaryKey, @@ -56,13 +58,13 @@ pub mod prelude { pub use crate::table::system_info::{IndexInfo, IndexKind, SystemInfo}; pub use crate::util::{OffsetEqLink, OrderedF32Def, OrderedF64Def}; pub use crate::{ - ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticStringKey, AvailableIndex, BatchDeleteError, BatchInsertError, - CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, + ArcticEntry, ArcticIndex, ArcticKey, ArcticMultiIndex, ArcticStringKey, AvailableIndex, BatchDeleteError, + BatchInsertError, CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, PersistentArcticIndex, PersistentArcticMultiIndex, PersistentArtIndex, PersistentCongeeIndex, PersistentWtiIndex, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, UpstreamIndexMap, UpstreamIndexPair, WorkTable, WorkTableError, vacuum::EmptyDataVacuum, - vacuum::VacuumPersistence, vacuum::WorkTableVacuum, + vacuum::VacuumPersistence, vacuum::WorkTableVacuum, validate_arctic_link, }; pub use data_bucket::{ DATA_VERSION, DataPage, GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, INNER_PAGE_SIZE, IndexPage, Interval, diff --git a/src/mem_stat/mod.rs b/src/mem_stat/mod.rs index 0b760647..8b45b3db 100644 --- a/src/mem_stat/mod.rs +++ b/src/mem_stat/mod.rs @@ -145,25 +145,28 @@ where impl MemStat for CongeeIndex where - K: CongeeKey, - V: Clone + Debug + Send + Sync + 'static, + K: CongeeKey + MemStat, + V: Clone + Debug + Send + Sync + MemStat + 'static, { fn heap_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + let values = self.iter_values().map(|(_, value)| value.heap_size()).sum::(); + self.allocated_node_bytes() + + self.len() * (std::mem::size_of::() + 2 * std::mem::size_of::()) + + values } fn used_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + self.heap_size() } } impl MemStat for ArcticIndex where - K: ArcticKey, - V: ArcticValue, + K: ArcticKey + MemStat, + V: ArcticValue + MemStat, { fn heap_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + self.allocated_node_bytes() + self.iter().map(|(_, value)| value.heap_size()).sum::() } fn used_size(&self) -> usize { @@ -173,11 +176,11 @@ where impl MemStat for ArcticMultiIndex where - K: ArcticKey, - V: Clone + Debug + PartialEq + Send + Sync + 'static, + K: ArcticKey + MemStat, + V: Clone + Debug + PartialEq + Send + Sync + MemStat + 'static, { fn heap_size(&self) -> usize { - self.len() * std::mem::size_of::<(K, V)>() + self.allocated_index_bytes() + self.iter().map(|(_, value)| value.heap_size()).sum::() } fn used_size(&self) -> usize { diff --git a/src/persistence/mod.rs b/src/persistence/mod.rs index 6a0d000e..b5e03ce3 100644 --- a/src/persistence/mod.rs +++ b/src/persistence/mod.rs @@ -18,18 +18,77 @@ pub use readonly_engine::ReadOnlyPersistenceEngine; pub use space::{ ArtPersistenceKey, IndexTableOfContents, SpaceArcticIndex, SpaceArcticMultiIndex, SpaceArcticStringIndex, SpaceCongeeIndex, SpaceData, SpaceDataOps, SpaceIndex, SpaceIndexOps, SpaceIndexUnsized, SpaceLogicalIndex, - SpaceLogicalIndexUnsized, SpaceSecondaryIndexOps, TocEntryOversizedError, map_index_pages_to_toc_and_general, - map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, + SpaceLogicalIndexUnsized, SpaceLogicalMultiIndex, SpaceLogicalMultiIndexUnsized, SpaceSecondaryIndexOps, + TocEntryOversizedError, map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general, + reconstruct_multi_index_nodes, }; pub use task::{PersistenceMonitor, PersistenceTask}; /// Result of retiring one Arc-owned persisted table generation. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct UnloadReport { - /// Memory attributed to the generation immediately before it was dropped. - pub released_bytes: usize, + /// Estimated live heap attributed to the generation when retirement was + /// requested. Successful unload proves the generation was destroyed; this + /// estimate is not an allocator RSS measurement. + pub estimated_released_bytes: usize, } +/// A generation could not be unloaded cleanly. +/// +/// Failures before shutdown return ownership of the generation so the caller +/// can keep serving it or retry. A failure returned by `close` has no retained +/// generation because shutdown was already attempted and consumed it. +pub struct UnloadFailure { + generation: Option>, + error: eyre::Report, +} + +impl UnloadFailure { + #[doc(hidden)] + pub fn retained(generation: std::sync::Arc, error: eyre::Report) -> Self { + Self { + generation: Some(generation), + error, + } + } + + #[doc(hidden)] + pub fn after_close(error: eyre::Report) -> Self { + Self { + generation: None, + error, + } + } + + /// Returns the still-live generation when shutdown never began. + pub fn into_generation(self) -> Option> { + self.generation + } + + /// The underlying unload error. + pub fn error(&self) -> &eyre::Report { + &self.error + } +} + +impl std::fmt::Debug for UnloadFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("UnloadFailure") + .field("generation_retained", &self.generation.is_some()) + .field("error", &self.error) + .finish() + } +} + +impl std::fmt::Display for UnloadFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.error.fmt(formatter) + } +} + +impl std::error::Error for UnloadFailure {} + mod engine; mod error; pub mod operation; diff --git a/src/persistence/operation/batch.rs b/src/persistence/operation/batch.rs index 9b376766..958bcfc0 100644 --- a/src/persistence/operation/batch.rs +++ b/src/persistence/operation/batch.rs @@ -44,10 +44,10 @@ worktable! ( pos: usize, }, indexes: { - operation_id_idx: operation_id, - page_id_idx: page_id, - link_idx: link, - op_type_idx: op_type, + operation_id_idx: operation_id using worktables_index, + page_id_idx: page_id using worktables_index, + link_idx: link using worktables_index, + op_type_idx: op_type using worktables_index, pos_idx: pos unique, }, queries: { diff --git a/src/persistence/space/art_index.rs b/src/persistence/space/art_index.rs index 08c29c39..d2db86c6 100644 --- a/src/persistence/space/art_index.rs +++ b/src/persistence/space/art_index.rs @@ -296,6 +296,12 @@ impl ArtFile { if crc32fast::hash(payload) != payload_crc { bail!("ART WAL frame at byte {position} checksum mismatch"); } + if K::WIDTH == 0 { + let key_len = u32::from_le_bytes(payload[9..13].try_into().unwrap()) as usize; + if 13usize.checked_add(key_len).and_then(|end| end.checked_add(12)) != Some(payload_len) { + bail!("ART WAL frame at byte {position} has inconsistent embedded key length {key_len}"); + } + } wal.push(decode_wal_record::(payload)?); position = payload_end; durable_end = payload_end; @@ -432,6 +438,9 @@ fn decode_wal_record(bytes: &[u8]) -> eyre::Result WalOp::Set(link), 2 => WalOp::Remove, @@ -441,6 +450,10 @@ fn decode_wal_record(bytes: &[u8]) -> eyre::Result eyre::Result<()> { + crate::validate_arctic_link(link) +} + fn logical_record(event: ChangeEvent>) -> eyre::Result> { match event { ChangeEvent::InsertAt { @@ -1052,11 +1065,13 @@ impl<'a> Decoder<'a> { let page_id = u32::from_le_bytes(self.take(4)?.try_into().unwrap()); let offset = u32::from_le_bytes(self.take(4)?.try_into().unwrap()); let length = u32::from_le_bytes(self.take(4)?.try_into().unwrap()); - Ok(Link { + let link = Link { page_id: PageId::from(page_id), offset, length, - }) + }; + validate_arctic_link(link)?; + Ok(link) } fn finish(self) -> eyre::Result<()> { diff --git a/src/persistence/space/index/mod.rs b/src/persistence/space/index/mod.rs index 060db15d..82394290 100644 --- a/src/persistence/space/index/mod.rs +++ b/src/persistence/space/index/mod.rs @@ -20,6 +20,7 @@ use data_bucket::{ use eyre::eyre; use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; +use indexset::concurrent::multimap::BTreeMultiMap; use indexset::core::pair::Pair; use rkyv::de::Pool; use rkyv::rancor::Strategy; @@ -332,6 +333,29 @@ where Ok(indexset) } + + /// Reconstructs a non-unique WTI index without changing its persisted + /// topology. Keeping the original node boundaries is required because + /// subsequent CDC events address pages by their current node maxima. + pub async fn parse_index_multimap(&mut self, index_name: &str) -> eyre::Result> { + let size = get_index_page_size_from_data_length::(INNER_PAGE_SIZE as usize); + let indexset = BTreeMultiMap::::with_maximum_node_size(size); + let mut pages = Vec::with_capacity(self.table_of_contents.iter().count()); + for ((key, link), page_id) in self.table_of_contents.iter() { + let page = parse_page::, INNER_PAGE_SIZE>(&mut self.index_file, (*page_id).into()).await?; + pages.push(( + Pair { + key: key.clone(), + value: *link, + }, + page.inner.get_node(), + )); + } + for node in reconstruct_multi_index_nodes(index_name, pages) { + indexset.attach_multi_node(node); + } + Ok(indexset) + } } impl SpaceIndexOps for SpaceIndex diff --git a/src/persistence/space/index/unsized_.rs b/src/persistence/space/index/unsized_.rs index 125a3ed0..8b815139 100644 --- a/src/persistence/space/index/unsized_.rs +++ b/src/persistence/space/index/unsized_.rs @@ -12,6 +12,7 @@ use data_bucket::{ use eyre::eyre; use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; +use indexset::concurrent::multimap::BTreeMultiMap; use indexset::core::pair::Pair; use rkyv::de::Pool; use rkyv::rancor::Strategy; @@ -26,7 +27,7 @@ use tokio::io::AsyncWriteExt; use super::page_aliases::PageAliases; use crate::UnsizedNode; use crate::persistence::space::BatchChangeEvent; -use crate::persistence::{IndexTableOfContents, SpaceIndex, SpaceIndexOps}; +use crate::persistence::{IndexTableOfContents, SpaceIndex, SpaceIndexOps, reconstruct_multi_index_nodes}; use crate::prelude::WT_INDEX_EXTENSION; #[derive(Debug)] @@ -341,6 +342,31 @@ where Ok(indexset) } + + /// Variable-key counterpart to `SpaceIndex::parse_index_multimap`. + pub async fn parse_index_multimap( + &mut self, + index_name: &str, + ) -> eyre::Result>>> { + let indexset = BTreeMultiMap::>::with_maximum_node_size(DATA_LENGTH as usize); + let mut pages = Vec::with_capacity(self.table_of_contents.iter().count()); + for ((key, link), page_id) in self.table_of_contents.iter() { + let page = + parse_page::, DATA_LENGTH>(&mut self.index_file, (*page_id).into()) + .await?; + pages.push(( + Pair { + key: key.clone(), + value: *link, + }, + page.inner.get_node(), + )); + } + for node in reconstruct_multi_index_nodes(index_name, pages) { + indexset.attach_multi_node(UnsizedNode::from_inner(node, DATA_LENGTH as usize)); + } + Ok(indexset) + } } impl SpaceIndexOps for SpaceIndexUnsized diff --git a/src/persistence/space/logical_index.rs b/src/persistence/space/logical_index.rs index 7f8fd860..e2e948fc 100644 --- a/src/persistence/space/logical_index.rs +++ b/src/persistence/space/logical_index.rs @@ -12,6 +12,8 @@ use std::path::{Path, PathBuf}; use data_bucket::{Link, SizeMeasurable, SpaceId, VariableSizeMeasurable}; use indexset::cdc::change::ChangeEvent; use indexset::concurrent::map::BTreeMap; +use indexset::concurrent::multimap::BTreeMultiMap; +use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; use rkyv::de::Pool; @@ -24,6 +26,7 @@ use rkyv::{Archive, Deserialize, Serialize, rancor}; use tokio::fs::File; use crate::UnsizedNode; +use crate::convert_multi_change_events; use crate::persistence::space::BatchChangeEvent; use crate::persistence::{PersistenceIndexCorruption, SpaceIndex, SpaceIndexOps, SpaceIndexUnsized}; use crate::prelude::WT_INDEX_EXTENSION; @@ -94,6 +97,64 @@ where Ok(structural) } +fn translate_logical_multi_event( + index_path: &Path, + shadow: &BTreeMultiMap, + event: ChangeEvent>, +) -> Result>>, PersistenceIndexCorruption> +where + T: Debug + Eq + Hash + Clone + Send + Ord + 'static, + Node: NodeLike> + Send + 'static, +{ + match event { + ChangeEvent::InsertAt { + max_value, + value, + index: 0, + .. + } if max_value.key == value.key && max_value.value == value.value => { + let (_, events) = shadow.insert_cdc(value.key, value.value); + Ok(convert_multi_change_events(events)) + } + ChangeEvent::RemoveAt { + max_value, + value, + index: 0, + .. + } if max_value.key == value.key && max_value.value == value.value => { + let (found, events) = shadow.remove_cdc(&value.key, &value.value); + if found.is_none() { + return Err(PersistenceIndexCorruption::new( + index_path, + format!("logical WTI multimap shadow diverged while removing pair {:?}", value,), + )); + } + Ok(convert_multi_change_events(events)) + } + _ => Err(PersistenceIndexCorruption::new( + index_path, + "logical WTI multimap persistence received a structural or malformed event", + )), + } +} + +fn translate_logical_multi_batch( + index_path: &Path, + shadow: &BTreeMultiMap, + mut events: BatchChangeEvent, +) -> Result, PersistenceIndexCorruption> +where + T: Debug + Eq + Hash + Clone + Send + Ord + 'static, + Node: NodeLike> + Send + 'static, +{ + events.sort_by_key(ChangeEvent::id); + let mut structural = Vec::new(); + for event in events { + structural.extend(translate_logical_multi_event(index_path, shadow, event)?); + } + Ok(structural) +} + /// Sized-key WTI persistence with foreground logical CDC and a background /// structural shadow. The wrapped `SpaceIndex` retains the existing file /// layout byte-for-byte. @@ -304,6 +365,216 @@ where } } +/// Sized-key WTI persistence for a non-unique runtime backend that emits +/// logical `(key, link)` mutations. The WTI file layout and its node topology +/// remain compatible with earlier WorkTable releases. +pub struct SpaceLogicalMultiIndex +where + T: Debug + Send + Ord + Eq + Clone + 'static, +{ + index_path: PathBuf, + shadow: BTreeMultiMap, + disk: SpaceIndex, +} + +impl Debug for SpaceLogicalMultiIndex +where + T: Debug + Send + Ord + Eq + Clone + 'static, +{ + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("SpaceLogicalMultiIndex").finish_non_exhaustive() + } +} + +impl SpaceLogicalMultiIndex +where + T: Archive + + Ord + + Eq + + Hash + + Clone + + Default + + Debug + + SizeMeasurable + + for<'a> Serialize, Share>, rancor::Error>> + + Send + + Sync + + 'static, + ::Archived: Deserialize> + + Ord + + Eq + + Debug + + for<'a> rkyv::bytecheck::CheckBytes>, +{ + async fn new(path: String, version: u32) -> eyre::Result { + let index_path = PathBuf::from(&path); + let mut disk = SpaceIndex::new(path, SpaceId::from(0), version).await?; + let shadow = disk.parse_index_multimap(index_path.to_string_lossy().as_ref()).await?; + Ok(Self { + index_path, + shadow, + disk, + }) + } +} + +impl SpaceIndexOps for SpaceLogicalMultiIndex +where + T: Archive + + Ord + + Eq + + Hash + + Clone + + Default + + Debug + + SizeMeasurable + + for<'a> Serialize, Share>, rancor::Error>> + + Send + + Sync + + 'static, + ::Archived: Deserialize> + + Ord + + Eq + + Debug + + for<'a> rkyv::bytecheck::CheckBytes>, +{ + async fn primary_from_table_files_path + Send>(path: S, version: u32) -> eyre::Result { + Self::new(format!("{}/primary{}", path.as_ref(), WT_INDEX_EXTENSION), version).await + } + + async fn secondary_from_table_files_path + Send, S2: AsRef + Send>( + path: S1, + name: S2, + version: u32, + ) -> eyre::Result { + Self::new( + format!("{}/{}{}", path.as_ref(), name.as_ref(), WT_INDEX_EXTENSION), + version, + ) + .await + } + + async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { + SpaceIndex::::bootstrap(file, table_name, version).await + } + + async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { + let events = translate_logical_multi_event(&self.index_path, &self.shadow, event)?; + self.disk.process_change_event_batch(events).await + } + + async fn process_change_event_batch(&mut self, events: BatchChangeEvent) -> eyre::Result<()> { + let events = translate_logical_multi_batch(&self.index_path, &self.shadow, events)?; + self.disk.process_change_event_batch(events).await + } +} + +/// Variable-sized-key counterpart to [`SpaceLogicalMultiIndex`]. +pub struct SpaceLogicalMultiIndexUnsized +where + T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, +{ + index_path: PathBuf, + shadow: BTreeMultiMap>>, + disk: SpaceIndexUnsized, +} + +impl Debug for SpaceLogicalMultiIndexUnsized +where + T: Send + Ord + Eq + Clone + Default + Debug + SizeMeasurable + VariableSizeMeasurable + 'static, +{ + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SpaceLogicalMultiIndexUnsized") + .finish_non_exhaustive() + } +} + +impl SpaceLogicalMultiIndexUnsized +where + T: Archive + + Ord + + Eq + + Hash + + Clone + + Default + + Debug + + SizeMeasurable + + VariableSizeMeasurable + + for<'a> Serialize, Share>, rancor::Error>> + + Send + + Sync + + 'static, + ::Archived: Deserialize> + + Ord + + Eq + + Debug + + for<'a> rkyv::bytecheck::CheckBytes>, +{ + async fn new(path: String, version: u32) -> eyre::Result { + let index_path = PathBuf::from(&path); + let mut disk = SpaceIndexUnsized::new(path, SpaceId::from(0), version).await?; + let shadow = disk.parse_index_multimap(index_path.to_string_lossy().as_ref()).await?; + Ok(Self { + index_path, + shadow, + disk, + }) + } +} + +impl SpaceIndexOps for SpaceLogicalMultiIndexUnsized +where + T: Archive + + Ord + + Eq + + Hash + + Clone + + Default + + Debug + + SizeMeasurable + + VariableSizeMeasurable + + for<'a> Serialize, Share>, rancor::Error>> + + Send + + Sync + + 'static, + ::Archived: Deserialize> + + Ord + + Eq + + Debug + + for<'a> rkyv::bytecheck::CheckBytes>, +{ + async fn primary_from_table_files_path + Send>(path: S, version: u32) -> eyre::Result { + Self::new(format!("{}/primary{}", path.as_ref(), WT_INDEX_EXTENSION), version).await + } + + async fn secondary_from_table_files_path + Send, S2: AsRef + Send>( + path: S1, + name: S2, + version: u32, + ) -> eyre::Result { + Self::new( + format!("{}/{}{}", path.as_ref(), name.as_ref(), WT_INDEX_EXTENSION), + version, + ) + .await + } + + async fn bootstrap(file: &mut File, table_name: String, version: u32) -> eyre::Result<()> { + SpaceIndexUnsized::::bootstrap(file, table_name, version).await + } + + async fn process_change_event(&mut self, event: ChangeEvent>) -> eyre::Result<()> { + let events = translate_logical_multi_event(&self.index_path, &self.shadow, event)?; + self.disk.process_change_event_batch(events).await + } + + async fn process_change_event_batch(&mut self, events: BatchChangeEvent) -> eyre::Result<()> { + let events = translate_logical_multi_batch(&self.index_path, &self.shadow, events)?; + self.disk.process_change_event_batch(events).await + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap as StdBTreeMap; @@ -425,6 +696,61 @@ mod tests { assert_eq!(shadow.get(&7).map(|entry| entry.get().value), Some(link(8))); } + #[test] + fn logical_multi_batch_preserves_each_pair_and_event_order() { + let shadow = BTreeMultiMap::::default(); + let first = Pair { key: 7, value: link(1) }; + let second = Pair { key: 7, value: link(2) }; + let remove_first = ChangeEvent::RemoveAt { + event_id: 2.into(), + max_value: first.clone(), + value: first.clone(), + index: 0, + }; + let insert_second = ChangeEvent::InsertAt { + event_id: 1.into(), + max_value: second.clone(), + value: second, + index: 0, + }; + let insert_first = ChangeEvent::InsertAt { + event_id: 0.into(), + max_value: first.clone(), + value: first, + index: 0, + }; + + let structural = translate_logical_multi_batch( + Path::new("multi.wt.idx"), + &shadow, + vec![remove_first, insert_second, insert_first], + ) + .unwrap(); + + assert!(!structural.is_empty()); + assert_eq!(shadow.iter().collect::>(), vec![(7, link(2))]); + } + + #[test] + fn logical_multi_remove_detects_a_diverged_shadow() { + let shadow = BTreeMultiMap::::default(); + let pair = Pair { key: 7, value: link(9) }; + let error = translate_logical_multi_event( + Path::new("multi.wt.idx"), + &shadow, + ChangeEvent::RemoveAt { + event_id: 0.into(), + max_value: pair.clone(), + value: pair, + index: 0, + }, + ) + .unwrap_err(); + + assert_eq!(error.path(), Path::new("multi.wt.idx")); + assert!(error.reason().contains("shadow diverged")); + } + #[test] fn shuffled_large_logical_batch_replays_in_event_id_order() { let shadow = BTreeMap::::default(); diff --git a/src/persistence/space/mod.rs b/src/persistence/space/mod.rs index 34b22b39..4f9d280b 100644 --- a/src/persistence/space/mod.rs +++ b/src/persistence/space/mod.rs @@ -21,7 +21,9 @@ pub use index::{ IndexTableOfContents, SpaceIndex, SpaceIndexUnsized, TocEntryOversizedError, map_index_pages_to_toc_and_general, map_unsized_index_pages_to_toc_and_general, reconstruct_multi_index_nodes, }; -pub use logical_index::{SpaceLogicalIndex, SpaceLogicalIndexUnsized}; +pub use logical_index::{ + SpaceLogicalIndex, SpaceLogicalIndexUnsized, SpaceLogicalMultiIndex, SpaceLogicalMultiIndexUnsized, +}; pub type BatchData = HashMap)>>; diff --git a/src/persistence/task.rs b/src/persistence/task.rs index be444ba7..fc67f09f 100644 --- a/src/persistence/task.rs +++ b/src/persistence/task.rs @@ -30,9 +30,9 @@ worktable! ( pos: usize, }, indexes: { - operation_id_idx: operation_id, - page_id_idx: page_id, - link_idx: link, + operation_id_idx: operation_id using worktables_index, + page_id_idx: page_id using worktables_index, + link_idx: link using worktables_index, }, ); diff --git a/src/table/mod.rs b/src/table/mod.rs index de1e6a64..eeba4d01 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -146,6 +146,7 @@ where { let path = path.as_ref(); let mut links = HashSet::with_capacity(self.primary_index.pk_map.len()); + let mut cells_by_page = std::collections::HashMap::::new(); for (primary_key, offset_link) in self.primary_index.pk_map.iter_values() { if !links.insert(offset_link) { @@ -173,8 +174,27 @@ where format!("primary key {primary_key:?} has an invalid cell: {error}"), ) })?; + let page_count = cells_by_page.entry(offset_link.0.page_id).or_default(); + *page_count = page_count + .checked_add(1) + .ok_or_else(|| PersistenceLoadError::corrupt(path, "live-cell count exceeds u32"))?; } + for (page_id, expected) in cells_by_page { + let actual = self.data.page_live_cell_count(page_id).map_err(|error| { + PersistenceLoadError::corrupt(path, format!("cannot audit page {page_id:?}: {error}")) + })?; + if actual != expected { + return Err(PersistenceLoadError::corrupt( + path, + format!("page {page_id:?} has {actual} live cells; primary index references {expected}"), + )); + } + } + self.data + .set_loaded_row_count(links.len()) + .map_err(|error| PersistenceLoadError::corrupt(path, format!("cannot restore row count: {error}")))?; + Ok(()) } diff --git a/src/table/vacuum/manager.rs b/src/table/vacuum/manager.rs index 24bf092c..566ed452 100644 --- a/src/table/vacuum/manager.rs +++ b/src/table/vacuum/manager.rs @@ -7,7 +7,7 @@ use tokio::task::AbortHandle; use parking_lot::RwLock; use smart_default::SmartDefault; -use crate::vacuum::WorkTableVacuum; +use crate::vacuum::{VacuumDiagnosticsSnapshot, WorkTableVacuum}; /// How long a sweep waits before checking a table nothing has woken it about. /// @@ -108,6 +108,21 @@ impl VacuumManager { id } + /// Aggregate live activity from every registered table vacuum. + pub fn diagnostic_snapshot(&self) -> VacuumDiagnosticsSnapshot { + self.vacuums.read().values().map(|vacuum| vacuum.diagnostics()).fold( + VacuumDiagnosticsSnapshot::default(), + |mut total, item| { + total.requests += item.requests; + total.work_batches += item.work_batches; + total.pages_examined += item.pages_examined; + total.pages_reclaimed += item.pages_reclaimed; + total.completions += item.completions; + total + }, + ) + } + /// Starts a background task that periodically checks fragmentation and runs /// vacuum. /// @@ -183,7 +198,7 @@ impl VacuumManager { final_consolidation_ran = true; } { - log::debug!("Vacuuming {}", info.table_name); + log::debug!("vacuum requested for {}; waiting for a quiet window", info.table_name); match vacuum.vacuum().await { Ok(stats) => { self.stats.sweeps.fetch_add(1, Ordering::Relaxed); diff --git a/src/table/vacuum/mod.rs b/src/table/vacuum/mod.rs index a7163165..74fc55db 100644 --- a/src/table/vacuum/mod.rs +++ b/src/table/vacuum/mod.rs @@ -71,6 +71,21 @@ pub trait WorkTableVacuum { /// the thing that got fragmented. Callers still want a fallback interval /// alongside this, for a table whose threshold is never reached. async fn wait_until_worth_running(&self); + + /// Live cumulative counters, including work that has started but has not + /// completed a sweep yet. + fn diagnostics(&self) -> VacuumDiagnosticsSnapshot; +} + +/// Cumulative vacuum activity. Unlike manager sweep counts, these counters +/// expose partial work while a sweep is still waiting or running. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct VacuumDiagnosticsSnapshot { + pub requests: u64, + pub work_batches: u64, + pub pages_examined: u64, + pub pages_reclaimed: u64, + pub completions: u64, } /// Represents vacuum statistics after a vacuum operation diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index 77212485..47030bca 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -2,6 +2,7 @@ use std::collections::VecDeque; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; /// How long retirements have to stop arriving for a delete burst to count as @@ -26,9 +27,9 @@ use crate::in_memory::{ArchivedRowWrapper, DataPages, RowWrapper, StorableRow}; use crate::lock::{Lock, LockGuard, LockMap, RowLock}; use crate::prelude::{OffsetEqLink, TablePrimaryKey}; use crate::vacuum::VacuumPersistence; -use crate::vacuum::VacuumStats; use crate::vacuum::WorkTableVacuum; use crate::vacuum::fragmentation_info::FragmentationInfo; +use crate::vacuum::{VacuumDiagnosticsSnapshot, VacuumStats}; use crate::vacuum::{VacuumGate, VacuumPacing}; use crate::{ AvailableIndex, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, @@ -51,6 +52,27 @@ enum CandidateMove { Failed, } +#[derive(Debug, Default)] +struct VacuumDiagnostics { + requests: AtomicU64, + work_batches: AtomicU64, + pages_examined: AtomicU64, + pages_reclaimed: AtomicU64, + completions: AtomicU64, +} + +impl VacuumDiagnostics { + fn snapshot(&self) -> VacuumDiagnosticsSnapshot { + VacuumDiagnosticsSnapshot { + requests: self.requests.load(Ordering::Relaxed), + work_batches: self.work_batches.load(Ordering::Relaxed), + pages_examined: self.pages_examined.load(Ordering::Relaxed), + pages_reclaimed: self.pages_reclaimed.load(Ordering::Relaxed), + completions: self.completions.load(Ordering::Relaxed), + } + } +} + #[derive(derive_more::Debug)] pub struct EmptyDataVacuum< Row, @@ -88,6 +110,8 @@ pub struct EmptyDataVacuum< /// The bit a caller flips to hold the sweep off. gate: Arc, + diagnostics: VacuumDiagnostics, + phantom_data: PhantomData<(SecondaryEvents, AvailableTypes, AvailableIndexes)>, } @@ -149,6 +173,7 @@ where persistence: None, pacing: VacuumPacing::default(), gate: Arc::new(VacuumGate::default()), + diagnostics: VacuumDiagnostics::default(), phantom_data: PhantomData, } } @@ -208,6 +233,7 @@ where } async fn defragment(&self) -> eyre::Result { + self.diagnostics.requests.fetch_add(1, Ordering::Relaxed); // The first batch needs the same permission as every later batch. If // this check lives only at the batch boundary, a sweep woken during a // sustained mutation stream still moves `batch_pages` sources before @@ -220,6 +246,8 @@ where if self.pacing.batch_pages > 0 { self.pacing.wait_until_quiet(&*self.lock_manager, &self.gate).await; } + self.diagnostics.work_batches.fetch_add(1, Ordering::Relaxed); + log::debug!("vacuum work starting for {}", self.table_name); let now = Instant::now(); @@ -291,6 +319,7 @@ where let mut pages_since_yield = 0usize; for info in info_iter { + self.diagnostics.pages_examined.fetch_add(1, Ordering::Relaxed); pages_since_yield += 1; if self.pacing.batch_pages > 0 && pages_since_yield > self.pacing.batch_pages { pages_since_yield = 1; @@ -302,6 +331,8 @@ where // idle and the sweep went in on top of the workload. drop(registry_lock.take()); self.pacing.wait_until_quiet(&*self.lock_manager, &self.gate).await; + self.diagnostics.work_batches.fetch_add(1, Ordering::Relaxed); + log::debug!("vacuum work resuming for {} after a quiet recheck", self.table_name); registry_lock = Some(registry.lock_vacuum().await); } @@ -403,6 +434,7 @@ where } } self.data_pages.mark_page_empty(page_from); + self.diagnostics.pages_reclaimed.fetch_add(1, Ordering::Relaxed); pages_freed += 1; } @@ -412,6 +444,7 @@ where self.finalize_staged_pages(free_pages, defragmented_pages)?; drop(registry_lock); + self.diagnostics.completions.fetch_add(1, Ordering::Relaxed); Ok(VacuumStats { pages_processed, pages_freed, @@ -696,6 +729,10 @@ where self.data_pages.empty_links_registry().wait_for_fragmentation().await; self.settle_after_wake().await; } + + fn diagnostics(&self) -> VacuumDiagnosticsSnapshot { + self.diagnostics.snapshot() + } } #[cfg(test)] @@ -1094,7 +1131,7 @@ mod tests { ) -> EmptyDataVacuum< TestRow, TestPrimaryKey, - IndexMap>, + ArcticIndex>, TestIndex, TestAvaiableTypes, TestAvailableIndexes, diff --git a/src/util/epoch.rs b/src/util/epoch.rs index 03456fb9..74360f37 100644 --- a/src/util/epoch.rs +++ b/src/util/epoch.rs @@ -29,13 +29,10 @@ //! WorkTable's Arctic adapter now also selects `ps-reclaim`; Arctic supports //! other SMRs for its general users, but WorkTable keeps one progress model. //! -//! # crossbeam is still in the tree -//! -//! This removes it from WorkTable's own reclamation, not from the build. It -//! still arrives three ways: `congee-wt` depends on `crossbeam-epoch` directly -//! and re-exports its `Guard`, which `src/index/congee.rs` names; and -//! `crossbeam-skiplist` comes in under both `WorkTablesIndex` and `indexset`. -//! Say "no crossbeam in the reclamation path", not "no crossbeam", until -//! `congee-wt` is ported. See `docs/TODO.md`. +//! # Crossbeam benchmark comparison +//! +//! WorkTablesIndex 0.0.11 removes Crossbeam from its runtime topology. Its +//! development benchmark keeps `crossbeam-skiplist` only as a comparison; it +//! is not in WorkTable's runtime dependency graph. pub(crate) use ps_reclaim::{Domain as EpochDomain, Guard}; diff --git a/tests/generation_swap_requirement.rs b/tests/generation_swap_requirement.rs index e1d66c70..fc6968ab 100644 --- a/tests/generation_swap_requirement.rs +++ b/tests/generation_swap_requirement.rs @@ -118,8 +118,11 @@ async fn a_retired_generation_releases_its_memory() { }) .await .expect("the generation retires"); - assert_eq!(report.released_bytes, held); - assert!(report.released_bytes > 0, "the memory came back"); + assert_eq!(report.estimated_released_bytes, held); + assert!( + report.estimated_released_bytes > 0, + "the generation had memory to release" + ); let _ = std::fs::remove_dir_all(DIR); } @@ -138,6 +141,38 @@ async fn a_generation_can_report_what_it_holds() { let _ = std::fs::remove_dir_all(DIR); } +#[tokio::test] +async fn a_failed_unload_returns_the_live_generation() { + let dir = "tests/data/generation_swap/retained_failure"; + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir).expect("a directory"); + + let generation = Arc::new(attach(dir).await); + generation + .insert(GenerationSwapRow { + id: generation.get_next_pk().into(), + blob: "still serving".to_owned(), + }) + .await + .expect("a row"); + generation.wait_for_ops().await.expect("the queue drains"); + + let outstanding_reader = Arc::clone(&generation); + let failure = generation + .unload_gracefully(DRAIN_TIMEOUT, || async {}) + .await + .expect_err("an outstanding Arc lease prevents unload"); + let generation = failure + .into_generation() + .expect("failure before close retains the generation"); + assert_eq!(generation.select_all().execute().expect("a read").len(), 1); + + drop(outstanding_reader); + let generation = Arc::try_unwrap(generation).expect("the caller recovered sole ownership"); + generation.close().await.expect("the recovered generation closes"); + let _ = std::fs::remove_dir_all(dir); +} + /// **Already works.** Kept so nobody spends a day building it: two handles on /// one on-disk generation coexist, and the outgoing one keeps answering. #[tokio::test] diff --git a/tests/persistence/sync/option.rs b/tests/persistence/sync/option.rs index 0a164804..98e8031a 100644 --- a/tests/persistence/sync/option.rs +++ b/tests/persistence/sync/option.rs @@ -464,7 +464,7 @@ worktable! ( }, indexes: { another_idx: another unique, - test_idx: test, + test_idx: test using worktables_index, exchnage_idx: exchange, }, queries: { diff --git a/tests/persistence/sync/uuid_.rs b/tests/persistence/sync/uuid_.rs index 2100ae0c..28708029 100644 --- a/tests/persistence/sync/uuid_.rs +++ b/tests/persistence/sync/uuid_.rs @@ -14,8 +14,8 @@ worktable!( second: Uuid, }, indexes: { - first_idx: first, - second_idx: second unique, + first_idx: first using worktables_index, + second_idx: second unique using worktables_index, }, ); diff --git a/tests/persistence/tuple_primary_key.rs b/tests/persistence/tuple_primary_key.rs index fb7f4944..2f31bc1e 100644 --- a/tests/persistence/tuple_primary_key.rs +++ b/tests/persistence/tuple_primary_key.rs @@ -7,8 +7,8 @@ worktable! ( name: PersistedTuplePrimaryKey, persist: true, columns: { - tenant_id: u64 primary_key, - record_id: u64 primary_key, + tenant_id: u64 primary_key using worktables_index, + record_id: u64 primary_key using worktables_index, value: i64, }, ); diff --git a/tests/worktable/borrowed_primary_key.rs b/tests/worktable/borrowed_primary_key.rs index 7babb52b..aef33b0f 100644 --- a/tests/worktable/borrowed_primary_key.rs +++ b/tests/worktable/borrowed_primary_key.rs @@ -20,8 +20,8 @@ worktable!( worktable!( name: BorrowedTupleKey, columns: { - tenant: String primary_key, - record: String primary_key, + tenant: String primary_key using worktables_index, + record: String primary_key using worktables_index, value: u64, }, ); diff --git a/tests/worktable/custom_pk.rs b/tests/worktable/custom_pk.rs index 4ebe1348..4dd4457e 100644 --- a/tests/worktable/custom_pk.rs +++ b/tests/worktable/custom_pk.rs @@ -45,7 +45,7 @@ impl TablePrimaryKey for TestPrimaryKey { worktable! ( name: Test, columns: { - id: CustomId primary_key custom, + id: CustomId primary_key custom using worktables_index, test: u64 } ); diff --git a/tests/worktable/float.rs b/tests/worktable/float.rs index 1c054f66..8de0a366 100644 --- a/tests/worktable/float.rs +++ b/tests/worktable/float.rs @@ -14,9 +14,9 @@ worktable! ( exchange: String }, indexes: { - test_idx: test unique, - exchnage_idx: exchange, - another_idx: another + test_idx: test unique using worktables_index, + exchnage_idx: exchange using worktables_index, + another_idx: another using worktables_index } ); @@ -28,7 +28,7 @@ worktable! ( value: f64, }, indexes: { - value_idx: value unique, + value_idx: value unique using worktables_index, } ); diff --git a/tests/worktable/key_widths.rs b/tests/worktable/key_widths.rs index 37ab5f35..f0429fcb 100644 --- a/tests/worktable/key_widths.rs +++ b/tests/worktable/key_widths.rs @@ -3,7 +3,8 @@ //! `validate_index_backends` rejects a declaration whose key type the backend //! cannot serve, and the accepted sets are narrow and specific: congee takes //! `u8`, `u16`, `u32`, `u64`, `usize`; arctic takes `u16`, `u32`, `u64`, -//! `u128`. That list is a promise the macro makes to a consumer. +//! `u128`, including the narrow values widened losslessly by its adapter. That +//! list is a promise the macro makes to a consumer. //! //! Nothing tested it. Before this file the suite instantiated `u64` with both //! backends and `u128` once, so most of the advertised matrix had never been @@ -85,11 +86,15 @@ width_case!(congee_u32, congee, u32, "congee"); width_case!(congee_u64, congee, u64, "congee"); width_case!(congee_usize, congee, usize, "congee"); -// Arctic: u16, u32, u64, u128. +// Arctic: narrow, native-width, and signed adapters. +width_case!(arctic_u8, arctic, u8, "arctic"); width_case!(arctic_u16, arctic, u16, "arctic"); width_case!(arctic_u32, arctic, u32, "arctic"); width_case!(arctic_u64, arctic, u64, "arctic"); width_case!(arctic_u128, arctic, u128, "arctic"); +width_case!(arctic_i8, arctic, i8, "arctic"); +width_case!(arctic_i64, arctic, i64, "arctic"); +width_case!(arctic_usize, arctic, usize, "arctic"); // The default backend takes any ordered key, so it is the control: if a width // fails here too, the problem is not the backend. diff --git a/tests/worktable/nid.rs b/tests/worktable/nid.rs index e42cef5b..54d13607 100644 --- a/tests/worktable/nid.rs +++ b/tests/worktable/nid.rs @@ -9,7 +9,7 @@ type PackedNanoid21 = packed_nanoid_type!(21, Base64UrlAlphabet); worktable!( name: Test, columns: { - id: PackedNanoid21 primary_key, + id: PackedNanoid21 primary_key using worktables_index, another: i64, } ); diff --git a/tests/worktable/option.rs b/tests/worktable/option.rs index 47999a60..7c82185d 100644 --- a/tests/worktable/option.rs +++ b/tests/worktable/option.rs @@ -319,7 +319,7 @@ worktable! ( }, indexes: { another_idx: another unique, - test_idx: test, + test_idx: test using worktables_index, exchnage_idx: exchange, }, queries: { diff --git a/tests/worktable/tuple_primary_key.rs b/tests/worktable/tuple_primary_key.rs index 7c9eec39..35894164 100644 --- a/tests/worktable/tuple_primary_key.rs +++ b/tests/worktable/tuple_primary_key.rs @@ -4,8 +4,8 @@ use worktable::worktable; worktable! ( name: Test, columns: { - id: u64 primary_key, - test: u64 primary_key, + id: u64 primary_key using worktables_index, + test: u64 primary_key using worktables_index, another: i64, } ); diff --git a/tests/worktable/uuid.rs b/tests/worktable/uuid.rs index e29cb5c9..92bc906d 100644 --- a/tests/worktable/uuid.rs +++ b/tests/worktable/uuid.rs @@ -5,7 +5,7 @@ use worktable::worktable; worktable! ( name: Test, columns: { - id: Uuid primary_key, + id: Uuid primary_key using worktables_index, another: i64, } ); From 151f162be1e8530df4deaa09f5bea20be01cb2b6 Mon Sep 17 00:00:00 2001 From: meh Date: Sat, 5 Sep 2026 09:13:46 +0700 Subject: [PATCH 5/5] Preserve primary key traits with Arctic defaults --- codegen/src/generators/index_backend.rs | 7 +-- codegen/src/persist_table/generator/space.rs | 51 +++++++++----------- codegen/src/worktable/mod.rs | 4 +- src/in_memory/pages.rs | 3 ++ 4 files changed, 28 insertions(+), 37 deletions(-) diff --git a/codegen/src/generators/index_backend.rs b/codegen/src/generators/index_backend.rs index edb61fc2..b56f6f26 100644 --- a/codegen/src/generators/index_backend.rs +++ b/codegen/src/generators/index_backend.rs @@ -108,13 +108,8 @@ pub(crate) fn primary_key_backend_impl( } IndexBackend::Arctic => { let field = single_supported_field(backend, fields, supported_types(backend))?; - let derive = if primitive_name(field).as_deref() == Some("String") { - quote! {} - } else { - quote! { Copy, } - }; Ok(( - derive, + quote! {}, quote! { impl ArcticKey for #primary_key { type Raw = <#field as ArcticKey>::Raw; diff --git a/codegen/src/persist_table/generator/space.rs b/codegen/src/persist_table/generator/space.rs index 01ed6376..a48ad1ff 100644 --- a/codegen/src/persist_table/generator/space.rs +++ b/codegen/src/persist_table/generator/space.rs @@ -36,35 +36,28 @@ impl Generator { let space_secondary_indexes = name_generator.get_space_secondary_index_ident(); let space_secondary_indexes_events = name_generator.get_space_secondary_index_events_ident(); let avt_index_ident = name_generator.get_available_indexes_ident(); - let space_index_type = if self.attributes.pk_arctic_string { - quote! { - SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }>, - } - } else if self.attributes.pk_unsized && self.attributes.pk_wti_logical { - quote! { - SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }>, - } - } else if self.attributes.pk_unsized { - quote! { - SpaceIndexUnsized<#primary_key_type, { #inner_const_name as u32 }>, - } - } else if self.attributes.pk_wti_logical { - quote! { - SpaceLogicalIndex<#primary_key_type, { #inner_const_name as u32 }>, - } - } else if self.attributes.pk_arctic { - quote! { - SpaceLogicalIndex<#primary_key_type, { #inner_const_name as u32 }>, - } - } else if self.attributes.pk_congee { - quote! { - SpaceCongeeIndex<#primary_key_type, { #inner_const_name as u32 }>, - } - } else { - quote! { - SpaceIndex<#primary_key_type, { #inner_const_name as u32 }>, - } - }; + let space_index_type = + if self.attributes.pk_arctic_string || (self.attributes.pk_unsized && self.attributes.pk_wti_logical) { + quote! { + SpaceLogicalIndexUnsized<#primary_key_type, { #inner_const_name as u32 }>, + } + } else if self.attributes.pk_unsized { + quote! { + SpaceIndexUnsized<#primary_key_type, { #inner_const_name as u32 }>, + } + } else if self.attributes.pk_wti_logical || self.attributes.pk_arctic { + quote! { + SpaceLogicalIndex<#primary_key_type, { #inner_const_name as u32 }>, + } + } else if self.attributes.pk_congee { + quote! { + SpaceCongeeIndex<#primary_key_type, { #inner_const_name as u32 }>, + } + } else { + quote! { + SpaceIndex<#primary_key_type, { #inner_const_name as u32 }>, + } + }; quote! { pub type #ident = DiskPersistenceEngine< diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index fe5e3276..b7820ef8 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -250,7 +250,7 @@ mod tests { #[cfg(feature = "logical-index-persistence")] #[test] - fn logical_persistence_wraps_only_default_wti_backends() { + fn logical_persistence_wraps_explicit_wti_backends() { let output = expand(quote! { name: LogicalDefaultBackend, persist: true, @@ -261,7 +261,7 @@ mod tests { arctic_value: u64, }, indexes: { - wti_idx: wti_value unique, + wti_idx: wti_value unique using worktables_index, congee_idx: congee_value unique using congee, arctic_idx: arctic_value unique using arctic, }, diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index 77e2ab74..f118dff5 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -86,6 +86,9 @@ impl PageDirectoryChunk { #[derive(Debug)] struct PageDirectory { roots: [AtomicPtr>; PAGE_DIRECTORY_ROOTS], + // Each chunk needs a stable address after the vector grows because + // `roots` publishes pointers to the chunk allocations. + #[allow(clippy::vec_box)] chunks: Mutex>>>, }