Complete beta18 reclamation, memory, vacuum, and backend work

CHANGELOG.md

@@ -1,6 +1,46 @@
 Change Log
 ==========
 
+## [1.0.0-beta.18]
+
+### Added
+
+- Selectable `worktables_index`, `arctic`, and `congee` backends for generated
+  primary and supported secondary indexes, including persisted topology load.
+- Async and batched insert/delete paths, with bulk-mutation signaling used by
+  reactive vacuum scheduling.
+- `MemStat` for generated persisted/read-only tables and Arc-owned
+  `unload_gracefully` for generation swaps.
+- Strict persisted-state validation, schema metadata, recovery loading, and
+  Arctic string/non-unique index support.
+
+### Changed
+
+- WorkTable row/page reclamation uses the local `ps-reclaim` domain regardless
+  of the selected index backend. Arctic and Congee also select their local
+  `ps-reclaim` SMR implementations; WorkTablesIndex retains its structural
+  skip-list reclamation internally.
+- Readers now synchronize on the exact physical cell. Unrelated rows cannot
+  block because of a hashed lock collision.
+- Vacuum discovers move candidates from a transient primary-index snapshot and
+  keeps only one live-cell counter per page, removing the previous four-byte
+  per-row directory.
+- Vacuum waits for three quiet observations after mutation activity and yields
+  throughout a bulk mutation instead of competing with foreground work.
+- The archived wrapper retains the beta.17 inner-row position so legacy stores
+  without bundled schema metadata remain readable.
+
+### Fixed
+
+- Torn reads and premature physical-link reuse during concurrent update,
+  delete, and vacuum activity.
+- In-place replacement synchronizes through the runtime side-table cell lock;
+  the beta.17 archived row bytes remain unchanged.
+- Whole-map Arctic destruction uses an unordered physical drain instead of
+  repeatedly searching for the next logical key.
+- Persisted primary/secondary index reconstruction and validation failures that
+  could otherwise expose missing, duplicate, or mismatched rows.
+
 ## [0.4.1]
 
 ### Added
@@ -57,4 +97,4 @@ Change Log
 
 ### Fixed
 
-- `Clippy` errors in macro declaration about unused `Result`'s.
+- `Clippy` errors in macro declaration about unused `Result`'s.

Cargo.toml

@@ -3,7 +3,7 @@ members = ["codegen", "dsl", "examples", "performance_measurement", "performance
 
 [package]
 name = "worktable"
-version = "1.0.0-beta.17"
+version = "1.0.0-beta.18"
 edition = "2024"
 authors = ["Handy-caT"]
 license = "MIT"
@@ -33,34 +33,33 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+# Read-mostly snapshots own page Arcs while the fixed directory supplies a
+# pointer-only fast path. Publication is append-only and asserted at each swap.
+arc-swap = "1"
 async-trait = "0.1"
-arctic = { package = "arctic-wt", version = "0.1" }
-congee = { package = "congee-wt", version = "0.4" }
+arctic = { package = "arctic-wt", version = "^0.1", default-features = false, features = ["smr-ps-reclaim"] }
+congee = { package = "congee-wt", version = "^0.4" }
 convert_case = "0.6"
 crc32fast = "1"
-# Already in the dependency graph transitively (indexset's concurrent
-# structures); used directly for read-side grace periods.
-data_bucket = "0.5"
-# data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" }
-# data_bucket = { path = "../DataBucket", version = "0.3.14" }
+data_bucket = { version = "^0.5" }
 derive_more = { version = "2", features = ["from", "error", "display", "debug", "into"] }
 eyre = "0.6"
 fastrand = "2"
 futures = "0.3"
-indexset = { package = "WorkTablesIndex", version = "0.0", default-features = false, features = ["concurrent", "cdc", "multimap"] }
+indexset = { package = "WorkTablesIndex", version = "^0.0", default-features = false, features = ["concurrent", "cdc", "multimap"] }
 vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"] }
 # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] }
-# indexset = { package = "wt-indexset", version = "=0.12.12", features = ["concurrent", "cdc", "multimap"] }
+# indexset = { package = "wt-indexset", version = "^0.12", features = ["concurrent", "cdc", "multimap"] }
 log = "0.4"
 ordered-float = "5"
 parking_lot = "0.12"
-performance_measurement = { path = "performance_measurement", version = "0.1.0", optional = true }
-performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true }
+performance_measurement = { path = "performance_measurement", version = "^0.1", optional = true }
+performance_measurement_codegen = { path = "performance_measurement/codegen", version = "^0.1", optional = true }
 prettytable-rs = "0.10"
 psc-nanoid = { version = "3", features = ["rkyv", "packed"] }
 rkyv = { version = "0.8", features = ["uuid-1"] }
 reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] }
-ps-reclaim = "0.1"
+ps-reclaim = { version = "^0.1" }
 rustc-hash = "2"
 rusty-s3 = { version = "0.10", optional = true }
 smart-default = "0.7"
@@ -69,14 +68,15 @@ tracing = "0.1"
 url = { version = "2", optional = true }
 uuid = { version = "1", features = ["v4", "v7"] }
 walkdir = { version = "2", optional = true }
-# Exact for the same reason as `worktable_dsl` in codegen/Cargo.toml: carets do
-# not match pre-releases, and these move as one train.
-worktable_codegen = { path = "codegen", version = "=1.0.0-beta.17" }
+# These pre-release workspace crates move as one train. The explicit caret
+# keeps the dependency policy consistent while the local path selects this
+# checkout during validation.
+worktable_codegen = { path = "codegen", version = "^1.0.0-beta.18" }
 # Re-exported below. Each generated table carries its declaration as a const
 # whose documentation says to read it with `worktable_dsl::Schema::parse`; that
 # instruction is only true if a plain `worktable` dependency can reach the
-# crate. Exact for the same pre-release reason as codegen.
-worktable_dsl = { path = "dsl", version = "=1.0.0-beta.18" }
+# crate.
+worktable_dsl = { path = "dsl", version = "^1.0.0-beta.18" }
 
 [dev-dependencies]
 chrono = "0.4"

README.md

@@ -30,7 +30,7 @@ cargo add worktable@1.0.0-beta.5
 | **Concurrency** | Lock-free concurrent indexes with change-data-capture, plus a row-level `LockMap` for ordered access. |
 | **Optional persistence** | `PersistedWorkTable` writes to local disk; the `s3-support` feature syncs that to S3. Both opt-in, so a purely in-memory table pays for neither. |
 | **Schema migration** | `worktable_version!` and `migration_engine!` version a table's schema and generate migrations between versions. See [docs/migration.md](docs/migration.md). |
-| **Memory accounting** | `MemStat` reports actual memory held. |
+| **Memory accounting** | `MemStat` estimates live heap; resident benchmarks measure allocator and SMR overhead. |
 
 ## Persistence
 

codegen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "worktable_codegen"
-version = "1.0.0-beta.17"
+version = "1.0.0-beta.18"
 edition = "2024"
 license = "MIT"
 description = "Proc-macro companion crate for worktable: the worktable! macro and its derives."
@@ -25,7 +25,7 @@ proc-macro = true
 # these three crates are one release train: the macro generates code against a
 # specific runtime, so they must move together. Every other dependency here is
 # a caret at minor granularity.
-worktable_dsl = { path = "../dsl", version = "=1.0.0-beta.18" }
+worktable_dsl = { path = "../dsl", version = "^1.0.0-beta.18" }
 rkyv = { version = "0.8" }
 syn = { version = "2", features = ["full"] }
 quote = "1"

codegen/src/generators/index_backend.rs

@@ -109,7 +109,7 @@ pub(crate) fn primary_key_backend_impl(
         IndexBackend::Arctic => {
             let field = single_supported_field(backend, fields, supported_types(backend))?;
             Ok((
-                quote! { Copy, },
+                quote! {},
                 quote! {
                     impl ArcticKey for #primary_key {
                         type Raw = <#field as ArcticKey>::Raw;

codegen/src/generators/persist/index/cdc.rs

@@ -446,7 +446,7 @@ mod tests {
         let mut columns = parser.parse_columns().unwrap();
         let mut parser = Parser::new(quote! {
             indexes: {
-                price_idx: price,
+                price_idx: price using worktables_index,
             }
         });
         columns.indexes = parser.parse_indexes().unwrap();

codegen/src/generators/persist/table/impls.rs

@@ -252,40 +252,37 @@ impl PersistGenerator {
         } else {
             quote! { IndexMap }
         };
-        let index_setup = if pk_types_unsized {
+        let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic {
             quote! {
-                inner.primary_index = std::sync::Arc::new(PrimaryIndex {
-                    pk_map: #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name),
-                    reverse_pk_map: IndexMap::new(),
-                });
+                inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map(
+                    PersistentArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default()
+                ));
+            }
+        } else if pk_types_unsized {
+            quote! {
+                inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map(
+                    #wti_map::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name)
+                ));
             }
         } else {
             match self.columns.primary_index_backend {
                 crate::common::model::IndexBackend::WorktablesIndex => quote! {
                     let size = get_index_page_size_from_data_length::<#pk_type>(#const_name);
-                    inner.primary_index = std::sync::Arc::new(PrimaryIndex {
-                        pk_map: #wti_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size),
-                        reverse_pk_map: IndexMap::new(),
-                    });
+                    inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map(
+                        #wti_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size)
+                    ));
                 },
                 crate::common::model::IndexBackend::Indexset => quote! {
                     let size = get_index_page_size_from_data_length::<#pk_type>(#const_name);
-                    inner.primary_index = std::sync::Arc::new(PrimaryIndex {
-                        pk_map: UpstreamIndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size),
-                        reverse_pk_map: IndexMap::new(),
-                    });
-                },
-                crate::common::model::IndexBackend::Arctic => quote! {
-                    inner.primary_index = std::sync::Arc::new(PrimaryIndex {
-                        pk_map: PersistentArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default(),
-                        reverse_pk_map: IndexMap::new(),
-                    });
+                    inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map(
+                        UpstreamIndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size)
+                    ));
                 },
+                crate::common::model::IndexBackend::Arctic => unreachable!("handled before variable-size dispatch"),
                 crate::common::model::IndexBackend::Congee => quote! {
-                    inner.primary_index = std::sync::Arc::new(PrimaryIndex {
-                        pk_map: PersistentCongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default(),
-                        reverse_pk_map: IndexMap::new(),
-                    });
+                    inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map(
+                        PersistentCongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default()
+                    ));
                 },
             }
         };

codegen/src/generators/persist/table/mod.rs

@@ -14,6 +14,7 @@ impl PersistGenerator {
         let page_size_consts = self.gen_page_size_consts();
         let version_const = self.gen_version_const();
         let type_ = self.gen_table_type()?;
+        let mem_stat_impl = self.gen_table_mem_stat_impl();
         let impl_ = self.gen_table_impl();
         let index_fns = self.gen_table_index_fns()?;
         let select_query_executor_impl = self.gen_table_select_query_executor_impl();
@@ -23,13 +24,29 @@ impl PersistGenerator {
             #page_size_consts
             #version_const
             #type_
+            #mem_stat_impl
             #impl_
             #index_fns
             #select_query_executor_impl
             #column_range_type
         })
     }
 
+    fn gen_table_mem_stat_impl(&self) -> TokenStream {
+        let ident = WorktableNameGenerator::from_table_name(self.name.to_string()).get_work_table_ident();
+        quote! {
+            impl MemStat for #ident {
+                fn heap_size(&self) -> usize {
+                    self.0.heap_size()
+                }
+
+                fn used_size(&self) -> usize {
+                    self.0.used_size()
+                }
+            }
+        }
+    }
+
     fn gen_page_size_consts(&self) -> TokenStream {
         let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string());
         let page_const_name = name_generator.get_page_size_const_ident();
@@ -97,6 +114,10 @@ impl PersistGenerator {
                     #[table(pk_unsized, pk_wti_logical)]
                 }
             }
+            (true, crate::common::model::IndexBackend::Arctic) => quote! {
+                #[derive(Debug, PersistTable)]
+                #[table(pk_arctic_string)]
+            },
             (true, _) => quote! {
                 #[derive(Debug, PersistTable)]
                 #[table(pk_unsized)]

codegen/src/generators/read_only/table/impls.rs

@@ -235,25 +235,34 @@ impl ReadOnlyGenerator {
             })
             .collect::<Vec<_>>();
         let pk_types_unsized = is_unsized_vec(pk_types);
-        let pk_map = match self.columns.primary_index_backend {
-            crate::common::model::IndexBackend::Indexset => quote! { UpstreamIndexMap },
-            _ => quote! { IndexMap },
-        };
-
-        let index_setup = if pk_types_unsized {
+        let index_setup = if self.columns.primary_index_backend == crate::common::model::IndexBackend::Arctic {
+            quote! {
+                inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map(
+                    ArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default()
+                ));
+            }
+        } else if self.columns.primary_index_backend == crate::common::model::IndexBackend::Congee {
             quote! {
-                inner.primary_index = std::sync::Arc::new(PrimaryIndex {
-                    pk_map: IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name),
-                    reverse_pk_map: IndexMap::new(),
-                });
+                inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map(
+                    CongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default()
+                ));
+            }
+        } else if pk_types_unsized {
+            quote! {
+                inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map(
+                    IndexMap::<#pk_type, OffsetEqLink<#const_name>, UnsizedNode<_>>::with_maximum_node_size(#const_name)
+                ));
             }
         } else {
+            let pk_map = match self.columns.primary_index_backend {
+                crate::common::model::IndexBackend::Indexset => quote! { UpstreamIndexMap },
+                _ => quote! { IndexMap },
+            };
             quote! {
                 let size = get_index_page_size_from_data_length::<#pk_type>(#const_name);
-                inner.primary_index = std::sync::Arc::new(PrimaryIndex {
-                    pk_map: #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size),
-                    reverse_pk_map: IndexMap::new(),
-                });
+                inner.primary_index = std::sync::Arc::new(PrimaryIndex::from_map(
+                    #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size)
+                ));
             }
         };