Race writers across every backend, above the count the suite reached

.github/workflows/rust.yml

@@ -74,16 +74,29 @@ jobs:
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
         run: |
-          if [ -z "$CARGO_REGISTRY_TOKEN" ]; then echo "CARGO_REGISTRY_TOKEN not set; skipping publish"; exit 0; fi
-          v=$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -1)
-          if curl -fsSL "https://index.crates.io/wo/rk/worktable" | sed -n 's/.*"vers":"\([^"]*\)".*/\1/p' | grep -qx "$v"; then
-            echo "worktable $v is already on crates.io; nothing to publish"
-            exit 0
-          fi
-          # The main crate pins its codegen twin exactly, so the twin must land
-          # on the registry first; cargo waits for availability between the two.
-          cv=$(sed -n 's/^version = "\(.*\)"$/\1/p' codegen/Cargo.toml | head -1)
-          if ! curl -fsSL "https://index.crates.io/wo/rk/worktable_codegen" | sed -n 's/.*"vers":"\([^"]*\)".*/\1/p' | grep -qx "$cv"; then
-            cargo publish -p worktable_codegen
-          fi
-          cargo publish -p worktable
+          set -euo pipefail
+          if [ -z "${CARGO_REGISTRY_TOKEN:-}" ]; then echo "CARGO_REGISTRY_TOKEN not set; skipping publish"; exit 0; fi
+
+          # Publish in dependency order. worktable_codegen depends on
+          # worktable_dsl and worktable depends on worktable_codegen, both by
+          # path with an exact version, so each must be on the registry before
+          # the next is packaged. Omitting worktable_dsl here is what made
+          # `cargo publish -p worktable_codegen` fail with "no matching package
+          # named `worktable_dsl` found" the moment the DSL extraction landed.
+          publish_if_new() {
+            crate="$1"
+            manifest="$2"
+            version=$(sed -n 's/^version = "\(.*\)"$/\1/p' "$manifest" | head -1)
+            # crates.io index paths: four or more characters is {first two}/{next two}/{name}.
+            if curl -fsSL "https://index.crates.io/wo/rk/$crate" \
+              | sed -n 's/.*"vers":"\([^"]*\)".*/\1/p' | grep -qx "$version"; then
+              echo "$crate $version is already on crates.io; skipping"
+              return 0
+            fi
+            echo "publishing $crate $version"
+            cargo publish -p "$crate"
+          }
+
+          publish_if_new worktable_dsl dsl/Cargo.toml
+          publish_if_new worktable_codegen codegen/Cargo.toml
+          publish_if_new worktable Cargo.toml

Cargo.toml

@@ -1,9 +1,9 @@
 [workspace]
-members = ["codegen", "examples", "performance_measurement", "performance_measurement/codegen"]
+members = ["codegen", "dsl", "examples", "performance_measurement", "performance_measurement/codegen"]
 
 [package]
 name = "worktable"
-version = "1.0.0-beta.16"
+version = "1.0.0-beta.17"
 edition = "2024"
 authors = ["Handy-caT"]
 license = "MIT"
@@ -40,15 +40,15 @@ convert_case = "0.6.0"
 crc32fast = "1.5.0"
 # Already in the dependency graph transitively (indexset's concurrent
 # structures); used directly for read-side grace periods.
-data_bucket = "=0.5.5"
+data_bucket = "0.5.5"
 # data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" }
 # data_bucket = { path = "../DataBucket", version = "0.3.14" }
 derive_more = { version = "2.0.1", features = ["from", "error", "display", "debug", "into"] }
 eyre = "0.6.12"
 fastrand = "2.3.0"
 futures = "0.3.30"
-indexset = { package = "WorkTablesIndex", version = "=0.0.9", default-features = false, features = ["concurrent", "cdc", "multimap"] }
-vanilla_indexset = { package = "indexset", version = "=0.15.0", features = ["concurrent", "cdc", "multimap"] }
+indexset = { package = "WorkTablesIndex", version = "0.0.9", default-features = false, features = ["concurrent", "cdc", "multimap"] }
+vanilla_indexset = { package = "indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] }
 # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] }
 # indexset = { package = "wt-indexset", version = "=0.12.12", features = ["concurrent", "cdc", "multimap"] }
 log = "0.4.29"
@@ -60,7 +60,7 @@ prettytable-rs = "^0.10"
 psc-nanoid = { version = "3.1.1", features = ["rkyv", "packed"] }
 rkyv = { version = "0.8.17", features = ["uuid-1"] }
 reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] }
-ps-reclaim = "0.1.0"
+ps-reclaim = "0.1.1"
 rustc-hash = "2.1.1"
 rusty-s3 = { version = "0.10.2", optional = true }
 smart-default = "0.7.1"
@@ -69,7 +69,7 @@ tracing = "0.1"
 url = { version = "2", optional = true }
 uuid = { version = "1.24.0", features = ["v4", "v7"] }
 walkdir = { version = "2", optional = true }
-worktable_codegen = { path = "codegen", version = "=1.0.0-beta.15" }
+worktable_codegen = { path = "codegen", version = "=1.0.0-beta.17" }
 
 [dev-dependencies]
 chrono = "0.4.43"

codegen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "worktable_codegen"
-version = "1.0.0-beta.15"
+version = "1.0.0-beta.17"
 edition = "2024"
 license = "MIT"
 description = "Proc-macro companion crate for worktable: the worktable! macro and its derives."
@@ -18,6 +18,9 @@ path = "src/lib.rs"
 proc-macro = true
 
 [dependencies]
+# The schema language, extracted so consumers other than this macro can read
+# a declaration. See its crate docs for why that needed a separate crate.
+worktable_dsl = { path = "../dsl", version = "=1.0.0-beta.17" }
 rkyv = { version = "0.8.17" }
 syn = { version = "2.0.74", features = ["full"] }
 quote = "1.0.36"

codegen/src/common/mod.rs

@@ -1,7 +1,13 @@
-pub mod model;
+//! What stayed behind when the schema language moved out.
+//!
+//! `model` and `parser` are `worktable_dsl` now, so anything can read a
+//! declaration. `name_generator` is not part of that language: it invents Rust
+//! identifiers for generated code, which is this crate's concern and nobody
+//! else's.
+//!
+//! It also could not have gone. Generators here define inherent `impl`s on
+//! `WorktableNameGenerator`, and the orphan rule forbids that for a type owned
+//! by another crate. The compiler makes the same argument the design does.
 pub mod name_generator;
-pub mod parser;
 
-#[allow(unused_imports)]
-pub use model::*;
-pub use parser::Parser;
+pub use worktable_dsl::{Parser, model, parser};

codegen/src/common/name_generator.rs

@@ -120,6 +120,18 @@ impl WorktableNameGenerator {
         )
     }
 
+    /// The name of the const carrying the table's own declaration.
+    ///
+    /// It follows `get_version_const_ident`'s shape because it answers the
+    /// question next to it: the version says *which* schema, and this says
+    /// *what* that schema is.
+    pub fn get_schema_const_ident(&self) -> Ident {
+        let upper_snake_case_name = self.name.from_case(Case::Pascal).to_case(Case::UpperSnake);
+        Ident::new(
+            format!("{}_SCHEMA", upper_snake_case_name.to_uppercase()).as_str(),
+            Span::mixed_site(),
+        )
+    }
     pub fn get_space_secondary_index_ident(&self) -> Ident {
         Ident::new(format!("{}SpaceSecondaryIndex", self.name).as_str(), Span::mixed_site())
     }

codegen/src/generators/in_memory/table/impls.rs

@@ -17,6 +17,7 @@ impl InMemoryGenerator {
         let select_range_fn = self.gen_table_select_range_fn();
         let insert_fn = self.gen_table_insert_fn();
         let insert_many_fn = self.gen_table_insert_many_fn();
+        let delete_many_fn = self.gen_table_delete_many_fn();
         let reinsert_fn = self.gen_table_reinsert_fn();
         let upsert_fn = self.gen_table_upsert_fn();
         let get_next_fn = self.gen_table_get_next_fn();
@@ -35,6 +36,7 @@ impl InMemoryGenerator {
                 #select_range_fn
                 #insert_fn
                 #insert_many_fn
+                #delete_many_fn
                 #reinsert_fn
                 #upsert_fn
                 #count_fn
@@ -161,6 +163,63 @@ impl InMemoryGenerator {
         }
     }
 
+    fn gen_table_delete_many_fn(&self) -> TokenStream {
+        let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string());
+        let primary_key_type = name_generator.get_primary_key_type_ident();
+
+        quote! {
+            /// Deletes every row named by `pks`, behind one grace marker.
+            ///
+            /// `async` although it awaits nothing today. Every other write on
+            /// this table is async, because cell-level locking makes an update
+            /// wait on the readers of the cells it touches, and a write surface
+            /// where the caller has to know which operations happen to need
+            /// that is a surface where `let _ = table.upsert(row)` silently
+            /// drops a write. Uniformity is worth more here than the marginal
+            /// honesty of a sync signature, and the batch paths may need to
+            /// wait once they take cell locks rather than the striped gate.
+            ///
+            /// A delete is a bit flip: the row is marked deleted in place, its
+            /// index entries are removed, and its storage becomes reusable once
+            /// no reader can still reach it. `vacuum` is what later compacts
+            /// pages and hands whole ones back.
+            ///
+            /// Batching matters because the per-row cost is dominated by the
+            /// reclamation bookkeeping each retirement takes, not by the bit
+            /// flip: `n` deletes take `n` domain advances where a batch takes
+            /// one.
+            ///
+            /// Unlike `insert_many` this is **not** all-or-nothing. A delete
+            /// that fails partway has already ghosted rows and removed their
+            /// index entries, and those rows are genuinely gone, so the error
+            /// reports how many succeeded rather than pretending to rewind.
+            /// A key that is not present is skipped rather than failing the
+            /// batch.
+            ///
+            /// Returns the keys actually deleted, in the order given.
+            /// Deletes every row whose primary key falls in `range`.
+            ///
+            /// The shape bulk eviction has: a caller dropping a generation
+            /// knows the span it wants gone rather than the individual keys.
+            /// The span is collected from the primary index in one ordered
+            /// walk and then deleted exactly as `delete_many` would, keys
+            /// still resolved under their mutation guards.
+            pub async fn delete_range<R>(&self, range: R)
+                -> core::result::Result<Vec<#primary_key_type>, BatchDeleteError<#primary_key_type>>
+            where R: core::ops::RangeBounds<#primary_key_type>
+            {
+                self.0.delete_range(range)
+            }
+
+            pub async fn delete_many<Pk>(&self, pks: Vec<Pk>)
+                -> core::result::Result<Vec<#primary_key_type>, BatchDeleteError<#primary_key_type>>
+            where #primary_key_type: From<Pk>
+            {
+                self.0.delete_many(pks.into_iter().map(core::convert::Into::into).collect())
+            }
+        }
+    }
+
     fn gen_table_reinsert_fn(&self) -> TokenStream {
         let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string());
         let row_type = name_generator.get_row_type_ident();

codegen/src/generators/persist/table/impls.rs

@@ -18,6 +18,7 @@ impl PersistGenerator {
         let select_range_fn = self.gen_table_select_range_fn();
         let insert_fn = self.gen_table_insert_fn();
         let insert_many_fn = self.gen_table_insert_many_fn();
+        let delete_many_fn = self.gen_table_delete_many_fn();
         let reinsert_fn = self.gen_table_reinsert_fn();
         let upsert_fn = self.gen_table_upsert_fn();
         let get_next_fn = self.gen_table_get_next_fn();
@@ -39,6 +40,7 @@ impl PersistGenerator {
                 #select_range_fn
                 #insert_fn
                 #insert_many_fn
+                #delete_many_fn
                 #reinsert_fn
                 #upsert_fn
                 #count_fn
@@ -483,6 +485,63 @@ impl PersistGenerator {
         }
     }
 
+    fn gen_table_delete_many_fn(&self) -> TokenStream {
+        let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string());
+        let primary_key_type = name_generator.get_primary_key_type_ident();
+
+        quote! {
+            /// Deletes every row named by `pks`, behind one grace marker.
+            ///
+            /// `async` although it awaits nothing today. Every other write on
+            /// this table is async, because cell-level locking makes an update
+            /// wait on the readers of the cells it touches, and a write surface
+            /// where the caller has to know which operations happen to need
+            /// that is a surface where `let _ = table.upsert(row)` silently
+            /// drops a write. Uniformity is worth more here than the marginal
+            /// honesty of a sync signature, and the batch paths may need to
+            /// wait once they take cell locks rather than the striped gate.
+            ///
+            /// A delete is a bit flip: the row is marked deleted in place, its
+            /// index entries are removed, and its storage becomes reusable once
+            /// no reader can still reach it. `vacuum` is what later compacts
+            /// pages and hands whole ones back.
+            ///
+            /// Batching matters because the per-row cost is dominated by the
+            /// reclamation bookkeeping each retirement takes, not by the bit
+            /// flip: `n` deletes take `n` domain advances where a batch takes
+            /// one.
+            ///
+            /// Unlike `insert_many` this is **not** all-or-nothing. A delete
+            /// that fails partway has already ghosted rows and removed their
+            /// index entries, and those rows are genuinely gone, so the error
+            /// reports how many succeeded rather than pretending to rewind.
+            /// A key that is not present is skipped rather than failing the
+            /// batch.
+            ///
+            /// Returns the keys actually deleted, in the order given.
+            /// Deletes every row whose primary key falls in `range`.
+            ///
+            /// The shape bulk eviction has: a caller dropping a generation
+            /// knows the span it wants gone rather than the individual keys.
+            /// The span is collected from the primary index in one ordered
+            /// walk and then deleted exactly as `delete_many` would, keys
+            /// still resolved under their mutation guards.
+            pub async fn delete_range<R>(&self, range: R)
+                -> core::result::Result<Vec<#primary_key_type>, BatchDeleteError<#primary_key_type>>
+            where R: core::ops::RangeBounds<#primary_key_type>
+            {
+                self.0.delete_range(range)
+            }
+
+            pub async fn delete_many<Pk>(&self, pks: Vec<Pk>)
+                -> core::result::Result<Vec<#primary_key_type>, BatchDeleteError<#primary_key_type>>
+            where #primary_key_type: From<Pk>
+            {
+                self.0.delete_many(pks.into_iter().map(core::convert::Into::into).collect())
+            }
+        }
+    }
+
     fn gen_table_reinsert_fn(&self) -> TokenStream {
         let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string());
         let row_type = name_generator.get_row_type_ident();

codegen/src/lib.rs

@@ -1,3 +1,11 @@
+// `common` is now a thin front for `worktable_dsl`, which holds the schema
+// model and parser so that anything other than this macro can read a
+// declaration. Kept as a module rather than an alias because the name
+// generator stays here: generators define inherent `impl`s on it, which the
+// orphan rule allows only in the crate that owns the type.
+//
+// The 127 `crate::common::` paths across this crate are unchanged, so the diff
+// is a move rather than a sweep.
 mod common;
 mod generators;
 mod mem_stat;