WorkTable 1.9.0-alpha1: off tokio and tokio::fs, columnar side indexes, and a schema-selected runtime

.gitattributes

@@ -0,0 +1 @@
+*.pdf binary

.github/workflows/rust.yml

@@ -13,6 +13,18 @@ permissions:
   contents: read
 
 jobs:
+  # `scripts/ci-local.sh` runs `cargo fmt --all --check` as its first step. CI
+  # did not, which made the script stricter than CI instead of equal to it, and
+  # formatting drift reached master unnoticed. Same command, same arguments.
+  fmt:
+    name: Formatting
+    runs-on: ubicloud-standard-2
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - name: Formatting (cargo fmt --all --check)
+        run: cargo fmt --all --check
+
   build:
     name: Build and test (${{ matrix.name }})
     runs-on: ubicloud-standard-2

.gitignore

@@ -16,3 +16,10 @@ tests/data/*
 # cargo-mutants run output
 /mutants.out/
 /mutants.out.old/
+
+# Generated documents. `docs/wt-user-guide.typ` and the columnar guide are the
+# sources; the PDFs are build output and were tracked, so every edit to a guide
+# put a new binary blob in the history. Rebuild with:
+#   typst compile docs/wt-user-guide.typ docs/wt-user-guide.pdf
+*.pdf
+.DS_Store

Cargo.toml

@@ -3,7 +3,7 @@ members = ["codegen", "dsl", "examples", "performance_measurement", "performance
 
 [package]
 name = "worktable"
-version = "1.0.0-beta.19"
+version = "1.9.0-alpha1"
 edition = "2024"
 authors = ["Handy-caT"]
 license = "MIT"
@@ -15,8 +15,31 @@ keywords = ["database", "embedded", "in-memory", "index", "storage"]
 categories = ["database-implementations", "data-structures", "caching"]
 
 [features]
-default = ["wti-predictable-search"]
+default = ["std", "wti-predictable-search", "vanilla-index"]
+# `futures/std` belongs here rather than on the dependency
+# line: written as `features = ["std"]` beside the version, a
+# `--no-default-features` build of this crate would still turn them on. The
+# s3 tests reach `futures::io`, whose traits live behind that feature.
+std = ["nagoya/std", "ps-reclaim/std", "congee/std", "indexset/std", "futures/std", "worktable_codegen/std"]
+# The tokio backend for `Runtime`, selectable with `runtime: tokio` in a
+# schema. **Off, and it stays off.** Getting tokio out of the normal dependency
+# graph is the work this builds on: it used to arrive through the `tokio::`
+# paths `worktable!` emitted into consumer crates, so a runtime was part of the
+# macro's contract whether a consumer ran one or not. The check is
+# `cargo tree -e normal -i tokio` printing nothing in the default feature set.
+tokio-runtime = ["dep:tokio", "std"]
+# The upstream IndexSet backend, selectable per index with `using indexset`.
+# Optional because it reaches `crossbeam-utils`, `parking_lot` and `serde`,
+# none of which build without `std`, and because it is one backend of four
+# rather than something the table needs.
+vanilla-index = ["dep:vanilla_indexset"]
 perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"]
+# Test-only, and empty on purpose: it compiles the four-backend arms of
+# `tests/worktable/runtime_backends.rs`, which declare `runtime:` on a table.
+# Off by default because the DSL keyword and the `Runtime` trait land
+# separately, and a default-on flag would make this branch red until they do.
+# The arm that runs today declares no runtime and is not behind this flag.
+runtime-backends = []
 s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktable_codegen/s3-support"]
 # Moves unique WorkTablesIndex structural CDC work out of the table mutation
 # path and into the background persistence worker. The persisted page format
@@ -35,54 +58,119 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"]
 [dependencies]
 # Read-mostly snapshots own page Arcs while the fixed directory supplies a
 # pointer-only fast path. Publication is append-only and asserted at each swap.
-arc-swap = "1"
+arc-swap = { version = "1", default-features = false }
 async-trait = "0.1"
-arctic = { package = "arctic-wt", version = "^0.1, >=0.1.9", default-features = false, features = ["smr-ps-reclaim"] }
-congee = { package = "congee-wt", version = "^0.4, >=0.4.4" }
-convert_case = "0.6"
-crc32fast = "1"
-data_bucket = { version = "^0.5, >=0.5.7" }
-derive_more = { version = "2", features = ["from", "error", "display", "debug", "into"] }
+arctic = { package = "arctic-wt", version = "^0.1, >=0.1.11", default-features = false, features = ["smr-ps-reclaim"] }
+# `default-features = false`, or its `std` turns `ps-reclaim/std` on and this
+# crate links the standard library through a dependency. That matters for
+# EKOPathRS bootstrapping, which is what `no_std` here is for, not for any
+# bare-metal target: we ship on Linux, macOS and Windows.
+congee = { package = "congee-wt", version = "^0.4, >=0.4.4", default-features = false }
+convert_case = { version = "0.6", default-features = false }
+crc32fast = { version = "1", default-features = false }
+# 0.6 because the page stride is a parameter now and the error type is
+# concrete: `^0.5` cannot resolve the crate this depends on.
+data_bucket = { version = "^0.6" }
+derive_more = { version = "2", default-features = false, features = ["from", "error", "display", "debug", "into"] }
 eyre = "0.6"
 fastrand = "2"
-futures = "0.3"
+hashbrown = "0.15"
+futures = { version = "0.3", default-features = false, features = ["alloc"] }
+# The file traits and the host implementation. `std` here; the crate takes it
+# without default features wherever the filesystem is not involved.
+# `^0.1.1` and not `^0.1`: 0.1.1 is the first release with
+# `Runtime::with_tuning`, which is the only way to get a pool at a chosen
+# tuning whose threads are marked as pool workers, and that marker is what
+# makes `local_wakes` do anything. Against 0.1.0 this crate does not build,
+# and naming the version says so rather than leaving a resolver to.
+#
+# It also re-exports `Tuning`, which is why `ps-st3` is no longer a direct
+# dependency here: it was named for that one type.
+nagoya = { version = "^0.1.1", default-features = false }
 indexset = { package = "WorkTablesIndex", version = "^0.0, >=0.0.12", default-features = false, features = ["concurrent", "cdc", "multimap"] }
-vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"] }
+# Pulls `serde` and `serde_core` into every build, and nothing here uses them.
+# They are not reachable from this line: `indexset` 0.15 declares `ftree` with
+# `features = ["serde"]` unconditionally, so no `default-features = false` here
+# removes them and forking `ftree` would not either. This is one of four
+# selectable index backends and making it optional is the fix.
+vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"], optional = true }
 # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] }
 # indexset = { package = "wt-indexset", version = "^0.12", features = ["concurrent", "cdc", "multimap"] }
-log = "0.4"
-ordered-float = "5"
-parking_lot = "0.12"
+log = { version = "0.4", default-features = false }
+ordered-float = { version = "5", default-features = false }
+# The house fork, not vanilla: `parking_lot_lite_hack` is what every other
+# consumer here takes. Same crate name via `package`, so no call site changes.
+#
+# Vanilla is still in the default-feature graph, and no longer for the reason
+# it was: it used to arrive through tokio, and now arrives through `indexset
+# 0.15` behind the `vanilla-index` default feature. So it is gone from a
+# `--no-default-features` build and present in a normal one, until that backend
+# is deselected.
+#
+# Two copies of the fork are also in the lock, because `WorkTablesIndex` takes
+# the published 0.12.7 while this takes the branch. That is not a version skew
+# to wait out: 0.12.7's own module comment says `FairMutex` is among what it
+# removed, so the branch is the only copy that has what is used here. It
+# collapses when the branch publishes as 0.12.8.
+parking_lot = { package = "parking_lot_lite_hack", git = "https://github.com/pathscale/parking_lot_lite_hack", branch = "feat/fair-mutex", default-features = false }
 performance_measurement = { path = "performance_measurement", version = "^0.1", optional = true }
 performance_measurement_codegen = { path = "performance_measurement/codegen", version = "^0.1", optional = true }
-prettytable-rs = "0.10"
 psc-nanoid = { version = "3", features = ["rkyv", "packed"] }
-rkyv = { version = "0.8", features = ["uuid-1"] }
+rkyv = { version = "0.8", default-features = false, features = ["alloc", "bytecheck", "uuid-1"] }
 reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] }
-ps-reclaim = { version = "^0.1, >=0.1.3" }
-rustc-hash = "2"
+# `spin` rather than `std`: without it this crate reaches for `thread_local!`,
+# which needs a platform. See its `slot` module for what it falls back to.
+ps-reclaim = { version = "^0.1, >=0.1.4", default-features = false, features = ["spin"] }
+rustc-hash = { version = "2", default-features = false }
 rusty-s3 = { version = "0.10", optional = true }
 smart-default = "0.7"
-tokio = { version = "1", features = ["full"] }
-tracing = "0.1"
+# Optional, off by default, and reachable only through the `tokio-runtime`
+# feature. See that feature for why this is not a plain dependency. The feature
+# list is what `TokioRt` calls and no more: `full` would put an I/O driver and
+# a process reaper in the graph for a backend that only schedules.
+tokio = { version = "1", default-features = false, features = [
+    "rt-multi-thread",
+    "sync",
+    "time",
+], optional = true }
+tracing = { version = "0.1", default-features = false }
 url = { version = "2", optional = true }
 uuid = { version = "1", features = ["v4", "v7"] }
 walkdir = { version = "2", optional = true }
 # These pre-release workspace crates move as one train. The explicit caret
 # keeps the dependency policy consistent while the local path selects this
 # checkout during validation.
-worktable_codegen = { path = "codegen", version = "^1.0.0-beta.19" }
+worktable_codegen = { path = "codegen", version = "^1.9.0-alpha1" }
 # Re-exported below. Each generated table carries its declaration as a const
 # whose documentation says to read it with `worktable_dsl::Schema::parse`; that
 # instruction is only true if a plain `worktable` dependency can reach the
 # crate.
-worktable_dsl = { path = "dsl", version = "^1.0.0-beta.18.1" }
+worktable_dsl = { path = "dsl", version = "^1.0.0-beta.19" }
 
 [dev-dependencies]
 chrono = "0.4"
+# For one test, and only one: the flavor registry lives here and the DSL
+# carries a mirror of it so the parser does not have to depend on the runtime
+# crate. A dev-dependency is what lets a test compare the two lists without
+# putting the mirror's `syn` and `proc-macro2` into anybody's runtime graph.
+worktable_dsl = { path = "dsl" }
 criterion = { version = "0.5", features = ["async_tokio"] }
 rand = "0.9"
+# A dev-dependency, and that is the whole of the no_std claim. It was a normal
+# dependency with every feature on, so `cargo tree --no-default-features -e
+# normal -i tokio` found it linked into a build that had asked for no std at
+# all. `cargo check --no-default-features` passed the entire time, because it
+# only ever proved this crate's own source was std-free and never its closure.
+# What kept it there was not the engine: it was six `tokio::` paths emitted by
+# `worktable!` into consumer crates, which made the runtime part of the macro's
+# contract. Those go through `worktable::prelude` now.
+tokio = { version = "1", features = ["full"] }
 tracing-subscriber = "0.3"
+# Drives `tests/ui.rs`. Half of what `worktable!` promises is a refusal, and a
+# refusal is only testable by compiling something that must not compile. The
+# expected diagnostic is a committed `.stderr` beside each case, so the test
+# asserts on the message and not merely on failure.
+trybuild = "1"
 
 # Only under `--cfg loom`, so a normal build and a normal `cargo test` never
 # resolve it. See src/partition/loom_tests.rs for how to run the models.

README.md

@@ -16,6 +16,19 @@ from a macro, and that persisting it is one feature flag away.
 cargo add worktable@1.0.0-beta.5
 ```
 
+## New in 1.9
+
+- **`no_std`.** `default-features = false` and the macro still works. Persistence,
+  vacuum and the disk index need an operating system and are gated out.
+- **Columnar fields and indexes.** `columnar` on a column, `columnar_indexes` with
+  `cluster_by`, so a scan over one field reads only that field's bytes.
+- **A schema-selected runtime.** `runtime: nagoya(<flavor>)` or `runtime: tokio`.
+- **`page_size` on a persisted table**, at any size above a 512-byte floor.
+- **The default index backend is `arctic`**, not `worktables_index`. Arctic cannot
+  key an optional or variable-width column, so an index over `String optional`
+  must now say `using worktables_index`. Only `congee` still requires `persist`
+  to be stated explicitly.
+
 ## What you get
 
 | | |

benches/cases/full_featured.rs

@@ -19,7 +19,7 @@ fn insert(c: &mut Criterion) {
                     another: format!("another_{}", fastrand::u64(..)),
                     something: fastrand::u64(..),
                 };
-                futures::executor::block_on(table.insert(black_box(row)))
+                nagoya::block_on(table.insert(black_box(row)))
             },
             BatchSize::SmallInput,
         )
@@ -37,7 +37,7 @@ fn select_by_pk(c: &mut Criterion) {
                 another: format!("another_{}", fastrand::u64(..)),
                 something: fastrand::u64(..),
             };
-            futures::executor::block_on(table.insert(row)).unwrap()
+            nagoya::block_on(table.insert(row)).unwrap()
         })
         .collect();
 
@@ -60,7 +60,7 @@ fn select_by_unique_index(c: &mut Criterion) {
             another: format!("another_{}", i),
             something: i,
         };
-        futures::executor::block_on(table.insert(row)).unwrap();
+        nagoya::block_on(table.insert(row)).unwrap();
     }
 
     c.bench_function("full_featured_select_by_val1", |b| {
@@ -82,7 +82,7 @@ fn select_by_non_unique_index(c: &mut Criterion) {
             another: format!("cat_{}", i % 10),
             something: i,
         };
-        futures::executor::block_on(table.insert(row)).unwrap();
+        nagoya::block_on(table.insert(row)).unwrap();
     }
 
     c.bench_function("full_featured_select_by_another", |b| {
@@ -196,7 +196,7 @@ fn in_place_update(c: &mut Criterion) {
             another: "test".to_string(),
             something: 0,
         };
-        futures::executor::block_on(table.insert(row)).unwrap().into()
+        nagoya::block_on(table.insert(row)).unwrap().into()
     };
 
     c.bench_function("full_featured_in_place_update_val", |b| {
@@ -219,7 +219,7 @@ fn delete(c: &mut Criterion) {
                     another: format!("temp_{}", fastrand::u64(..)),
                     something: fastrand::u64(..),
                 };
-                futures::executor::block_on(table.insert(row)).unwrap()
+                nagoya::block_on(table.insert(row)).unwrap()
             },
             |pk: FullFeaturedPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }),
             BatchSize::SmallInput,
@@ -242,7 +242,7 @@ fn delete_by_index_query(c: &mut Criterion) {
                     another: another.clone(),
                     something: fastrand::u64(..),
                 };
-                futures::executor::block_on(table.insert(row)).unwrap();
+                nagoya::block_on(table.insert(row)).unwrap();
                 another
             },
             |another: String| rt.block_on(async { table.delete_by_another(another).await.unwrap() }),
@@ -319,7 +319,7 @@ fn batch_insert(c: &mut Criterion) {
                             another: format!("another_{}", i),
                             something: i as u64,
                         };
-                        futures::executor::block_on(table.insert(black_box(row))).unwrap();
+                        nagoya::block_on(table.insert(black_box(row))).unwrap();
                     }
                     black_box(table)
                 },
@@ -347,7 +347,7 @@ fn batch_select_pk(c: &mut Criterion) {
                     another: format!("another_{}", fastrand::u64(..)),
                     something: fastrand::u64(..),
                 };
-                futures::executor::block_on(table.insert(row)).unwrap()
+                nagoya::block_on(table.insert(row)).unwrap()
             })
             .collect();
 

benches/cases/non_unique_index.rs

@@ -33,7 +33,7 @@ fn select_by_pk(c: &mut Criterion) {
                 value: fastrand::u64(..),
                 category: fastrand::u64(0..10),
             };
-            futures::executor::block_on(table.insert(row)).unwrap()
+            nagoya::block_on(table.insert(row)).unwrap()
         })
         .collect();
 
@@ -54,7 +54,7 @@ fn select_by_non_unique_index(c: &mut Criterion) {
             value: fastrand::u64(..),
             category: i % 10,
         };
-        futures::executor::block_on(table.insert(row)).unwrap();
+        nagoya::block_on(table.insert(row)).unwrap();
     }
 
     c.bench_function("non_unique_index_select_by_category", |b| {
@@ -108,7 +108,7 @@ fn delete(c: &mut Criterion) {
                     value: fastrand::u64(..),
                     category: fastrand::u64(0..10),
                 };
-                futures::executor::block_on(table.insert(row)).unwrap()
+                nagoya::block_on(table.insert(row)).unwrap()
             },
             |pk: NonUniqueIndexPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }),
             BatchSize::SmallInput,
@@ -176,7 +176,7 @@ fn batch_insert(c: &mut Criterion) {
                             value: i as u64,
                             category: (i % 10) as u64,
                         };
-                        futures::executor::block_on(table.insert(black_box(row))).unwrap();
+                        nagoya::block_on(table.insert(black_box(row))).unwrap();
                     }
                     black_box(table)
                 },
@@ -202,7 +202,7 @@ fn batch_select_pk(c: &mut Criterion) {
                     value: fastrand::u64(..),
                     category: fastrand::u64(0..10),
                 };
-                futures::executor::block_on(table.insert(row)).unwrap()
+                nagoya::block_on(table.insert(row)).unwrap()
             })
             .collect();