1.0.0-beta.17

.github/workflows/publish-to-crates-vip.yml

@@ -0,0 +1,63 @@
+name: publish-to-crates.vip
+
+# Publishes this workspace's crates to the private registry at crates.vip.
+#
+# The heavy lifting lives in api.crates.vip's reusable workflow: it packages the
+# crate, derives the sparse-index entry from `cargo metadata`, and writes both to
+# Tigris under conditional requests. Nothing is installed — the runner image
+# already ships `aws` and `jq`.
+#
+# Publishing is manual on purpose. A published version is immutable, and
+# `cargo package` embeds the git sha, so a version republished from a different
+# commit is refused rather than silently replaced.
+#
+# The reusable workflow is pinned to a commit sha rather than `@master`. This
+# job hands it the registry's Tigris credentials, so whoever can move that
+# branch can read the secrets; a sha is the only ref that cannot be moved under
+# us. Updating it is deliberate: read the diff on the pinned range first, then
+# bump the three `uses:` lines together, since they are one workflow and a
+# split pin means two versions publishing one workspace.
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        description: Derive the index entry and print it, but write nothing
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  # The workspace publishes bottom-up. `worktable_codegen` depends on
+  # `worktable_dsl` by path *and* version, so the DSL entry has to exist in the
+  # registry before codegen's is written, exactly as it does on crates.io.
+  dsl:
+    uses: pathscale/crates.vip-backend/.github/workflows/publish-crate.yml@017e29a705cb8b4780dd6079b3739b4b5a84fda5
+    with:
+      manifest-path: dsl/Cargo.toml
+      dry-run: ${{ inputs.dry-run }}
+    secrets:
+      tigris-key-id: ${{ secrets.CRATES_VIP_TIGRIS_KEY_ID }}
+      tigris-key-secret: ${{ secrets.CRATES_VIP_TIGRIS_KEY_SECRET }}
+
+  codegen:
+    needs: dsl
+    uses: pathscale/crates.vip-backend/.github/workflows/publish-crate.yml@017e29a705cb8b4780dd6079b3739b4b5a84fda5
+    with:
+      manifest-path: codegen/Cargo.toml
+      dry-run: ${{ inputs.dry-run }}
+    secrets:
+      tigris-key-id: ${{ secrets.CRATES_VIP_TIGRIS_KEY_ID }}
+      tigris-key-secret: ${{ secrets.CRATES_VIP_TIGRIS_KEY_SECRET }}
+
+  # `worktable` depends on `worktable_codegen`, so the codegen crate has to be
+  # resolvable in the registry before this entry is written.
+  worktable:
+    needs: codegen
+    uses: pathscale/crates.vip-backend/.github/workflows/publish-crate.yml@017e29a705cb8b4780dd6079b3739b4b5a84fda5
+    with:
+      manifest-path: Cargo.toml
+      dry-run: ${{ inputs.dry-run }}
+    secrets:
+      tigris-key-id: ${{ secrets.CRATES_VIP_TIGRIS_KEY_ID }}
+      tigris-key-secret: ${{ secrets.CRATES_VIP_TIGRIS_KEY_SECRET }}

.github/workflows/rust.yml

@@ -62,9 +62,43 @@ jobs:
       - name: Clippy (deny warnings)
         run: cargo clippy --workspace --all-targets ${{ matrix.args }} -- -D warnings
 
+  duplicate_index_crates:
+    name: One version of each shared index crate
+    runs-on: ubicloud-standard-2
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: "true"
+          add-job-id-key: "false"
+      # Two versions of WorkTablesIndex in one graph split the type identity of
+      # `Pair` and `ChangeEvent`, and the build then fails with "expected
+      # ChangeEvent<Pair<T, Link>>, found a different ChangeEvent<Pair<T,
+      # Link>>" across dozens of unrelated-looking lines. It happens whenever
+      # data_bucket and worktable disagree about which version they want, which
+      # is every time one is bumped and released without the other.
+      #
+      # The caret requirements are correct and stay. This turns the mismatch
+      # into one named failure instead of a wall of trait errors, which is the
+      # guard WT-8 asked for.
+      - name: Refuse a duplicated WorkTablesIndex or data_bucket
+        run: |
+          duplicates=$(cargo tree --duplicates --edges normal 2>/dev/null \
+            | grep -E '^(WorkTablesIndex|data_bucket) v' | sort -u)
+          if [ -n "$duplicates" ]; then
+            echo "More than one version of a shared index crate is in the graph:"
+            echo "$duplicates"
+            echo
+            echo "WorkTablesIndex, data_bucket and worktable move as one train."
+            echo "Publish them in lockstep, or pin them to agree."
+            exit 1
+          fi
+          echo "one version of each: ok"
+
   publish:
     if: github.event_name == 'push' && github.ref == 'refs/heads/master'
-    needs: [build, clippy_check]
+    needs: [build, clippy_check, duplicate_index_crates]
     runs-on: ubicloud-standard-2
     timeout-minutes: 45
     steps:
@@ -74,16 +108,29 @@ jobs:
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
         run: |
-          if [ -z "$CARGO_REGISTRY_TOKEN" ]; then echo "CARGO_REGISTRY_TOKEN not set; skipping publish"; exit 0; fi
-          v=$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -1)
-          if curl -fsSL "https://index.crates.io/wo/rk/worktable" | sed -n 's/.*"vers":"\([^"]*\)".*/\1/p' | grep -qx "$v"; then
-            echo "worktable $v is already on crates.io; nothing to publish"
-            exit 0
-          fi
-          # The main crate pins its codegen twin exactly, so the twin must land
-          # on the registry first; cargo waits for availability between the two.
-          cv=$(sed -n 's/^version = "\(.*\)"$/\1/p' codegen/Cargo.toml | head -1)
-          if ! curl -fsSL "https://index.crates.io/wo/rk/worktable_codegen" | sed -n 's/.*"vers":"\([^"]*\)".*/\1/p' | grep -qx "$cv"; then
-            cargo publish -p worktable_codegen
-          fi
-          cargo publish -p worktable
+          set -euo pipefail
+          if [ -z "${CARGO_REGISTRY_TOKEN:-}" ]; then echo "CARGO_REGISTRY_TOKEN not set; skipping publish"; exit 0; fi
+
+          # Publish in dependency order. worktable_codegen depends on
+          # worktable_dsl and worktable depends on worktable_codegen, both by
+          # path with an exact version, so each must be on the registry before
+          # the next is packaged. Omitting worktable_dsl here is what made
+          # `cargo publish -p worktable_codegen` fail with "no matching package
+          # named `worktable_dsl` found" the moment the DSL extraction landed.
+          publish_if_new() {
+            crate="$1"
+            manifest="$2"
+            version=$(sed -n 's/^version = "\(.*\)"$/\1/p' "$manifest" | head -1)
+            # crates.io index paths: four or more characters is {first two}/{next two}/{name}.
+            if curl -fsSL "https://index.crates.io/wo/rk/$crate" \
+              | sed -n 's/.*"vers":"\([^"]*\)".*/\1/p' | grep -qx "$version"; then
+              echo "$crate $version is already on crates.io; skipping"
+              return 0
+            fi
+            echo "publishing $crate $version"
+            cargo publish -p "$crate"
+          }
+
+          publish_if_new worktable_dsl dsl/Cargo.toml
+          publish_if_new worktable_codegen codegen/Cargo.toml
+          publish_if_new worktable Cargo.toml

AGENTS.md

@@ -35,6 +35,10 @@ success.
   error is not something you introduced, and saying so requires checking.
 - A build that finishes suspiciously fast was cached, not rebuilt. Force a real rebuild when
   the rebuild is the thing you're verifying.
+- **`ci-local.sh` runs your default toolchain; CI runs the newest stable.** A clippy lint
+  added between the two is invisible here and fails there, on code nobody touched. The
+  script prints both so a local pass is not mistaken for a CI pass. If they differ and the
+  lint matters, get the toolchain updated rather than pushing to find out.
 
 ## PR discipline
 

Cargo.toml

@@ -1,9 +1,9 @@
 [workspace]
-members = ["codegen", "examples", "performance_measurement", "performance_measurement/codegen"]
+members = ["codegen", "dsl", "examples", "performance_measurement", "performance_measurement/codegen"]
 
 [package]
 name = "worktable"
-version = "1.0.0-beta.16"
+version = "1.0.0-beta.17"
 edition = "2024"
 authors = ["Handy-caT"]
 license = "MIT"
@@ -33,49 +33,56 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1.89"
-arctic = { package = "arctic-wt", version = "0.1.6" }
-congee = { package = "congee-wt", version = "0.4.3" }
-convert_case = "0.6.0"
-crc32fast = "1.5.0"
+async-trait = "0.1"
+arctic = { package = "arctic-wt", version = "0.1" }
+congee = { package = "congee-wt", version = "0.4" }
+convert_case = "0.6"
+crc32fast = "1"
 # Already in the dependency graph transitively (indexset's concurrent
 # structures); used directly for read-side grace periods.
-data_bucket = "=0.5.5"
+data_bucket = "0.5"
 # data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" }
 # data_bucket = { path = "../DataBucket", version = "0.3.14" }
-derive_more = { version = "2.0.1", features = ["from", "error", "display", "debug", "into"] }
-eyre = "0.6.12"
-fastrand = "2.3.0"
-futures = "0.3.30"
-indexset = { package = "WorkTablesIndex", version = "=0.0.9", default-features = false, features = ["concurrent", "cdc", "multimap"] }
-vanilla_indexset = { package = "indexset", version = "=0.15.0", features = ["concurrent", "cdc", "multimap"] }
+derive_more = { version = "2", features = ["from", "error", "display", "debug", "into"] }
+eyre = "0.6"
+fastrand = "2"
+futures = "0.3"
+indexset = { package = "WorkTablesIndex", version = "0.0", default-features = false, features = ["concurrent", "cdc", "multimap"] }
+vanilla_indexset = { package = "indexset", version = "0.15", features = ["concurrent", "cdc", "multimap"] }
 # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] }
 # indexset = { package = "wt-indexset", version = "=0.12.12", features = ["concurrent", "cdc", "multimap"] }
-log = "0.4.29"
-ordered-float = "5.0.0"
-parking_lot = "0.12.3"
+log = "0.4"
+ordered-float = "5"
+parking_lot = "0.12"
 performance_measurement = { path = "performance_measurement", version = "0.1.0", optional = true }
 performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true }
-prettytable-rs = "^0.10"
-psc-nanoid = { version = "3.1.1", features = ["rkyv", "packed"] }
-rkyv = { version = "0.8.17", features = ["uuid-1"] }
+prettytable-rs = "0.10"
+psc-nanoid = { version = "3", features = ["rkyv", "packed"] }
+rkyv = { version = "0.8", features = ["uuid-1"] }
 reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] }
-ps-reclaim = "0.1.0"
-rustc-hash = "2.1.1"
-rusty-s3 = { version = "0.10.2", optional = true }
-smart-default = "0.7.1"
+ps-reclaim = "0.1"
+rustc-hash = "2"
+rusty-s3 = { version = "0.10", optional = true }
+smart-default = "0.7"
 tokio = { version = "1", features = ["full"] }
 tracing = "0.1"
 url = { version = "2", optional = true }
-uuid = { version = "1.24.0", features = ["v4", "v7"] }
+uuid = { version = "1", features = ["v4", "v7"] }
 walkdir = { version = "2", optional = true }
-worktable_codegen = { path = "codegen", version = "=1.0.0-beta.15" }
+# Exact for the same reason as `worktable_dsl` in codegen/Cargo.toml: carets do
+# not match pre-releases, and these move as one train.
+worktable_codegen = { path = "codegen", version = "=1.0.0-beta.17" }
+# Re-exported below. Each generated table carries its declaration as a const
+# whose documentation says to read it with `worktable_dsl::Schema::parse`; that
+# instruction is only true if a plain `worktable` dependency can reach the
+# crate. Exact for the same pre-release reason as codegen.
+worktable_dsl = { path = "dsl", version = "=1.0.0-beta.18" }
 
 [dev-dependencies]
-chrono = "0.4.43"
+chrono = "0.4"
 criterion = { version = "0.5", features = ["async_tokio"] }
-rand = "0.9.1"
-tracing-subscriber = "0.3.23"
+rand = "0.9"
+tracing-subscriber = "0.3"
 
 # Only under `--cfg loom`, so a normal build and a normal `cargo test` never
 # resolve it. See src/partition/loom_tests.rs for how to run the models.

benches/cases/full_featured.rs

@@ -19,7 +19,7 @@ fn insert(c: &mut Criterion) {
                     another: format!("another_{}", fastrand::u64(..)),
                     something: fastrand::u64(..),
                 };
-                table.insert(black_box(row))
+                futures::executor::block_on(table.insert(black_box(row)))
             },
             BatchSize::SmallInput,
         )
@@ -37,7 +37,7 @@ fn select_by_pk(c: &mut Criterion) {
                 another: format!("another_{}", fastrand::u64(..)),
                 something: fastrand::u64(..),
             };
-            table.insert(row).unwrap()
+            futures::executor::block_on(table.insert(row)).unwrap()
         })
         .collect();
 
@@ -60,7 +60,7 @@ fn select_by_unique_index(c: &mut Criterion) {
             another: format!("another_{}", i),
             something: i,
         };
-        table.insert(row).unwrap();
+        futures::executor::block_on(table.insert(row)).unwrap();
     }
 
     c.bench_function("full_featured_select_by_val1", |b| {
@@ -82,7 +82,7 @@ fn select_by_non_unique_index(c: &mut Criterion) {
             another: format!("cat_{}", i % 10),
             something: i,
         };
-        table.insert(row).unwrap();
+        futures::executor::block_on(table.insert(row)).unwrap();
     }
 
     c.bench_function("full_featured_select_by_another", |b| {
@@ -107,7 +107,7 @@ fn update(c: &mut Criterion) {
                 another: format!("another_{}", i),
                 something: i,
             };
-            pks.push(table.insert(row).unwrap());
+            pks.push(table.insert(row).await.unwrap());
         }
         pks
     });
@@ -141,7 +141,7 @@ fn update_by_pk_query(c: &mut Criterion) {
                 another: format!("another_{}", i),
                 something: i,
             };
-            table.insert(row).unwrap();
+            table.insert(row).await.unwrap();
         }
     });
 
@@ -169,7 +169,7 @@ fn update_by_unique_index_query(c: &mut Criterion) {
                 another: format!("another_{}", i),
                 something: i,
             };
-            table.insert(row).unwrap();
+            table.insert(row).await.unwrap();
         }
     });
 
@@ -196,7 +196,7 @@ fn in_place_update(c: &mut Criterion) {
             another: "test".to_string(),
             something: 0,
         };
-        table.insert(row).unwrap().into()
+        futures::executor::block_on(table.insert(row)).unwrap().into()
     };
 
     c.bench_function("full_featured_in_place_update_val", |b| {
@@ -219,7 +219,7 @@ fn delete(c: &mut Criterion) {
                     another: format!("temp_{}", fastrand::u64(..)),
                     something: fastrand::u64(..),
                 };
-                table.insert(row).unwrap()
+                futures::executor::block_on(table.insert(row)).unwrap()
             },
             |pk: FullFeaturedPrimaryKey| rt.block_on(async { table.delete(black_box(pk)).await.unwrap() }),
             BatchSize::SmallInput,
@@ -242,7 +242,7 @@ fn delete_by_index_query(c: &mut Criterion) {
                     another: another.clone(),
                     something: fastrand::u64(..),
                 };
-                table.insert(row).unwrap();
+                futures::executor::block_on(table.insert(row)).unwrap();
                 another
             },
             |another: String| rt.block_on(async { table.delete_by_another(another).await.unwrap() }),
@@ -319,7 +319,7 @@ fn batch_insert(c: &mut Criterion) {
                             another: format!("another_{}", i),
                             something: i as u64,
                         };
-                        table.insert(black_box(row)).unwrap();
+                        futures::executor::block_on(table.insert(black_box(row))).unwrap();
                     }
                     black_box(table)
                 },
@@ -347,7 +347,7 @@ fn batch_select_pk(c: &mut Criterion) {
                     another: format!("another_{}", fastrand::u64(..)),
                     something: fastrand::u64(..),
                 };
-                table.insert(row).unwrap()
+                futures::executor::block_on(table.insert(row)).unwrap()
             })
             .collect();