diff --git a/Cargo.toml b/Cargo.toml index e32eb50c..e18d14ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"] # pointer-only fast path. Publication is append-only and asserted at each swap. arc-swap = "1" async-trait = "0.1" -arctic = { package = "arctic-wt", version = "^0.1, >=0.1.9", default-features = false, features = ["smr-ps-reclaim"] } +arctic = { package = "arctic-wt", version = "^0.1, >=0.1.11", default-features = false, features = ["smr-ps-reclaim"] } congee = { package = "congee-wt", version = "^0.4, >=0.4.4" } convert_case = "0.6" crc32fast = "1" @@ -59,7 +59,7 @@ prettytable-rs = "0.10" psc-nanoid = { version = "3", features = ["rkyv", "packed"] } rkyv = { version = "0.8", features = ["uuid-1"] } reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] } -ps-reclaim = { version = "^0.1, >=0.1.3" } +ps-reclaim = { version = "^0.1, >=0.1.4" } rustc-hash = "2" rusty-s3 = { version = "0.10", optional = true } smart-default = "0.7" diff --git a/docs/paper-2-plan.md b/docs/paper-2-plan.md new file mode 100644 index 00000000..474414be --- /dev/null +++ b/docs/paper-2-plan.md @@ -0,0 +1,115 @@ +# Paper two: plan and evidence map + +Written 2026-09-06 against master `d5656e6` (1.0.0-beta.19). Companion to the CIDR 2027 +submission (beta.6, submitted for the 2026-08-04 deadline; notification 2026-10-06). + +## Thesis candidates + +The CIDR paper argued *compile-time engine specialization*. It deferred (§5, §7): the lock +discipline scaling comparison, crash consistency, formal checking of the protocols, the cost +of monomorphization, and external baselines beyond redb/LMDB. Paper two should be the paper +those deferrals point at, not a re-statement of the thesis. + +**A. Lifecycle paper (recommended).** "Row lifecycle in a specialized engine: exact-cell +locking, epoch reclamation, and reactive vacuum without a transaction manager." Everything +in it has landed since beta.6 and has numbers. Contributions: + +1. Exact-cell synchronization replacing hashed stripes and the table-global barrier + (`src/in_memory/data.rs` `CellLocks`; `docs/versioned-row-publication.md`). +2. Quiescent-state reclamation via a one-word `!Send` guard (`ps-reclaim`), replacing a + global reader counter; the `Send`-guard use-after-free found on the way is a good + cautionary section (`docs/TODO.md` "ps-reclaim 0.1.1"). +3. Reactive vacuum planned from the free-range registry, gated by a mutation lease and a + quiet epoch; root-cause note that all four reclaim bugs came from indexes storing + physical addresses (`docs/vacuum-design-directions.md`, `src/table/vacuum/`). +4. The 1-32 thread grid across three index backends: the lock-discipline scaling result the + CIDR paper promised (`docs/beta18-validation.md`). +5. `partition_by` with the loom model of the slot protocol (`src/partition/loom_tests.rs`), + the first model-checked component; HFT review that produced `partition_ref` + (`wt-review.md`). + +**B. Persistence paper.** "Index persistence by replaying the tree's own CDC stream, and what +it costs." Needs the durability work first: structural CDC is ~28% of a 126 ns insert +(`docs/wti-dirty-generation-persistence-plan.md`); ART logical WAL exists +(`src/persistence/space/art_index.rs`) but data pages and WTI still end at `flush()`; the +watermark/fsync design is proposal only (`docs/durability-visibility-proposal.md`). Not +ready before a Q1 2027 deadline unless the journal lands. + +**C. Schema-as-IR paper (PL venue).** `worktable_dsl`: schema read as data, diff with a +per-change cost model, declarations baked into generated code, `wt-dsl` CLI, migration +engine with on-disk version detection (`dsl/`, `docs/migration.md`). Fits PEPM / OOPSLA +better than a DB venue; overlaps with the PEPM plan in the other session. + +## Evidence already in hand (all M4 Max, local trees; needs a pinned Linux rerun) + +| Claim | Number | Source | +|---|---|---| +| Hot-page writer, exact-cell vs hashed | 322 ns vs 1,536 ns | beta18-validation | +| Per-page vs table-global barrier | +25% @4, +65% @8 disjoint writers | versioned-row-publication | +| Generation retire, beta.18 vs 15 | 33.6x (WTI), 16.1x (Arctic) | beta18-validation | +| Read scaling best/1T at 16 threads | 3.6x / 3.3x / 2.6x per backend | beta18-validation | +| 10%-write mix ceiling | peaks at 4 threads | beta18-validation (also beta.13/15) | +| Reactive vacuum foreground penalty | -1.1..-5.1% vs 16-41% unpaced | beta17/18-validation | +| Vacuum reclamation | 18/18 cells, 196 pages, 100% | beta17-validation | +| Point lookup vs stripped index | 15.65 ns vs 9.25 ns bare Arctic vs 33.5 Vec+BTreeMap | beta18-validation | +| Memory overhead | 0.14 B/row over control | beta18-validation | +| Partition route | 0.73 ns Vec vs 9.5 ns string hash; `partition_ref` 3.35 ns | TODO.md, partition docs | +| Persisted insert, beta.15 to 18 | -33..-42% (Arctic 6,130 to 3,753 ns/row) | beta18-validation | +| CDC share of insert | ~35 ns of 126 ns | wti-dirty-generation plan | + +Not in hand: `paper-bench/results/` (never committed), `compile_cost.sh` never run, no +sled/SQLite/DashMap baselines, no beta.19 rerun of Table 2. + +## Consumer workloads (surveyed 2026-09-06, all under ~/code) + +| Repo | Shape | What it gives the paper | Open? | +|---|---|---|---| +| `agentcode` | 11 tables, 8 persisted, Arctic on nearly every index, u128 keys. Stress = 8 concurrent 800-file `update_latency` processes. | The concurrency bug that motivates the paper: `docs/known-defects.md:70-137`, torn/corrupt page header in secondary-index batch apply at beta.11, 6/8 runs failing, 28/28 after moving to Arctic. Map it to the beta.18 fix ("torn reads and premature physical-link reuse") and show the same harness clean on beta.19. Also: Arctic vs WTI at 20k rows, insert 2.18M/s vs 0.89M/s, lookup 17.0M/s vs 5.0M/s (`docs/benchmarks/index-backends.md`); state 42.4 to 22.2 MB after u128 keys (`state-growth.md`); the request for a non-unique fixed-width index that became `ArcticMultiIndex`. | Proprietary | +| `agencyzero` | 19 persisted tables, String PKs, migration engine, `LoadMode::Recovery`, single-writer flock, QA fixture of 248 projects (~30 MB store). | The reclamation case study: `docs/store-recovery.md` records four production corruptions, including a variable-width index page that forgot fragmentation across restart (174 live entries, 2,664 B dead, 64 B tail overlap) and beta.5 whole-row rebuilds churning `pr_project_idx` (38 disagreeing rows) fixed by in-place updates. Production migrations via schema fingerprint. This is the "why indexes must not store physical addresses" story with real data. | Private (GitHub) | +| `karen` | 2 persisted tables, tiny (~700 rows). | Row-level, queryable, durable learned session state instead of one blob: the per-turn Confirm write-through gives 15-20 points top-1. One paragraph of motivation, not evaluation. Uses `unload_gracefully`. | Closed | +| `ekopathrs` | No `worktable!` at all. Uses `worktable-vec::AtomicKeyTable` for two in-memory profiling tables. | The honest negative: `docs/STORAGE-REVIEW.md` rejects full WorkTable (318-package resolve, no `no_std`) for 399 entries. Cite as the boundary of the design space; `worktable-vec` is the lock-free, `no_std` sibling. | Private | + +Use agentcode as the headline stress workload in §4 alongside the shadow-state harness; use +agencyzero as the recovery/fragmentation case study; mention karen and ekopathrs in one +paragraph each in the experience section. Get written OK before naming private repos. + +## Gaps to close for option A + +- Rerun the beta.18 grid and `paper-bench` on a quiet pinned x86 box; commit `results/`. +- Lock-discipline ablation as a proper figure: field vs row vs table lock, 1-32 threads, + skewed keys (`paper-bench/src/bin/contention`). +- Semi-formal statement of the cell-lock + reclamation invariants; ideally extend loom + beyond partitions to the cell/retire path (the CIDR reviewers will ask). +- Wart sweep: 9 `todo!()` sites remain (`codegen/.../queries/in_place.rs`, `update.rs`, + `src/features/s3_support.rs`), `Avaiable` typo in 2 files. +- Merge or explicitly exclude `feat/columnar-fields-indexes` (branch, Aug 6, unmerged). + +## Target: EDBT 2027, 3rd cycle (verified 2026-09-06) + +- Submission **2026-10-07, 5pm PST** (31 days out). Author feedback 11-19, notification + Acc/Rej/Revise 12-05, revised paper 2027-01-04, final 01-27, camera-ready 02-10. + Conference Lille, April 6-9, 2027. +- Paper types: Research long (12p) or short (6p, title prefixed "[Short Paper]"), + Experiments & Analysis, Vision (6p). Topics list includes "Concurrency control, recovery, + and transaction management", "Storage, indexing, and physical database design", + "Data management on modern hardware", "Benchmarking and performance evaluation". +- The revise cycle matters: a paper that gets "revise" on Dec 5 has until Jan 4 to add + the pinned-Linux rerun, so the Oct 7 draft can ship on the M4 grid with the caveat stated. +- CIDR notification is Oct 6, one day before: paper two cannot depend on the outcome and + must not overlap the CIDR text (still under review until then). Option A is disjoint by + construction; cite the CIDR paper as "under submission". + +Alternatives if A slips: ICDE 2027 R2 (2026-11-11), PVLDB rolling (monthly to 2027-03-01), +SIGMOD R4 (2026-10-17). DaMoN 2027 CFP not posted. + +## 31-day schedule for option A (long paper) + +| Week | Dates | Deliverable | +|---|---|---| +| 1 | Sep 7-13 | Freeze the claim list. Run `paper-bench` contention (field/row/table/inplace, 1-32 tasks) and beta.18 grid on the pinned Linux box if available, else M4 with three rotated passes; commit `results/`. Decide long vs short by Sep 13 based on whether the scaling figure holds. | +| 2 | Sep 14-20 | Draft §2 protocols (cell lock, retire, vacuum lease) with invariants stated; §3 partition + loom. Wart sweep PR (`todo!()`, `Avaiable`). | +| 3 | Sep 21-27 | Draft §4 evaluation from `results/`; figures; related work (Hekaton, epoch/QSBR: Fraser, Hart et al., Bw-tree, OLC, DaMoN vacuum/compaction lineage). | +| 4 | Sep 28-Oct 4 | Full read-through, internal review, page trim to 12. | +| 5 | Oct 5-7 | Buffer. Submit by Oct 6 evening local time (Oct 7 5pm PST is 07:00 Oct 8 in Bangkok, but do not use it). | + +Short-paper fallback (6p): contributions 1-3 only, one scaling figure, one vacuum figure. diff --git a/tests/slotted_page_requirement.rs b/tests/slotted_page_requirement.rs new file mode 100644 index 00000000..8b21460d --- /dev/null +++ b/tests/slotted_page_requirement.rs @@ -0,0 +1,204 @@ +//! What a data page has to say about itself, for beta.20. +//! +//! # Where this comes from +//! +//! beta.19 changed the on-disk format and every existing `.wt.data` had to be +//! thrown away and rebuilt, because nothing could read the old shape. That is a +//! regeneration event, and it happened on 6 September 2026 across every store on +//! this machine. +//! +//! It does not have to happen again. The reason it did is that a data page +//! cannot be read without the index that points into it: +//! +//! ```ignore +//! pub struct DataPage { +//! pub length: u32, +//! pub data: [u8; DATA_LENGTH], +//! } +//! ``` +//! +//! Rows are bump allocated into `data` and `length` is a high water mark. There +//! are no delimiters, so nothing can tell where one row ends and the next +//! begins. `empty_links_list` in the `SpaceInfoPage` records freed ranges and is +//! explicitly lossy: `bound_empty_links_list` truncates it when it outgrows the +//! info page and logs "space leak, not corruption". +//! +//! **The schema is already there and this is not asking for it again.** +//! `SpaceInfoPage` carries `row_schema`, `primary_key_fields` and +//! `secondary_index_types`, and `ensure_schema` refuses a mismatch by name. A +//! reader already knows how to decode a row. What it cannot do is find one. +//! +//! # The two things, in order of how much they matter +//! +//! 1. **A row directory in the data page**, the usual slotted layout: an +//! `(offset, length)` per row growing down from the end of the page, with a +//! count. Then a page describes itself, a reader needs no index, and the CRC +//! on that page validates the directory together with the rows it points at. +//! +//! 2. **A reader for the format beta.19 writes**, so beta.20 is an upgrade +//! rather than another regeneration. One already exists and is switched off: +//! `src/page/iterators.rs` in DataBucket, where `LinksIterator` walks index +//! pages for links and `DataIterator` follows them, decoding through +//! `row_schema`. It is 226 lines, commented out at `src/page/mod.rs:4`, and +//! enabling it produces nine errors that are bit rot rather than design: +//! `crate::IndexData` and `super::SpaceInfo` were renamed, and one call site +//! predates the API going async. +//! +//! # How the two fit together +//! +//! `DATA_VERSION` is 2 today and lives in every page's `GeneralHeader`, so it is +//! per page rather than per file. +//! +//! - **beta.20 ships both.** It writes 3 and reads 2 and 3. +//! - **beta.21 ships neither of the old ones.** The v2 path is deleted. +//! +//! So v2 is a one way ramp rather than dual support: a store is loaded through +//! it once, written back as v3, and never read that way again. It does not need +//! to be fast and it never needs append, which is most of why it is cheap. +//! +//! Two things to settle rather than discover: +//! +//! - Once a page has a directory and an index, both know where a row is and they +//! can disagree. One has to be authoritative. The directory is the better +//! candidate: it is local to the page and validated by the same CRC, where the +//! index is a separate structure with a different topology per backend. Under +//! `validate-reads` a load can compare the two and name a disagreement instead +//! of silently preferring one. +//! - Whether one file may hold both v2 and v3 pages. Per page versioning allows +//! it, which makes migration an append rather than a rewrite, but then no +//! reader may assume uniformity. +//! +//! # What is missing here, and is the next piece of work +//! +//! A committed `.wt.data` written by beta.19, so the ramp can be tested against +//! a real old file rather than against one this build just wrote. Until that +//! fixture exists, `a_store_reopens_without_being_rebuilt` below only proves the +//! current version reopens, which is the weaker half. + +use worktable::prelude::*; +use worktable::worktable; + +worktable!( + name: SlottedRow, + version: 1, + persist: true, + columns: { + id: u64 primary_key autoincrement, + blob: String, + }, +); + +/// Enough rows to fill more than one page, so a directory would be doing real +/// work rather than describing a single row. +const ROWS: u64 = 4_000; + +async fn filled(dir: &str) -> SlottedRowWorkTable { + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir).expect("a directory"); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("an engine"); + let table = SlottedRowWorkTable::load(engine).await.expect("a table"); + + for n in 0..ROWS { + table + .insert(SlottedRowRow { + id: table.get_next_pk().into(), + blob: format!("row {n}, long enough to make the page boundaries interesting"), + }) + .await + .expect("a row"); + } + table.wait_for_ops().await.expect("the queue drains"); + table +} + +fn data_file(dir: &str) -> std::path::PathBuf { + std::path::Path::new(dir) + .join(SlottedRowWorkTable::name_snake_case()) + .join(".wt.data") +} + +/// A data page should say where its rows are, without an index. +/// +/// **This is the beta.20 requirement.** The check goes straight at the bytes on +/// purpose. Reading the page through the engine would prove only that the index +/// still works, and the index is exactly what a self describing page is supposed +/// to make unnecessary. +/// +/// Written against bytes rather than against an API that does not exist yet, so +/// this file compiles today and fails on the missing behaviour rather than on a +/// missing symbol. +#[tokio::test] +#[ignore = "beta.20: a data page carries no row directory"] +async fn a_data_page_says_where_its_rows_are() { + let dir = "tests/data/slotted_page/self_describing"; + let table = filled(dir).await; + table.close().await.expect("the table closes"); + + let bytes = std::fs::read(data_file(dir)).expect("the file"); + assert!( + bytes.len() > PAGE_SIZE, + "the fixture has to span pages: {} bytes", + bytes.len() + ); + + // A slotted page keeps its directory at the end: a row count in the last + // four bytes, then that many (offset, length) pairs growing back up. Any + // layout would do; what matters is that something in the page delimits the + // rows. Today the tail is write padding, so this reads zero. + let mut described = 0usize; + let (pages, _) = bytes.as_chunks::(); + for page in pages.iter().skip(1) { + let mut tail = [0u8; 4]; + tail.copy_from_slice(&page[PAGE_SIZE - 4..]); + described += u32::from_le_bytes(tail) as usize; + } + + assert_eq!( + described, ROWS as usize, + "no page says how many rows it holds, so the {ROWS} rows in this file \ + cannot be found without the index. A row directory in the data page is \ + what makes a page readable on its own, and what makes the next format \ + change an upgrade instead of a regeneration." + ); + let _ = std::fs::remove_dir_all(dir); +} + +/// A store reopens without being deleted first. +/// +/// **Not ignored, and passing.** It guards the property at the current version, +/// so a format change that breaks reopening trips here rather than in somebody's +/// deploy. It is the weaker half of the requirement: proving beta.20 can read +/// beta.19 needs a beta.19 file committed as a fixture, which does not exist +/// yet. +#[tokio::test] +async fn a_store_reopens_without_being_rebuilt() { + let dir = "tests/data/slotted_page/reopen"; + let table = filled(dir).await; + table.close().await.expect("the table closes"); + + let engine = SlottedRowPersistenceEngine::new(DiskConfig::new_with_table_name( + dir, + SlottedRowWorkTable::name_snake_case(), + SlottedRowWorkTable::version(), + )) + .await + .expect("an engine"); + let reopened = SlottedRowWorkTable::load(engine) + .await + .expect("a store reopens rather than needing to be rebuilt"); + + assert_eq!( + reopened.select_all().execute().expect("a read").len(), + ROWS as usize, + "no rows are lost reopening a store" + ); + reopened.close().await.expect("the table closes"); + let _ = std::fs::remove_dir_all(dir); +}