Schema IR for the WorkTable designer: read a declaration as data, diff it, emit DSL and UML

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["codegen", "examples", "performance_measurement", "performance_measurement/codegen"]
+members = ["codegen", "dsl", "examples", "performance_measurement", "performance_measurement/codegen"]
 
 [package]
 name = "worktable"

codegen/Cargo.toml

@@ -18,6 +18,9 @@ path = "src/lib.rs"
 proc-macro = true
 
 [dependencies]
+# The schema language, extracted so consumers other than this macro can read
+# a declaration. See its crate docs for why that needed a separate crate.
+worktable_dsl = { path = "../dsl", version = "1.0.0-beta.14" }
 rkyv = { version = "0.8.17" }
 syn = { version = "2.0.74", features = ["full"] }
 quote = "1.0.36"

codegen/src/common/mod.rs

@@ -1,7 +1,13 @@
-pub mod model;
+//! What stayed behind when the schema language moved out.
+//!
+//! `model` and `parser` are `worktable_dsl` now, so anything can read a
+//! declaration. `name_generator` is not part of that language: it invents Rust
+//! identifiers for generated code, which is this crate's concern and nobody
+//! else's.
+//!
+//! It also could not have gone. Generators here define inherent `impl`s on
+//! `WorktableNameGenerator`, and the orphan rule forbids that for a type owned
+//! by another crate. The compiler makes the same argument the design does.
 pub mod name_generator;
-pub mod parser;
 
-#[allow(unused_imports)]
-pub use model::*;
-pub use parser::Parser;
+pub use worktable_dsl::{Parser, model, parser};

codegen/src/common/name_generator.rs

@@ -120,6 +120,18 @@ impl WorktableNameGenerator {
         )
     }
 
+    /// The name of the const carrying the table's own declaration.
+    ///
+    /// It follows `get_version_const_ident`'s shape because it answers the
+    /// question next to it: the version says *which* schema, and this says
+    /// *what* that schema is.
+    pub fn get_schema_const_ident(&self) -> Ident {
+        let upper_snake_case_name = self.name.from_case(Case::Pascal).to_case(Case::UpperSnake);
+        Ident::new(
+            format!("{}_SCHEMA", upper_snake_case_name.to_uppercase()).as_str(),
+            Span::mixed_site(),
+        )
+    }
     pub fn get_space_secondary_index_ident(&self) -> Ident {
         Ident::new(format!("{}SpaceSecondaryIndex", self.name).as_str(), Span::mixed_site())
     }

codegen/src/lib.rs

@@ -1,3 +1,11 @@
+// `common` is now a thin front for `worktable_dsl`, which holds the schema
+// model and parser so that anything other than this macro can read a
+// declaration. Kept as a module rather than an alias because the name
+// generator stays here: generators define inherent `impl`s on it, which the
+// orphan rule allows only in the crate that owns the type.
+//
+// The 127 `crate::common::` paths across this crate are unchanged, so the diff
+// is a move rather than a sweep.
 mod common;
 mod generators;
 mod mem_stat;

codegen/src/worktable/mod.rs

@@ -2,8 +2,17 @@ use proc_macro2::TokenStream;
 
 use crate::common::Parser;
 use crate::common::model::{Columns, IndexBackend, Persistence};
+use crate::common::name_generator::WorktableNameGenerator;
 
 pub fn expand(input: TokenStream) -> syn::Result<TokenStream> {
+    // Keep the tokens. The declaration is read a second time at the end, as
+    // data, so the generated table can carry its own schema. It happens at the
+    // end rather than here so that this function's diagnostics are the ones a
+    // bad declaration produces: both parses reject the same inputs, but only
+    // one of them knows to say that a separate `attributes` section is not
+    // part of the 1.0 grammar.
+    let declaration = input.clone();
+
     let mut parser = Parser::new(input);
     let mut columns = None;
     let mut queries = None;
@@ -84,9 +93,42 @@ pub fn expand(input: TokenStream) -> syn::Result<TokenStream> {
         generated.extend(crate::generators::partitions::expand(&name, &key, persistence));
     }
 
+    generated.extend(gen_schema_const(&worktable_dsl::Schema::from_tokens(declaration)?));
+
     Ok(generated)
 }
 
+/// Bake the declaration into the generated code, as the text it was written in.
+///
+/// The point is that a compiled binary should be able to say what schema it was
+/// built against, without the source. A migration planner needs it as the
+/// "declared" side of a comparison against what is on disk; a designer needs it
+/// to draw a diagram of an application it did not build.
+///
+/// The stored form is the DSL text rather than a serialised structure. It needs
+/// no format decision, no serde in the dependency graph of every user's build,
+/// and it is legible in a hex dump; `worktable_dsl` reads it back with the same
+/// parser that read the original, and `dsl/tests/round_trip.rs` holds that
+/// property against all 116 declarations in this repository.
+///
+/// `allow(dead_code)` because a `worktable!` inside a function body puts this
+/// const inside that body, where nothing refers to it and `-D warnings` would
+/// otherwise fail a user's build over a const they never asked for.
+fn gen_schema_const(schema: &worktable_dsl::Schema) -> TokenStream {
+    let ident = WorktableNameGenerator::from_table_name(schema.name.clone()).get_schema_const_ident();
+    let text = schema.to_dsl();
+    let doc = format!(
+        "The `worktable!` declaration `{}` was generated from, as text. Read it with `worktable_dsl::Schema::parse`.",
+        schema.name
+    );
+
+    quote::quote! {
+        #[doc = #doc]
+        #[allow(dead_code)]
+        pub const #ident: &str = #text;
+    }
+}
+
 /// data_bucket's on-disk layer seeks with its own hardcoded `PAGE_SIZE` of
 /// 16384 bytes (`seek_to_page_start`, `seek_by_link`, `persist_page`), while
 /// the generated table threads the user's `page_size` through its page-id and
@@ -704,3 +746,253 @@ mod position_tests {
         );
     }
 }
+
+/// What a designer needs from the schema IR: a declaration that has been read
+/// into [`worktable_dsl::Schema`] and written back out is a declaration this
+/// macro accepts.
+///
+/// `worktable_dsl` can test its own round trip, which shows nothing was lost
+/// between its parser and its emitter. It cannot show that the text it emits
+/// is a declaration *this* macro accepts, because it cannot call this macro:
+/// that check has to live on the near side of the proc-macro boundary.
+///
+/// The stronger claim — that the emitted declaration generates the *same code*
+/// — is not asserted here, and cannot be until the generator is deterministic.
+/// See `the_same_declaration_expands_the_same_way_twice` below, which is
+/// ignored because it currently fails on unmodified code.
+#[cfg(test)]
+mod emitted_declarations {
+    use quote::quote;
+    use worktable_dsl::Schema;
+
+    use super::expand;
+
+    fn survives_the_round_trip(declaration: proc_macro2::TokenStream) {
+        expand(declaration.clone()).expect("the original expands");
+
+        let schema = Schema::from_tokens(declaration).expect("the IR reads it");
+        let emitted = schema.to_dsl();
+        let reparsed: proc_macro2::TokenStream = syn::parse_str(&emitted)
+            .unwrap_or_else(|error| panic!("emitted text does not tokenise: {error}\n{emitted}"));
+
+        assert_eq!(
+            Schema::from_tokens(reparsed.clone()).expect("the emitted text reads back"),
+            schema,
+            "the emitted declaration describes a different schema\n{emitted}"
+        );
+        expand(reparsed).unwrap_or_else(|error| panic!("the emitted declaration does not expand: {error}\n{emitted}"));
+    }
+
+    #[test]
+    fn a_minimal_declaration() {
+        survives_the_round_trip(quote! {
+            name: Minimal,
+            columns: { id: u64 primary_key },
+        });
+    }
+
+    #[test]
+    fn a_persisted_declaration_with_indexes_and_queries() {
+        survives_the_round_trip(quote! {
+            name: Account,
+            version: 3,
+            persist: true,
+            columns: {
+                id: u64 primary_key autoincrement,
+                email: String,
+                tenant: u64,
+                nickname: String optional,
+                balance: f64,
+            },
+            indexes: {
+                email_idx: email unique,
+                tenant_idx: tenant,
+            },
+            queries: {
+                update: {
+                    Nickname(nickname) by id,
+                    Email(email) by tenant,
+                }
+                delete: {
+                    ById() by id,
+                }
+                in_place: {
+                    Balance(balance) by id,
+                }
+            }
+        });
+    }
+
+    #[test]
+    fn a_partitioned_declaration() {
+        survives_the_round_trip(quote! {
+            name: Price,
+            partition_by: symbol_id: u16,
+            columns: {
+                exchange_id: u8 primary_key,
+                bid: f64,
+            },
+        });
+    }
+
+    #[test]
+    fn a_composite_key_keeps_its_column_order() {
+        // The order of a composite key decides the field order of the
+        // generated `get_primary_key`, and so the layout of the key type. An
+        // emitter that wrote the columns back in a `HashMap`'s order would
+        // change it.
+        survives_the_round_trip(quote! {
+            name: CompositeKey,
+            persist: true,
+            columns: {
+                tenant_id: u64 primary_key,
+                record_id: u64 primary_key,
+                value: i64,
+            },
+        });
+    }
+
+    #[test]
+    fn an_explicit_backend_and_a_custom_page_size() {
+        survives_the_round_trip(quote! {
+            name: Tuned,
+            persist: false,
+            columns: {
+                id: u64 primary_key using congee,
+                value: u64,
+            },
+            indexes: {
+                value_idx: value unique using arctic,
+            },
+            config: {
+                page_size: 1024,
+                row_derives: Clone, Debug,
+            }
+        });
+    }
+}
+
+#[cfg(test)]
+mod generator_determinism {
+    use quote::quote;
+
+    use super::expand;
+
+    /// Expanding one declaration twice must produce one program. It does not.
+    ///
+    /// `Columns::columns_map` is a `std::collections::HashMap`, and several
+    /// generators iterate it directly to emit an ordered construct: the
+    /// `RowFields` enum and the `AvaiableTypes` enum among them. `RandomState`
+    /// seeds each map instance differently, so two expansions of the same
+    /// declaration in the same process emit those variants in different
+    /// orders, and two compilations of the same source can too.
+    ///
+    /// This is ignored rather than deleted because it is the evidence. It is
+    /// ignored rather than failing because the fix — ordering `columns_map`,
+    /// which `field_positions` already records the order for — changes the
+    /// generated code of every table and is a change to review on its own,
+    /// not a side effect of adding an emitter.
+    ///
+    /// Run it with `cargo test -p worktable_codegen -- --ignored`.
+    #[test]
+    #[ignore = "records a known generator bug: columns_map is a HashMap, so expansion is not deterministic"]
+    fn the_same_declaration_expands_the_same_way_twice() {
+        let declaration = quote! {
+            name: Twice,
+            persist: true,
+            columns: {
+                id: u64 primary_key autoincrement,
+                email: String,
+                tenant: u64,
+                balance: f64,
+            },
+        };
+
+        let first = expand(declaration.clone()).expect("expands").to_string();
+        let second = expand(declaration).expect("expands").to_string();
+        assert_eq!(first, second);
+    }
+}
+
+/// The generated table carries its own declaration.
+#[cfg(test)]
+mod schema_const {
+    use proc_macro2::{TokenStream, TokenTree};
+    use quote::quote;
+    use worktable_dsl::Schema;
+
+    use super::expand;
+
+    /// Pull the string out of `pub const <NAME>: &str = "..";` in generated code.
+    fn baked_schema(generated: TokenStream, const_name: &str) -> String {
+        let mut trees = generated
+            .into_iter()
+            .skip_while(|tree| !matches!(tree, TokenTree::Ident(ident) if ident == const_name));
+        assert!(trees.next().is_some(), "no `{const_name}` const in the generated code");
+        for tree in trees {
+            if let TokenTree::Literal(literal) = tree {
+                let text = literal.to_string();
+                return syn::parse_str::<syn::LitStr>(&text).expect("a string literal").value();
+            }
+        }
+        panic!("`{const_name}` has no value");
+    }
+
+    #[test]
+    fn a_persisted_table_carries_the_declaration_it_was_built_from() {
+        let declaration = quote! {
+            name: Account,
+            version: 3,
+            persist: true,
+            columns: {
+                id: u64 primary_key autoincrement,
+                email: String,
+                nickname: String optional,
+            },
+            indexes: { email_idx: email unique },
+            queries: { update: { Nickname(nickname) by id } }
+        };
+
+        let baked = baked_schema(expand(declaration.clone()).expect("expands"), "ACCOUNT_SCHEMA");
+
+        assert_eq!(
+            Schema::parse(&baked).expect("the baked text parses"),
+            Schema::from_tokens(declaration).expect("the declaration parses"),
+            "the baked declaration is not the one the table was generated from"
+        );
+    }
+
+    #[test]
+    fn an_in_memory_table_carries_it_too() {
+        // A designer reading a crate wants every table, not only the persisted
+        // ones, and the const costs a string either way.
+        let declaration = quote! {
+            name: Price,
+            partition_by: symbol_id: u16,
+            columns: { exchange_id: u8 primary_key, bid: f64 },
+        };
+
+        let baked = baked_schema(expand(declaration.clone()).expect("expands"), "PRICE_SCHEMA");
+
+        assert_eq!(
+            Schema::parse(&baked).expect("the baked text parses"),
+            Schema::from_tokens(declaration).expect("the declaration parses"),
+        );
+    }
+
+    #[test]
+    fn the_baked_text_is_a_declaration_the_macro_accepts() {
+        // Which is what makes it usable as the old table definition a
+        // migration would otherwise need kept by hand.
+        let declaration = quote! {
+            name: Regenerated,
+            persist: true,
+            columns: { id: u64 primary_key autoincrement, payload: String },
+            indexes: { payload_idx: payload unique }
+        };
+
+        let baked = baked_schema(expand(declaration).expect("expands"), "REGENERATED_SCHEMA");
+        let reparsed: TokenStream = syn::parse_str(&baked).expect("tokenises");
+        expand(reparsed).expect("the baked declaration expands");
+    }
+}