refactor: extract the schema language into worktable_dsl

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["codegen", "examples", "performance_measurement", "performance_measurement/codegen"]
+members = ["codegen", "dsl", "examples", "performance_measurement", "performance_measurement/codegen"]
 
 [package]
 name = "worktable"

codegen/Cargo.toml

@@ -18,6 +18,9 @@ path = "src/lib.rs"
 proc-macro = true
 
 [dependencies]
+# The schema language, extracted so consumers other than this macro can read
+# a declaration. See its crate docs for why that needed a separate crate.
+worktable_dsl = { path = "../dsl", version = "1.0.0-beta.14" }
 rkyv = { version = "0.8.17" }
 syn = { version = "2.0.74", features = ["full"] }
 quote = "1.0.36"

codegen/src/common/mod.rs

@@ -1,7 +1,14 @@
-pub mod model;
+//! What stayed behind when the schema language moved out.
+//!
+//! `model` and `parser` are `worktable_dsl` now, so anything can read a
+//! declaration. `name_generator` is not part of that language: it invents Rust
+//! identifiers for generated code, which is this crate's concern and nobody
+//! else's.
+//!
+//! It also could not have gone. Generators here define inherent `impl`s on
+//! `WorktableNameGenerator`, and the orphan rule forbids that for a type owned
+//! by another crate. The compiler makes the same argument the design does.
 pub mod name_generator;
-pub mod parser;
 
-#[allow(unused_imports)]
-pub use model::*;
-pub use parser::Parser;
+pub use worktable_dsl::Parser;
+pub use worktable_dsl::{model, parser};

codegen/src/lib.rs

@@ -1,3 +1,11 @@
+// `common` is now a thin front for `worktable_dsl`, which holds the schema
+// model and parser so that anything other than this macro can read a
+// declaration. Kept as a module rather than an alias because the name
+// generator stays here: generators define inherent `impl`s on it, which the
+// orphan rule allows only in the crate that owns the type.
+//
+// The 127 `crate::common::` paths across this crate are unchanged, so the diff
+// is a move rather than a sweep.
 mod common;
 mod generators;
 mod mem_stat;

dsl/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "worktable_dsl"
+version = "1.0.0-beta.14"
+edition = "2024"
+license = "MIT"
+description = "The worktable! schema language: its model and parser, readable outside the proc macro"
+repository = "https://github.com/pathscale/WorkTable"
+
+[dependencies]
+# Exactly what `codegen/src/common` already used. The move adds no dependency
+# and drops none; anything else would make this a rewrite rather than a lift.
+syn = { version = "2.0.74", features = ["full"] }
+quote = "1.0.36"
+proc-macro2 = "1.0.86"
+convert_case = "0.6.0"
+indexmap = "2"
+
+[dev-dependencies]
+# The integration test builds as its own crate, which is what makes it evidence
+# that this one is consumable from outside.
+proc-macro2 = "1.0.86"

dsl/src/lib.rs

@@ -0,0 +1,43 @@
+//! The `worktable!` schema language: its model, and the parser that reads it.
+//!
+//! # Why this is its own crate
+//!
+//! This was `codegen/src/common`, inside `worktable_codegen`, which is declared
+//! `proc-macro = true`. A proc-macro crate can export nothing but macros, so
+//! every type here — the columns, the primary key, the indexes, the queries —
+//! was unreachable from any other crate no matter how public it was declared.
+//! `mod common` was not even public at that crate's root.
+//!
+//! The consequence was not theoretical. A schema is written down exactly once,
+//! in a `worktable!` invocation, and anything that wants to *read* one — a
+//! diagram, a migration tool, a documentation generator, an editor — could not
+//! reach the parser that already understood it. The available options were to
+//! re-implement the grammar and drift from it, or to do without.
+//!
+//! Nothing here changed in the move. The model and the parser are the ones the
+//! macro has always used, and the macro still uses these: `worktable_codegen`
+//! depends on this crate, so there is one grammar rather than a copy that can
+//! disagree with the compiler about what a schema means.
+//!
+//! # Reading a declaration
+//!
+//! ```ignore
+//! use worktable_dsl::Parser;
+//! use syn::parse_str;
+//!
+//! let tokens: proc_macro2::TokenStream = parse_str(source)?;
+//! let mut parser = Parser::new(tokens);
+//! let name = parser.parse_name()?;
+//! let columns = parser.parse_columns()?;
+//! ```
+//!
+//! The parser is token-based rather than textual, so comments and string
+//! literals are handled by `proc_macro2` rather than by hand. The schema files
+//! this reads are more comment than code, which makes that difference matter.
+
+pub mod model;
+pub mod parser;
+
+#[allow(unused_imports)]
+pub use model::*;
+pub use parser::Parser;

codegen/src/common/model/column.rs → dsl/src/model/column.rs

@@ -1,8 +1,8 @@
 use indexmap::IndexMap;
 use std::collections::HashMap;
 
-use crate::common::model::index::Index;
-use crate::common::model::{GeneratorType, IndexBackend};
+use crate::model::index::Index;
+use crate::model::{GeneratorType, IndexBackend};
 use proc_macro2::{Ident, TokenStream};
 use quote::quote;
 use syn::spanned::Spanned;

codegen/src/common/model/queries.rs → dsl/src/model/queries.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use proc_macro2::Ident;
 
-use crate::common::model::Operation;
+use crate::model::Operation;
 
 #[derive(Debug, Default)]
 pub struct Queries {

codegen/src/common/parser/attribute.rs → dsl/src/parser/attribute.rs

@@ -1,8 +1,8 @@
 use proc_macro2::TokenTree;
 use syn::spanned::Spanned as _;
 
-use crate::common::model::{PARTITION_KEY_TYPES, PartitionKey, Persistence};
-use crate::common::parser::Parser;
+use crate::model::{PARTITION_KEY_TYPES, PartitionKey, Persistence};
+use crate::parser::Parser;
 
 // TODO: Move this to separate attributes section because now it only parses persist.
 impl Parser {
@@ -97,8 +97,8 @@ impl Parser {
 mod tests {
     use quote::quote;
 
-    use crate::common::Parser;
-    use crate::common::model::{PARTITION_KEY_TYPES, PartitionKey, Persistence};
+    use crate::Parser;
+    use crate::model::{PARTITION_KEY_TYPES, PartitionKey, Persistence};
 
     #[test]
     fn test_empty() {

codegen/src/common/parser/columns.rs → dsl/src/parser/columns.rs

@@ -1,8 +1,8 @@
 use proc_macro2::{Delimiter, TokenTree};
 use syn::spanned::Spanned as _;
 
-use crate::common::Parser;
-use crate::common::model::{Columns, GeneratorType, Row};
+use crate::Parser;
+use crate::model::{Columns, GeneratorType, Row};
 
 impl Parser {
     pub fn parse_columns(&mut self) -> syn::Result<Columns> {
@@ -129,7 +129,7 @@ mod tests {
 
     use quote::quote;
 
-    use crate::common::Parser;
+    use crate::Parser;
 
     #[test]
     fn test_columns_parse() {
@@ -321,7 +321,7 @@ mod tests {
             let mut parser = Parser::new(row_tokens);
             let row = parser.parse_row().unwrap();
 
-            assert_eq!(row.index_backend, Some(crate::common::model::IndexBackend::Congee));
+            assert_eq!(row.index_backend, Some(crate::model::IndexBackend::Congee));
         }
 
         #[test]

codegen/src/common/parser/config.rs → dsl/src/parser/config.rs

@@ -3,8 +3,8 @@ use std::str::FromStr;
 use proc_macro2::{Delimiter, TokenTree};
 use syn::spanned::Spanned;
 
-use crate::common::Parser;
-use crate::common::model::Config;
+use crate::Parser;
+use crate::model::Config;
 
 const CONFIG_FIELD_NAME: &str = "config";
 

codegen/src/common/parser/index.rs → dsl/src/parser/index.rs

@@ -1,5 +1,5 @@
-use crate::common::Parser;
-use crate::common::model::{Index, IndexBackend};
+use crate::Parser;
+use crate::model::{Index, IndexBackend};
 use indexmap::IndexMap;
 use proc_macro2::{Delimiter, Ident, TokenTree};
 use syn::spanned::Spanned;
@@ -141,8 +141,8 @@ impl Parser {
 mod tests {
     use quote::quote;
 
-    use crate::common::Parser;
-    use crate::common::model::IndexBackend;
+    use crate::Parser;
+    use crate::model::IndexBackend;
 
     #[test]
     fn absent_using_defaults_to_worktables_index() {

codegen/src/common/parser/name.rs → dsl/src/parser/name.rs

@@ -2,7 +2,7 @@ use proc_macro2::Ident;
 use proc_macro2::TokenTree;
 use syn::spanned::Spanned as _;
 
-use crate::common::parser::Parser;
+use crate::parser::Parser;
 
 impl Parser {
     pub fn parse_name(&mut self) -> syn::Result<Ident> {
@@ -73,7 +73,7 @@ impl Parser {
 mod tests {
     use quote::quote;
 
-    use crate::common::Parser;
+    use crate::Parser;
 
     #[test]
     fn test_name_parse() {

codegen/src/common/parser/punct.rs → dsl/src/parser/punct.rs

@@ -1,7 +1,7 @@
 use proc_macro2::TokenTree;
 use syn::spanned::Spanned;
 
-use crate::common::parser::Parser;
+use crate::parser::Parser;
 
 impl Parser {
     /// Parses ':' from [`proc_macro2::TokenStream`].

codegen/src/common/parser/queries/delete.rs → dsl/src/parser/queries/delete.rs

@@ -3,8 +3,8 @@ use std::collections::HashMap;
 use proc_macro2::{Ident, TokenTree};
 use syn::spanned::Spanned;
 
-use crate::common::Parser;
-use crate::common::model::Operation;
+use crate::Parser;
+use crate::model::Operation;
 
 impl Parser {
     pub fn parse_deletes(&mut self) -> syn::Result<HashMap<Ident, Operation>> {
@@ -40,7 +40,7 @@ mod tests {
     use proc_macro2::{Ident, Span};
     use quote::quote;
 
-    use crate::common::Parser;
+    use crate::Parser;
 
     #[test]
     fn test_update() {

codegen/src/common/parser/queries/in_place.rs → dsl/src/parser/queries/in_place.rs

@@ -3,8 +3,8 @@ use std::collections::HashMap;
 use proc_macro2::{Ident, TokenTree};
 use syn::spanned::Spanned;
 
-use crate::common::Parser;
-use crate::common::model::Operation;
+use crate::Parser;
+use crate::model::Operation;
 
 impl Parser {
     pub fn parse_in_place(&mut self) -> syn::Result<HashMap<Ident, Operation>> {
@@ -40,7 +40,7 @@ mod tests {
     use proc_macro2::{Ident, Span};
     use quote::quote;
 
-    use crate::common::Parser;
+    use crate::Parser;
 
     #[test]
     fn test_update() {

codegen/src/common/parser/queries/mod.rs → dsl/src/parser/queries/mod.rs

@@ -7,8 +7,8 @@ mod update;
 use proc_macro2::TokenTree;
 use syn::spanned::Spanned;
 
-use crate::common::Parser;
-use crate::common::model::Queries;
+use crate::Parser;
+use crate::model::Queries;
 
 impl Parser {
     pub fn parse_queries(&mut self) -> syn::Result<Queries> {

codegen/src/common/parser/queries/operation.rs → dsl/src/parser/queries/operation.rs

@@ -2,8 +2,8 @@ use proc_macro2::{Ident, TokenTree};
 use std::collections::HashMap;
 use syn::spanned::Spanned;
 
-use crate::common::model::Operation;
-use crate::common::parser::Parser;
+use crate::model::Operation;
+use crate::parser::Parser;
 
 impl Parser {
     pub fn parse_operations(&mut self) -> syn::Result<HashMap<Ident, Operation>> {
@@ -96,7 +96,7 @@ impl Parser {
 mod tests {
     use quote::quote;
 
-    use crate::common::parser::Parser;
+    use crate::parser::Parser;
 
     #[test]
     fn test_operation() {

codegen/src/common/parser/queries/select.rs → dsl/src/parser/queries/select.rs

@@ -3,8 +3,8 @@ use std::collections::HashMap;
 use proc_macro2::{Ident, TokenTree};
 use syn::spanned::Spanned;
 
-use crate::common::Parser;
-use crate::common::model::Operation;
+use crate::Parser;
+use crate::model::Operation;
 
 impl Parser {
     pub fn _parse_selects(&mut self) -> syn::Result<HashMap<Ident, Operation>> {
@@ -40,7 +40,7 @@ mod tests {
     use proc_macro2::{Ident, Span};
     use quote::quote;
 
-    use crate::common::Parser;
+    use crate::Parser;
 
     #[test]
     fn test_update() {

codegen/src/common/parser/queries/update.rs → dsl/src/parser/queries/update.rs

@@ -3,8 +3,8 @@ use std::collections::HashMap;
 use proc_macro2::{Ident, TokenTree};
 use syn::spanned::Spanned;
 
-use crate::common::Parser;
-use crate::common::model::Operation;
+use crate::Parser;
+use crate::model::Operation;
 
 impl Parser {
     pub fn parse_updates(&mut self) -> syn::Result<HashMap<Ident, Operation>> {
@@ -42,7 +42,7 @@ mod tests {
     use proc_macro2::{Ident, Span};
     use quote::quote;
 
-    use crate::common::Parser;
+    use crate::Parser;
 
     #[test]
     fn test_update() {