From 9fa01f4a4d3f8346805212c85d0648412828501a Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Mon, 10 Aug 2026 11:35:46 +0200 Subject: [PATCH 01/58] Finally a _helpful_ justfile with comments, test filtering and aliases --- packages/coln-query/justfile | 64 +++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/packages/coln-query/justfile b/packages/coln-query/justfile index a6dafa16..e5c17d30 100644 --- a/packages/coln-query/justfile +++ b/packages/coln-query/justfile @@ -1,52 +1,62 @@ # Task runner: https://github.com/casey/just -# From packages/coln-query, with Rust on PATH (e.g. `nix develop`): -# just # list available recipes -# just check # fmt-check + clippy + test -# just cargo-check # type-check coln-query targets -# just import -# just fix # apply cargo fixes + format -# just coverage # llvm-cov + Cobertura crate := "coln-query" +# List all just targets. default: @just --list +# Auto fix code formatting. fmt: - cargo fmt -p {{crate}} + cargo fmt -p {{ crate }} +# Check code formatting. fmt-check: - cargo fmt -p {{crate}} --check + cargo fmt -p {{ crate }} --check -cargo-check: - cargo check -p {{crate}} --all-targets +# Auto fix linting issues. +lint: + cargo fix -p {{ crate }} --all-targets --all-features --allow-dirty --allow-staged + cargo clippy -p {{ crate }} --fix --all-targets --all-features --allow-dirty --allow-staged -- -D warnings -clippy: - cargo clippy -p {{crate}} --all-targets --all-features -- -D warnings +# Check linting. +lint-check: + cargo check -p {{ crate }} --all-targets --all-features + cargo clippy -p {{ crate }} --all-targets --all-features -- -D warnings -import: - cargo fix -p {{crate}} --all-targets --all-features --allow-dirty --allow-staged - -fix: import fmt - -test: - cargo nextest run -p {{crate}} --all-targets +# Run tests with stdout and stderr suppressed. +test *TESTS: + cargo nextest run -p {{ crate }} --all-targets {{ TESTS }} # As of now nextest does not support doc tests, so we fallback to # cargo test to run them. - cargo test --doc -p {{crate}} + cargo test --doc -p {{ crate }} {{ TESTS }} -doc-build: - cargo doc +alias t := test-verbose +# Run tests but with stdout and stderr printed. +test-verbose *TESTS: + cargo nextest run -p {{ crate }} --all-targets --nocapture {{ TESTS }} +# Build the docs. +doc: + cargo doc -p {{ crate }} + +# Check the docs for broken links. doc-check: - cargo doc --no-deps --document-private-items --config 'build.rustdocflags=["-D", "warnings"]' + cargo doc -p {{ crate }} --no-deps --document-private-items --config 'build.rustdocflags=["-D", "warnings"]' + +# Run all code checks. Global command referenced by the root justfile. +check: fmt-check lint-check test doc-check -check: fmt-check clippy test doc-check +# Run all automated code fixes. Global command referenced by the root justfile. +fix: fmt lint +# Define what to do in the CI pipeline. ci: check +# Test coverage. coverage: - cargo llvm-cov -p {{crate}} --cobertura --output-path cobertura.xml + cargo llvm-cov -p {{ crate }} --cobertura --output-path cobertura.xml +# Run the crate. run: - cargo run -p {{crate}} + cargo run -p {{ crate }} From 613948410a62f9f947e08e9b24a265b9381fb565 Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Mon, 10 Aug 2026 14:07:13 +0200 Subject: [PATCH 02/58] [coln-flir-rs] Pretty-format Graph.json and Prim.json --- packages/coln-flir-rs/tests/data/Graph.json | 93 ++++++++++++++++++++- packages/coln-flir-rs/tests/data/Prim.json | 71 +++++++++++++++- 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/packages/coln-flir-rs/tests/data/Graph.json b/packages/coln-flir-rs/tests/data/Graph.json index 329354f2..df99172b 100644 --- a/packages/coln-flir-rs/tests/data/Graph.json +++ b/packages/coln-flir-rs/tests/data/Graph.json @@ -1 +1,92 @@ -{"entities":[{"path":[["Graph"],["E"]],"value":{"entityVariant":{"tag":"table"},"columns":[{"path":[["a"]],"type":{"tag":"rowId","path":[["Graph"],["V"]]}},{"path":[["b"]],"type":{"tag":"rowId","path":[["Graph"],["V"]]}}],"primaryKey":null}},{"path":[["Graph"],["V"]],"value":{"entityVariant":{"tag":"table"},"columns":[],"primaryKey":null}}],"rules":[{"path":[["Graph"],["E"],["foreignKey"]],"value":{"ruleVariant":"enforced","varNames":[[["a"]],[["b"]]],"varTypes":[{"tag":"rowId","path":[["Graph"],["V"]]},{"tag":"rowId","path":[["Graph"],["V"]]}],"antecedents":[{"tag":"atom","atom":{"entity":[["Graph"],["E"]],"rowId":null,"values":[{"column":0,"term":{"tag":"var","index":0}},{"column":1,"term":{"tag":"var","index":1}}]}}],"consequents":[{"tag":"atom","atom":{"entity":[["Graph"],["V"]],"rowId":{"tag":"var","index":0},"values":[]}},{"tag":"atom","atom":{"entity":[["Graph"],["V"]],"rowId":{"tag":"var","index":1},"values":[]}}]}},{"path":[["Graph"],["V"],["foreignKey"]],"value":{"ruleVariant":"enforced","varNames":[],"varTypes":[],"antecedents":[{"tag":"atom","atom":{"entity":[["Graph"],["V"]],"rowId":null,"values":[]}}],"consequents":[]}}]} \ No newline at end of file +{ + "entities": [ + { + "path": [["Graph"], ["E"]], + "value": { + "entityVariant": { "tag": "table" }, + "columns": [ + { + "path": [["a"]], + "type": { "tag": "rowId", "path": [["Graph"], ["V"]] } + }, + { + "path": [["b"]], + "type": { "tag": "rowId", "path": [["Graph"], ["V"]] } + } + ], + "primaryKey": null + } + }, + { + "path": [["Graph"], ["V"]], + "value": { + "entityVariant": { "tag": "table" }, + "columns": [], + "primaryKey": null + } + } + ], + "rules": [ + { + "path": [["Graph"], ["E"], ["foreignKey"]], + "value": { + "ruleVariant": "enforced", + "varNames": [[["a"]], [["b"]]], + "varTypes": [ + { "tag": "rowId", "path": [["Graph"], ["V"]] }, + { "tag": "rowId", "path": [["Graph"], ["V"]] } + ], + "antecedents": [ + { + "tag": "atom", + "atom": { + "entity": [["Graph"], ["E"]], + "rowId": null, + "values": [ + { "column": 0, "term": { "tag": "var", "index": 0 } }, + { "column": 1, "term": { "tag": "var", "index": 1 } } + ] + } + } + ], + "consequents": [ + { + "tag": "atom", + "atom": { + "entity": [["Graph"], ["V"]], + "rowId": { "tag": "var", "index": 0 }, + "values": [] + } + }, + { + "tag": "atom", + "atom": { + "entity": [["Graph"], ["V"]], + "rowId": { "tag": "var", "index": 1 }, + "values": [] + } + } + ] + } + }, + { + "path": [["Graph"], ["V"], ["foreignKey"]], + "value": { + "ruleVariant": "enforced", + "varNames": [], + "varTypes": [], + "antecedents": [ + { + "tag": "atom", + "atom": { + "entity": [["Graph"], ["V"]], + "rowId": null, + "values": [] + } + } + ], + "consequents": [] + } + } + ] +} diff --git a/packages/coln-flir-rs/tests/data/Prim.json b/packages/coln-flir-rs/tests/data/Prim.json index dd64b14a..e4bc7124 100644 --- a/packages/coln-flir-rs/tests/data/Prim.json +++ b/packages/coln-flir-rs/tests/data/Prim.json @@ -1 +1,70 @@ -{"entities":[{"path":[["T"],["N"]],"value":{"entityVariant":{"tag":"table"},"columns":[{"path":[["a"]],"type":{"tag":"builtin","type":"builtinInt"}}],"primaryKey":null}},{"path":[["T"],["S"]],"value":{"entityVariant":{"tag":"table"},"columns":[{"path":[["a"]],"type":{"tag":"builtin","type":"builtinString"}}],"primaryKey":null}}],"rules":[{"path":[["T"],["N"],["foreignKey"]],"value":{"ruleVariant":"enforced","varNames":[[["a"]]],"varTypes":[{"tag":"builtin","type":"builtinInt"}],"antecedents":[{"tag":"atom","atom":{"entity":[["T"],["N"]],"rowId":null,"values":[{"column":0,"term":{"tag":"var","index":0}}]}}],"consequents":[]}},{"path":[["T"],["S"],["foreignKey"]],"value":{"ruleVariant":"enforced","varNames":[[["a"]]],"varTypes":[{"tag":"builtin","type":"builtinString"}],"antecedents":[{"tag":"atom","atom":{"entity":[["T"],["S"]],"rowId":null,"values":[{"column":0,"term":{"tag":"var","index":0}}]}}],"consequents":[]}}]} \ No newline at end of file +{ + "entities": [ + { + "path": [["T"], ["N"]], + "value": { + "entityVariant": { "tag": "table" }, + "columns": [ + { + "path": [["a"]], + "type": { "tag": "builtin", "type": "builtinInt" } + } + ], + "primaryKey": null + } + }, + { + "path": [["T"], ["S"]], + "value": { + "entityVariant": { "tag": "table" }, + "columns": [ + { + "path": [["a"]], + "type": { "tag": "builtin", "type": "builtinString" } + } + ], + "primaryKey": null + } + } + ], + "rules": [ + { + "path": [["T"], ["N"], ["foreignKey"]], + "value": { + "ruleVariant": "enforced", + "varNames": [[["a"]]], + "varTypes": [{ "tag": "builtin", "type": "builtinInt" }], + "antecedents": [ + { + "tag": "atom", + "atom": { + "entity": [["T"], ["N"]], + "rowId": null, + "values": [{ "column": 0, "term": { "tag": "var", "index": 0 } }] + } + } + ], + "consequents": [] + } + }, + { + "path": [["T"], ["S"], ["foreignKey"]], + "value": { + "ruleVariant": "enforced", + "varNames": [[["a"]]], + "varTypes": [{ "tag": "builtin", "type": "builtinString" }], + "antecedents": [ + { + "tag": "atom", + "atom": { + "entity": [["T"], ["S"]], + "rowId": null, + "values": [{ "column": 0, "term": { "tag": "var", "index": 0 } }] + } + } + ], + "consequents": [] + } + } + ] +} From 7e5606ffe3a7634e9384e720a8185115d8bbd8ce Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Mon, 10 Aug 2026 19:47:07 +0200 Subject: [PATCH 03/58] [wip] map flirs to queries --- packages/coln-flir-rs/src/ir/mod.rs | 14 ++- packages/coln-query/src/api/query.rs | 131 +++++++++++++++++++++++++- packages/coln-query/src/api/schema.rs | 68 ++++++++++++- 3 files changed, 208 insertions(+), 5 deletions(-) diff --git a/packages/coln-flir-rs/src/ir/mod.rs b/packages/coln-flir-rs/src/ir/mod.rs index 5a146e15..5ace6512 100644 --- a/packages/coln-flir-rs/src/ir/mod.rs +++ b/packages/coln-flir-rs/src/ir/mod.rs @@ -62,9 +62,10 @@ impl<'de> Deserialize<'de> for BuiltinTy { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "tag", rename_all = "camelCase")] pub enum ColType { - RowId { - path: Path, - }, + /// A foreign key into another table by referencing its _row id_ through + /// provided the path. + RowId { path: Path }, + /// A data column with the scalar type [`BuiltinTy`]. #[serde(rename = "builtin")] BuiltinTy { #[serde(rename = "type")] @@ -110,7 +111,13 @@ pub struct ColumnEntry { #[serde(rename_all = "camelCase")] pub struct Schema { pub entity_variant: EntityVariant, + // TODO: Are the columns guaranteed to be in their physical order? pub columns: Vec, + // TODO: + // 1. What about multiple primary keys per table? Not a thing? + // 2. Why wrap the Vec in an Option? `null` could become the empty vec. + // 3. Why is this Vec/Vec? Why not Vec> which could + // just be indices into `Self::columns`. pub primary_key: Option>, } @@ -151,6 +158,7 @@ pub enum Prop { Eq { left: Term, right: Term }, } +// TODO: Why not make it Copy? #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum RuleVariant { diff --git a/packages/coln-query/src/api/query.rs b/packages/coln-query/src/api/query.rs index 952cd071..37281cb2 100644 --- a/packages/coln-query/src/api/query.rs +++ b/packages/coln-query/src/api/query.rs @@ -3,4 +3,133 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT //! This module converts coln's flattened lowered intermediate representation -//! (FLIR) into an AST of relational operators. +//! (FLIR) into a query program expressed in [`Statements`](crate::host::stmt::Stmt), +//! using [`HostExprs`](crate::host::expr::Expr) and [`RelExprs`](crate::relational::expr::RelExpr). + +use crate::api::schema::{Column, TableRef, TableSchema}; +use crate::error::SyntaxError; +use crate::host::Code; +use coln_flir_rs::ir::{ + self, Atom, EntityVariant, FlatRealm, Prop, Rule, RuleEntry, TableEntry, Term, +}; +use std::collections::HashMap; + +type BaseTableName = TableRef; +type DerivedViewName = TableRef; + +/// An identifier that uniquely identifies a table (globally across the store). +#[derive(Eq, PartialEq, Hash, Debug, Clone)] +pub struct RuleName { + inner: String, +} + +impl> From for RuleName { + fn from(value: T) -> Self { + RuleName { + inner: value.into(), + } + } +} + +struct QueryProgramBuilder { + program: Code, + base_tables: HashMap, + derived_views: HashMap, + rules: HashMap, +} + +impl QueryProgramBuilder { + fn new() -> Self { + Self { + program: Vec::new(), + base_tables: HashMap::new(), + derived_views: HashMap::new(), + rules: HashMap::new(), + } + } + pub fn from_flat_realm(flat_realm: &FlatRealm) -> Result { + let mut builder = QueryProgramBuilder::new(); + for table in &flat_realm.tables { + builder.table_declaration(table)?; + } + for rule in &flat_realm.rules { + builder.rule_declaration(rule)?; + } + Ok(builder) + } + + fn table_declaration(&mut self, table_entry: &TableEntry) -> Result<(), SyntaxError> { + let name = table_entry.path.to_string(); + let schema = &table_entry.table; + + match &schema.entity_variant { + EntityVariant::Table => self.base_table(BaseTableName::from(name), schema), + EntityVariant::View(materialization) => { + unimplemented!("[Initial models] Materialized views defined through a query"); + } + EntityVariant::Index { method, columns } => { + unimplemented!("[Not-yet specified] Indexes") + } + } + } + fn base_table(&mut self, name: BaseTableName, schema: &ir::Schema) -> Result<(), SyntaxError> { + let columns = schema.columns.iter().map(Column::from).collect(); + let primary_key = schema + .primary_key + .as_ref() + .map_or(Ok(Vec::new()), |compound_primary_key| { + compound_primary_key + .iter() + .map(|primary_key_column| { + schema + .columns + .iter() + .position(|column| column.path == *primary_key_column) + .ok_or_else(|| { + SyntaxError::new(format!( + "Primary key column {primary_key_column} not found in base table {name}", + )) + }) + }) + .collect::, SyntaxError>>() + })?; + let table_schema = TableSchema::new(name.clone(), columns, vec![primary_key]); + self.base_tables + .insert(name.clone(), table_schema) + .ok_or_else(|| SyntaxError::new(format!("Base table {name} defined multiple times"))) + .map(|_old_entry| ()) + } + + fn rule_declaration(&mut self, rule_entry: &RuleEntry) -> Result<(), SyntaxError> { + let name = rule_entry.path.to_string(); + let rule = &rule_entry.rule; + self.rules + .insert(RuleName::from(&name), rule.rule_variant.clone()) + .ok_or_else(|| SyntaxError::new(format!("Rule {name} defined multiple times")))?; + match &rule.rule_variant { + ir::RuleVariant::Enforced => { + todo!() + } + ir::RuleVariant::Monitored => todo!(), + ir::RuleVariant::Chased => todo!( + // TODO: clarify + "Chased rules produce a materialized view; how are they different from a materialized view defined in the table section?" + ), + } + } + fn rule(&mut self, rule: &Rule) -> Result<(), SyntaxError> { + todo!() + } + fn prop(&mut self, prop: &Prop) -> Result<(), SyntaxError> { + match prop { + Prop::Atom { atom } => todo!(), + Prop::Eq { left, right } => todo!(), + } + } + fn atom(&mut self, atom: &Atom) -> Result<(), SyntaxError> { + todo!() + } + fn term(&mut self, term: &Term) -> Result<(), SyntaxError> { + todo!() + } +} diff --git a/packages/coln-query/src/api/schema.rs b/packages/coln-query/src/api/schema.rs index cee5c575..f817aa91 100644 --- a/packages/coln-query/src/api/schema.rs +++ b/packages/coln-query/src/api/schema.rs @@ -5,7 +5,9 @@ //! This module provides implementations for passing references to tables //! ([TableRef]) and communicating a schema of a table ([TableSchema]). -use crate::scalarial::ScalarType; +use std::fmt::Display; + +use crate::scalarial::{ScalarType, ScalarTypedValue}; /// An identifier that uniquely identifies a table (globally across the store). #[derive(Eq, PartialEq, Hash, Debug, Clone)] @@ -13,6 +15,20 @@ pub struct TableRef { inner: String, } +impl Display for TableRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.inner.fmt(f) + } +} + +impl> From for TableRef { + fn from(value: T) -> Self { + TableRef { + inner: value.into(), + } + } +} + pub struct TableSchema { /// The table's unique identifier/name. name: TableRef, @@ -23,9 +39,59 @@ pub struct TableSchema { primary_keys: Vec>, } +impl TableSchema { + pub fn new(name: TableRef, columns: Vec, primary_keys: Vec>) -> Self { + Self { + name, + columns, + primary_keys, + } + } +} + pub struct Column { /// The column's name. name: String, /// Ihe column's (scalar) type. scalar_type: ScalarType, } + +use coln_flir_rs::ir::{self}; + +impl From<&ir::ColumnEntry> for Column { + fn from(value: &ir::ColumnEntry) -> Self { + // For now we use the flattened path representation in the query engine. + let name = value.path.to_string(); + let scalar_type = ScalarType::from(&value.col_type); + Column { name, scalar_type } + } +} + +impl From<&ir::ColType> for ScalarType { + fn from(value: &ir::ColType) -> Self { + match value { + ir::ColType::BuiltinTy { builtin_ty } => ScalarType::from(*builtin_ty), + // We assume that row ids will be sent as unsigned integers by coln-store. + ir::ColType::RowId { path } => ScalarType::Uint, + } + } +} + +impl From for ScalarType { + fn from(value: ir::BuiltinTy) -> Self { + match value { + // TODO: Discuss scalar types and their mappings. + ir::BuiltinTy::BuiltinInt => ScalarType::Iint, + ir::BuiltinTy::BuiltinStr => ScalarType::String, + } + } +} + +impl From<&ir::Lit> for ScalarTypedValue { + fn from(value: &ir::Lit) -> Self { + match value { + ir::Lit::Int { value } => ScalarTypedValue::Iint(*value), + ir::Lit::String { value } => ScalarTypedValue::String(value.clone()), + } + } +} From 0dcbdd015e442c1ebbc6576a2b07e0c67f978c3e Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Thu, 13 Aug 2026 14:33:18 +0200 Subject: [PATCH 04/58] [coln-flir-rs] Add doc comments and turn indices into unsigned types and fix propagated errors in coln-store --- packages/coln-flir-rs/src/ir/mod.rs | 81 +++++++++++++++++---- packages/coln-store/src/commit/wire/root.rs | 8 +- packages/coln-store/src/solver/compile.rs | 14 ++-- 3 files changed, 75 insertions(+), 28 deletions(-) diff --git a/packages/coln-flir-rs/src/ir/mod.rs b/packages/coln-flir-rs/src/ir/mod.rs index 5ace6512..104f4238 100644 --- a/packages/coln-flir-rs/src/ir/mod.rs +++ b/packages/coln-flir-rs/src/ir/mod.rs @@ -16,8 +16,17 @@ pub type QName = Vec; #[serde(transparent)] pub struct Path(pub Vec); +/// A column name is given by a [`Path`]. type ColName = Path; -pub type FId = i64; + +/// An index into the [`varNames`](Rule::var_names) and +/// [`varTypes`](Rule::var_types) arrays of a [`Rule`]. +/// +/// Note: An `FId` in `coln-compiler`. +pub type VarIdx = u64; + +/// An index into a relation's physical [`columns`](Schema::columns). +pub type ColumnIdx = u64; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BuiltinTy { @@ -63,7 +72,7 @@ impl<'de> Deserialize<'de> for BuiltinTy { #[serde(tag = "tag", rename_all = "camelCase")] pub enum ColType { /// A foreign key into another table by referencing its _row id_ through - /// provided the path. + /// the provided path. RowId { path: Path }, /// A data column with the scalar type [`BuiltinTy`]. #[serde(rename = "builtin")] @@ -90,8 +99,11 @@ pub enum IndexMethod { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "tag", rename_all = "camelCase")] pub enum EntityVariant { + /// A base table of the extensional database (EDB). Table, + /// A derived view of the intensional database (IDB). View(Materialization), + /// Tell `coln-store` to create an index and possibly hint to `coln-query`. Index { method: IndexMethod, columns: Vec, @@ -106,21 +118,25 @@ pub struct ColumnEntry { pub col_type: ColType, } -// This is really Entity on the Haskell IR side, but I feel schema matches it better +/// Describes a schema of a relation. +/// +/// Note: An `Entity` in `coln-compiler`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Schema { pub entity_variant: EntityVariant, - // TODO: Are the columns guaranteed to be in their physical order? + /// The columns of the table in their physical order. pub columns: Vec, - // TODO: - // 1. What about multiple primary keys per table? Not a thing? - // 2. Why wrap the Vec in an Option? `null` could become the empty vec. - // 3. Why is this Vec/Vec? Why not Vec> which could - // just be indices into `Self::columns`. + /// A `None` indicates that there is no primary key. `Some(vec![])` means + /// that there is at most one row in the table. `Some(vec![ColA, ColB])` + /// encodes a compound primary key consisting of the columns `ColA` and + /// `ColB`. + /// + /// At the moment there is only support for a single (compound) primary key. pub primary_key: Option>, } +/// A literal expression. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "tag", rename_all = "lowercase")] pub enum Lit { @@ -134,51 +150,82 @@ pub enum Lit { #[serde(tag = "tag", rename_all = "lowercase")] pub enum Term { Lit { lit: Lit }, - Var { index: FId }, + Var { index: VarIdx }, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ValueEntry { - pub column: i64, + pub column: ColumnIdx, + /// Note: A [`Term::Lit`] together with a [`ColumnIdx`](Self::column) does + /// not make sense, I suppose. pub term: Term, } +/// An [`Atom`] references an entity (a relation or a table) to bring some of +/// its fields into the scope of a [`Rule`]. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Atom { + /// The "name" of the entity being referenced by this [`Atom`]. pub entity: Path, + /// To bring the `row_id` of the [`Entity`](Self::entity) into scope. + /// + /// Note: A [`Some(Term::Lit)`](Term::Lit) does not make sense, I suppose. pub row_id: Option, + /// To bring some columns of the [`Entity`](Self::entity) into scope. pub values: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "tag", rename_all = "lowercase")] pub enum Prop { - Atom { atom: Atom }, - Eq { left: Term, right: Term }, + Atom { + atom: Atom, + }, + /// An equality condition between the left and the right term, that is, + /// we assert `left == right`. + Eq { + left: Term, + right: Term, + }, } -// TODO: Why not make it Copy? -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum RuleVariant { + /// _Chased_ rules are not yet fully alive but become relevant once initial + /// models land. Chased, + /// Violations of _enforced_ rules cause a transaction to abort. Enforced, + /// Violations of _monitored_ rules are just reported back to the user but + /// still allow a transaction to commit. Monitored, } +/// A `Rule` is an implication and must be true in all valid states of +/// `coln-store` and `coln-query`. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Rule { pub rule_variant: RuleVariant, + /// Assigns some names to the variables the rule binds. + /// + /// Note: Must be of the same arity as [`Self::var_types`]. pub var_names: Vec, + /// Tells the types of the variables the rule binds. + /// + /// Note: Must be of the same arity as [`Self::var_names`]. pub var_types: Vec, + /// The left-hand side of the implication. pub antecedents: Vec, + /// The right-hand side of the implication. pub consequents: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TableEntry { + /// The "name" of the table. pub path: Path, #[serde(rename = "value")] pub table: Schema, @@ -186,14 +233,18 @@ pub struct TableEntry { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RuleEntry { + /// The "name" of the rule. pub path: Path, #[serde(rename = "value")] pub rule: Rule, } +/// The top-level type of a flattened realm and the starting point of the FLIR. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FlatRealm { + /// The tables of the flattened realm. #[serde(rename = "entities")] pub tables: Vec, + /// The rules (laws) of the flattened realm. pub rules: Vec, } diff --git a/packages/coln-store/src/commit/wire/root.rs b/packages/coln-store/src/commit/wire/root.rs index ca188991..80784116 100644 --- a/packages/coln-store/src/commit/wire/root.rs +++ b/packages/coln-store/src/commit/wire/root.rs @@ -426,7 +426,7 @@ fn write_atom(buf: &mut Vec, atom: &Atom) -> Result<(), CodecError> { } write_count(buf, atom.values.len()); for value in &atom.values { - commit_leb128::write_i64(buf, value.column); + commit_leb128::write_u64(buf, value.column); write_term(buf, &value.term)?; } Ok(()) @@ -447,7 +447,7 @@ fn read_atom(data: &[u8], pos: &mut usize) -> Result { let mut values = Vec::with_capacity(value_count); for _ in 0..value_count { values.push(ValueEntry { - column: commit_leb128::read_i64(data, pos, "atom value column")?, + column: commit_leb128::read_u64(data, pos, "atom value column")?, term: read_term(data, pos)?, }); } @@ -466,7 +466,7 @@ fn write_term(buf: &mut Vec, term: &Term) -> Result<(), CodecError> { } Term::Var { index } => { write_u8(buf, 1)?; - commit_leb128::write_i64(buf, *index); + commit_leb128::write_u64(buf, *index); Ok(()) } } @@ -478,7 +478,7 @@ fn read_term(data: &[u8], pos: &mut usize) -> Result { lit: read_lit(data, pos)?, }), 1 => Ok(Term::Var { - index: commit_leb128::read_i64(data, pos, "term var index")?, + index: commit_leb128::read_u64(data, pos, "term var index")?, }), tag => Err(CodecError::DataFormatError(format!( "unknown term tag {tag}" diff --git a/packages/coln-store/src/solver/compile.rs b/packages/coln-store/src/solver/compile.rs index ab0e02bb..9d696508 100644 --- a/packages/coln-store/src/solver/compile.rs +++ b/packages/coln-store/src/solver/compile.rs @@ -14,9 +14,9 @@ pub enum CompileError { #[error("unsupported term")] UnsupportedTerm, #[error("invalid var index: {index} var_count {var_count}")] - InvalidVarIndex { index: i64, var_count: usize }, + InvalidVarIndex { index: u64, var_count: usize }, #[error("invalid column index {column}")] - InvalidColumnIndex { column: i64 }, + InvalidColumnIndex { column: u64 }, } /// A rule lowered into a small execution-oriented rule form. @@ -188,17 +188,13 @@ fn compile_atom(atom: &Atom, var_count: usize) -> Result fn compile_term(term: &Term, var_count: usize) -> Result { match term { Term::Var { index } => { - let index = usize::try_from(*index).map_err(|_| CompileError::InvalidVarIndex { - index: *index, - var_count, - })?; - if index >= var_count { + if *index >= var_count as u64 { return Err(CompileError::InvalidVarIndex { - index: index as i64, + index: *index, var_count, }); } - Ok(CompTerm::Var(index)) + Ok(CompTerm::Var(*index as usize)) } Term::Lit { lit } => Ok(CompTerm::Lit(lit.clone())), } From f3c9fde6376d4fea0889395b1523b941f9ba9a9e Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Thu, 13 Aug 2026 17:43:31 +0200 Subject: [PATCH 05/58] [coln-flir-rs] Add equality struct (because Rust doesn't allow enum variants as types) in an oblivious-to-JSON manner --- packages/coln-flir-rs/src/ir/mod.rs | 14 +++++++---- packages/coln-store/src/commit/wire/root.rs | 20 ++++++++++------ packages/coln-store/src/solver/bind.rs | 16 +++++++++---- packages/coln-store/src/solver/compile.rs | 26 ++++++++++++++------- packages/coln-store/src/solver/validate.rs | 14 +++++++---- 5 files changed, 61 insertions(+), 29 deletions(-) diff --git a/packages/coln-flir-rs/src/ir/mod.rs b/packages/coln-flir-rs/src/ir/mod.rs index 104f4238..f640923b 100644 --- a/packages/coln-flir-rs/src/ir/mod.rs +++ b/packages/coln-flir-rs/src/ir/mod.rs @@ -182,14 +182,20 @@ pub enum Prop { Atom { atom: Atom, }, - /// An equality condition between the left and the right term, that is, - /// we assert `left == right`. Eq { - left: Term, - right: Term, + #[serde(flatten)] + equality: Equality, }, } +/// An equality condition between the left and the right term, that is, +/// we assert `left == right`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Equality { + pub left: Term, + pub right: Term, +} + #[derive(Debug, Copy, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum RuleVariant { diff --git a/packages/coln-store/src/commit/wire/root.rs b/packages/coln-store/src/commit/wire/root.rs index 80784116..399e78fb 100644 --- a/packages/coln-store/src/commit/wire/root.rs +++ b/packages/coln-store/src/commit/wire/root.rs @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 OR MIT +use coln_flir_rs::ir::Equality; + use crate::{ commit::{ error::CodecError, @@ -392,10 +394,10 @@ fn write_prop(buf: &mut Vec, prop: &Prop) -> Result<(), CodecError> { write_u8(buf, 0)?; write_atom(buf, atom) } - Prop::Eq { left, right } => { + Prop::Eq { equality } => { write_u8(buf, 1)?; - write_term(buf, left)?; - write_term(buf, right) + write_term(buf, &equality.left)?; + write_term(buf, &equality.right) } } } @@ -406,8 +408,10 @@ fn read_prop(data: &[u8], pos: &mut usize) -> Result { atom: read_atom(data, pos)?, }), 1 => Ok(Prop::Eq { - left: read_term(data, pos)?, - right: read_term(data, pos)?, + equality: Equality { + left: read_term(data, pos)?, + right: read_term(data, pos)?, + }, }), tag => Err(CodecError::DataFormatError(format!( "unknown prop tag {tag}" @@ -618,8 +622,10 @@ mod tests { }, }], consequents: vec![Prop::Eq { - left: Term::Var { index: 0 }, - right: Term::Var { index: 0 }, + equality: Equality { + left: Term::Var { index: 0 }, + right: Term::Var { index: 0 }, + }, }], }, } diff --git a/packages/coln-store/src/solver/bind.rs b/packages/coln-store/src/solver/bind.rs index 6330cdbf..7f1d3de1 100644 --- a/packages/coln-store/src/solver/bind.rs +++ b/packages/coln-store/src/solver/bind.rs @@ -143,6 +143,8 @@ pub fn bind_rule(store: &Store, rule: &CompRule) -> Vec { #[cfg(test)] mod tests { + use coln_flir_rs::ir::Equality; + use super::*; use crate::{ ir::{ @@ -324,8 +326,10 @@ mod tests { }, }, ir::Prop::Eq { - left: ir::Term::Var { index: 0 }, - right: ir::Term::Var { index: 1 }, + equality: Equality { + left: ir::Term::Var { index: 0 }, + right: ir::Term::Var { index: 1 }, + }, }, ], vec![ir::Prop::Atom { @@ -368,9 +372,11 @@ mod tests { }, }, ir::Prop::Eq { - left: ir::Term::Var { index: 0 }, - right: ir::Term::Lit { - lit: ir::Lit::Int { value: 2 }, + equality: Equality { + left: ir::Term::Var { index: 0 }, + right: ir::Term::Lit { + lit: ir::Lit::Int { value: 2 }, + }, }, }, ], diff --git a/packages/coln-store/src/solver/compile.rs b/packages/coln-store/src/solver/compile.rs index 9d696508..4d9fec0a 100644 --- a/packages/coln-store/src/solver/compile.rs +++ b/packages/coln-store/src/solver/compile.rs @@ -148,9 +148,9 @@ fn compile_props(props: &[Prop], var_count: usize) -> Result Result { match prop { Prop::Atom { atom } => Ok(CompProp::Atom(compile_atom(atom, var_count)?)), - Prop::Eq { left, right } => Ok(CompProp::Eq(CompEq { - left: compile_term(left, var_count)?, - right: compile_term(right, var_count)?, + Prop::Eq { equality } => Ok(CompProp::Eq(CompEq { + left: compile_term(&equality.left, var_count)?, + right: compile_term(&equality.right, var_count)?, })), } } @@ -332,6 +332,8 @@ fn var_name(index: usize) -> String { #[cfg(test)] mod tests { + use coln_flir_rs::ir::Equality; + use super::*; use crate::ir::{BuiltinTy, ColType, Path, Rule, RuleEntry, RuleVariant}; @@ -523,8 +525,10 @@ mod tests { "T.eq_antecedent", vec![int_ty(), int_ty()], vec![Prop::Eq { - left: Term::Var { index: 0 }, - right: Term::Var { index: 1 }, + equality: Equality { + left: Term::Var { index: 0 }, + right: Term::Var { index: 1 }, + }, }], vec![Prop::Atom { atom: Atom { @@ -563,8 +567,10 @@ mod tests { }, }], vec![Prop::Eq { - left: Term::Var { index: 0 }, - right: Term::Var { index: 1 }, + equality: Equality { + left: Term::Var { index: 0 }, + right: Term::Var { index: 1 }, + }, }], ); @@ -617,8 +623,10 @@ mod tests { }, }, Prop::Eq { - left: Term::Var { index: 0 }, - right: Term::Var { index: 1 }, + equality: Equality { + left: Term::Var { index: 0 }, + right: Term::Var { index: 1 }, + }, }, ], ); diff --git a/packages/coln-store/src/solver/validate.rs b/packages/coln-store/src/solver/validate.rs index 5e2518c5..a308922f 100644 --- a/packages/coln-store/src/solver/validate.rs +++ b/packages/coln-store/src/solver/validate.rs @@ -148,6 +148,8 @@ pub fn check_rule(store: &Store, rule: &CompRule) -> Result<(), Box Date: Tue, 18 Aug 2026 16:29:32 +0200 Subject: [PATCH 06/58] [coln-flir-rs] Add schema views for storage, query, and compiler --- packages/coln-flir-rs/src/ir/mod.rs | 2 +- packages/coln-flir-rs/src/ir/path.rs | 13 + packages/coln-flir-rs/src/lib.rs | 1 + packages/coln-flir-rs/src/schema.rs | 358 +++++++++++++++++++++++++++ 4 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 packages/coln-flir-rs/src/schema.rs diff --git a/packages/coln-flir-rs/src/ir/mod.rs b/packages/coln-flir-rs/src/ir/mod.rs index f640923b..f0c8860c 100644 --- a/packages/coln-flir-rs/src/ir/mod.rs +++ b/packages/coln-flir-rs/src/ir/mod.rs @@ -17,7 +17,7 @@ pub type QName = Vec; pub struct Path(pub Vec); /// A column name is given by a [`Path`]. -type ColName = Path; +pub type ColName = Path; /// An index into the [`varNames`](Rule::var_names) and /// [`varTypes`](Rule::var_types) arrays of a [`Rule`]. diff --git a/packages/coln-flir-rs/src/ir/path.rs b/packages/coln-flir-rs/src/ir/path.rs index 792eb116..0a86c648 100644 --- a/packages/coln-flir-rs/src/ir/path.rs +++ b/packages/coln-flir-rs/src/ir/path.rs @@ -19,6 +19,19 @@ impl Deref for Path { } } +impl Path { + pub fn append(mut self, name: &str) -> Self { + self.0.push(vec![name.to_string()]); + self + } +} + +impl Into for Path { + fn into(self) -> String { + self.to_string() + } +} + impl Display for Path { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for (i, qname) in self.0.iter().enumerate() { diff --git a/packages/coln-flir-rs/src/lib.rs b/packages/coln-flir-rs/src/lib.rs index 7eea2a82..af1217cf 100644 --- a/packages/coln-flir-rs/src/lib.rs +++ b/packages/coln-flir-rs/src/lib.rs @@ -3,3 +3,4 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT pub mod ir; +pub mod schema; diff --git a/packages/coln-flir-rs/src/schema.rs b/packages/coln-flir-rs/src/schema.rs new file mode 100644 index 00000000..d41819fb --- /dev/null +++ b/packages/coln-flir-rs/src/schema.rs @@ -0,0 +1,358 @@ +use crate::ir::{self, Path}; + +pub struct BaseTableSchema { + /// The table's unique identifier/name. + name: ir::Path, + /// Fields of the table in their physical order from the perspective + /// of the _compiler_. The columns _do not_ include the implicit row id. + cols_compiler: CompilerCols, + /// Fields of the table in their physical order from the perspective + /// of the _storage engine_. + cols_store: StoreEngineCols, + /// Fields of the table in their physical order from the perspective + /// of the _query engine_. + cols_query: QueryEngineCols, + /// The list of (possibly compound) primary keys into the table, specified + /// as indices into the [compiler view](Self::cols_compiler). + primary_keys: Vec>, +} + +impl BaseTableSchema { + /// The name of the base table. + pub fn name(&self) -> &ir::Path { + &self.name + } + /// Returns [`None`] if `idx` is an index for an implicit row id. + pub fn get_compiler_col(&self, idx: CompilerColIdx) -> Option<&CompilerCol> { + match idx { + CompilerColIdx::RowId => None, + CompilerColIdx::Column(idx) => Some(&self.cols_compiler.0[idx as usize]), + } + } + pub fn get_storage_col(&self, idx: StoreEngineColIdx) -> &StoreEngineCol { + &self.cols_store.0[idx.0] + } + pub fn query_cols(&self) -> &QueryEngineCols { + &self.cols_query + } + pub fn get_query_col(&self, idx: QueryEngineColIdx) -> &QueryEngineCol { + &self.cols_query.0[idx.0] + } + /// Given a [`CompilerColIdx`] from the FLIR, indexing into the columns of + /// the compiler view, what are the corresponding column(s) according to the + /// query engine's view? This translation is necessary because row ids + /// flatten into two columns from the perspective of the query engine, + /// hence, a (compiler) index resolving to a row id column can result in + /// two columns. A (compiler) index to a non row id column results in + /// exactly one column. + pub fn resolve_query_cols(&self, idx: CompilerColIdx) -> impl Iterator { + let range = match idx { + CompilerColIdx::RowId => 0..StoreEngineCols::ROW_ID_COLS, + CompilerColIdx::Column(target_idx) => { + assert!( + (target_idx as usize) < self.cols_compiler.0.len(), + "Compiler idx out of bounds" + ); + // We account for the implicit row id columns by offsetting. + let mut query_idx = StoreEngineCols::ROW_ID_COLS; + let mut iter = self.cols_compiler.0.iter().enumerate(); + let target_col = loop { + let (idx, col) = iter.next().unwrap(); + if idx >= target_idx as usize { + break col; + } + match &col.ty { + // A column of a native scalar type also takes just one column. + ir::ColType::BuiltinTy { builtin_ty: _ } => query_idx += 1, + // A row id flattens into multiple columns in the query engine's + // view, so we have to advance more columns. + ir::ColType::RowId { path: _ } => query_idx += StoreEngineCols::ROW_ID_COLS, + }; + }; + match &target_col.ty { + ir::ColType::BuiltinTy { builtin_ty: _ } => query_idx..query_idx + 1, + ir::ColType::RowId { path: _ } => query_idx..query_idx + 2, + } + } + }; + self.cols_query.0[range].iter() + } + /// The list of (compound) primary key(s), given as indexes into the + /// compiler's column view. + /// + /// Hint: Compiler indexes can be converted into other views using the + /// [`resolve_*`](Self::resolve_query_cols) methods. + pub fn primary_keys(&self) -> &Vec> { + &self.primary_keys + } +} + +impl From<&ir::TableEntry> for Option { + fn from(value: &ir::TableEntry) -> Self { + let path = &value.path; + let schema = &value.table; + if !matches!(schema.entity_variant, ir::EntityVariant::Table) { + return None; // Only base tables allowed. + } + let columns_compiler = CompilerCols::from(schema.columns.as_slice()); + let columns_store = StoreEngineCols::from(columns_compiler.0.as_slice()); + let columns_query = QueryEngineCols::from(columns_store.0.as_slice()); + let primary_key = schema + .primary_key + .as_ref() + // Currently, `null` in JSON becomes the empty vector. + .map_or(Vec::new(), |compound_primary_key| { + compound_primary_key + .iter() + .map(|primary_key_column| { + schema + .columns + .iter() + .position(|column| column.path == *primary_key_column) + .map(|idx| CompilerColIdx::Column(idx as u64)) + .expect( + &format!( + "Primary key column {primary_key_column} not found in base table {path}", + ) + ) + }) + .collect::>() + }); + // Currently, the compiler supports only a single primary key. + let primary_keys = vec![primary_key]; + Some(BaseTableSchema { + name: path.clone(), + cols_compiler: columns_compiler, + cols_store: columns_store, + cols_query: columns_query, + primary_keys, + }) + } +} + +// Scalar types. + +/// Scalar types which are supported natively by both coln-store and coln-query. +#[derive(Clone, Copy, Debug)] +pub enum NativeScalarType { + /// Signed 64-bit integer. + Iint, + /// Unsigned 64-bit integer. + Uint, + /// String. + String, + // Add more :) +} + +impl From for NativeScalarType { + fn from(value: ir::BuiltinTy) -> Self { + match value { + ir::BuiltinTy::BuiltinStr => NativeScalarType::String, + ir::BuiltinTy::BuiltinInt => NativeScalarType::Iint, + // So far, no builtin uint. + } + } +} + +/// Scalar types which are supported by coln-store. +#[derive(Clone, Copy, Debug)] +pub enum StoreEngineScalarType { + /// A row id becomes a pair of `(CommitHash, Counter)`. + CommitHash, + /// A row id becomes a pair of `(CommitHash, Counter)`. + Counter, + Native(NativeScalarType), +} + +/// Scalar types which are supported by coln-query. +#[derive(Clone, Copy, Debug)] +pub enum QueryEngineScalarType { + Native(NativeScalarType), +} + +impl From for QueryEngineScalarType { + fn from(value: StoreEngineScalarType) -> Self { + match value { + StoreEngineScalarType::CommitHash => { + QueryEngineScalarType::Native(NativeScalarType::Uint) + } + StoreEngineScalarType::Counter => QueryEngineScalarType::Native(NativeScalarType::Uint), + StoreEngineScalarType::Native(native) => QueryEngineScalarType::Native(native), + } + } +} + +/// Generic column metadata representation. +pub struct Col { + /// The column's name. + name: ir::ColName, + /// The column's (scalar) type. + ty: T, + /// If the column is (part of) a foreign key, this links the referenced table. + references: R, +} + +impl Col { + pub fn name(&self) -> &ir::ColName { + &self.name + } +} + +/// Column metadata from the perspective of the compiler. +/// +/// The compiler encodes foreign keys as part of the type of a column (see the +/// [`ir::ColType::RowId`] variant of [`ir::ColType`]). +/// Hence, `R` becomes the unit type and is not required in this case. +pub type CompilerCol = Col; + +#[derive(Copy, Clone, Debug)] +pub enum CompilerColIdx { + /// A reference to the table's row id (the implicit primary key). + RowId, + /// A reference to a column is a (zero-indexed) column index. + Column(ir::ColumnIdx), +} + +impl CompilerColIdx { + pub fn for_row_id() -> Self { + CompilerColIdx::RowId + } +} + +impl From for CompilerColIdx { + fn from(value: ir::ColumnIdx) -> Self { + CompilerColIdx::Column(value) + } +} + +pub struct CompilerCols(Vec); + +pub type StoreEngineCol = Col>; + +#[derive(Copy, Clone, Debug)] +pub struct StoreEngineColIdx(usize); + +pub struct StoreEngineCols(Vec); + +impl StoreEngineCols { + /// To how many columns a row id expands to. + pub const ROW_ID_COLS: usize = 2; + /// The suffix of the hash column of a row id. + pub const HASH_COL_SUFFIX: &'static str = "RowIdHash"; + /// The suffix of the counter column of a row id. + pub const CTR_COL_SUFFIX: &'static str = "RowIdCtr"; + + /// From the perspective of coln-store, every base table has two implicitly + /// defined columns: The commit hash from the transaction which created the + /// row and a counter value, rendering the hash-counter-pair unique among + /// all insertions of a transaction. Coln-store assigns these counters. + fn implicit_row_id_cols() -> [StoreEngineCol; Self::ROW_ID_COLS] { + [ + StoreEngineCol { + name: Path::from(Self::HASH_COL_SUFFIX), + ty: StoreEngineScalarType::CommitHash, + references: None, + }, + StoreEngineCol { + name: Path::from(Self::CTR_COL_SUFFIX), + ty: StoreEngineScalarType::Counter, + references: None, + }, + ] + } + fn foreign_key_cols( + name: &ir::ColName, + foreign_entity: &Path, + ) -> [StoreEngineCol; Self::ROW_ID_COLS] { + [ + StoreEngineCol { + name: name.clone().append(Self::HASH_COL_SUFFIX), + ty: StoreEngineScalarType::CommitHash, + references: Some(foreign_entity.clone()), + }, + StoreEngineCol { + name: name.clone().append(Self::CTR_COL_SUFFIX), + ty: StoreEngineScalarType::Counter, + references: Some(foreign_entity.clone()), + }, + ] + } +} + +pub type QueryEngineCol = Col>; + +#[derive(Copy, Clone, Debug)] +pub struct QueryEngineColIdx(usize); + +pub struct QueryEngineCols(Vec); + +impl QueryEngineCols { + pub fn iter(&self) -> std::slice::Iter<'_, QueryEngineCol> { + self.0.iter() + } +} + +// Conversions from one view into another view. + +impl From<&[ir::ColumnEntry]> for CompilerCols { + fn from(ir_cols: &[ir::ColumnEntry]) -> Self { + CompilerCols( + ir_cols + .iter() + // It's an one-to-one mapping from FLIR's JSON representation + // to this intermediate representation. + .map(|col| CompilerCol { + name: col.path.clone(), + ty: col.col_type.clone(), + // Foreign keys are encoded in the `ty` for a CompilerColumn. + // Hence, references becomes the unit type. + references: (), + }) + .collect(), + ) + } +} + +impl From<&[CompilerCol]> for StoreEngineCols { + fn from(compiler_cols: &[CompilerCol]) -> Self { + let prepended_row_id_cols = StoreEngineCols::implicit_row_id_cols().into_iter(); + let schema_cols = compiler_cols.iter().flat_map(|col| { + let name = col.name.clone(); + let (first, second) = match &col.ty { + ir::ColType::RowId { path } => { + let [hash_col, ctr_col] = StoreEngineCols::foreign_key_cols(&name, &path); + (hash_col, Some(ctr_col)) + } + ir::ColType::BuiltinTy { builtin_ty } => ( + StoreEngineCol { + name, + ty: StoreEngineScalarType::Native(NativeScalarType::from(*builtin_ty)), + references: None, + }, + None, + ), + }; + std::iter::once(first).chain(second) + }); + StoreEngineCols(prepended_row_id_cols.chain(schema_cols).collect()) + } +} + +impl From<&[StoreEngineCol]> for QueryEngineCols { + fn from(store_engine_cols: &[StoreEngineCol]) -> Self { + QueryEngineCols( + store_engine_cols + .iter() + // It's a one-to-one mapping from the storage engine's schema + // view to the query engine's schema view; only the scalar types + // are different: The commit hash and counter become plain, + // unsigned ints, each. + .map(|col| QueryEngineCol { + name: col.name.clone(), + ty: QueryEngineScalarType::from(col.ty), + references: col.references.clone(), + }) + .collect(), + ) + } +} From 65e1ad54e6e52d4e9ce8b7b705603f152a908902 Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Tue, 18 Aug 2026 16:30:44 +0200 Subject: [PATCH 07/58] [coln-query] Add multi way equi join to RelExprs --- packages/coln-query/src/host/resolver.rs | 31 ++++++++++++- packages/coln-query/src/optimizer/mod.rs | 6 +++ packages/coln-query/src/relational/expr.rs | 44 +++++++++++++++++-- .../src/relational/incremental/interpreter.rs | 34 ++++++++++++-- .../coln-query/src/relational/relation.rs | 2 + .../coln-query/src/typing/type_resolver.rs | 28 ++++++++++-- 6 files changed, 133 insertions(+), 12 deletions(-) diff --git a/packages/coln-query/src/host/resolver.rs b/packages/coln-query/src/host/resolver.rs index e6b59348..422a6994 100644 --- a/packages/coln-query/src/host/resolver.rs +++ b/packages/coln-query/src/host/resolver.rs @@ -15,8 +15,8 @@ use crate::{ }, relational::expr::{ AliasExpr, AntiJoinExpr, CartesianProductExpr, DifferenceExpr, DistinctExpr, EquiJoinExpr, - FixedPointIterExpr, OutputExpr, ProjectionExpr, RelExpr, RelExprVisitorMut, SelectionExpr, - SourceExpr, UnionExpr, + FixedPointIterExpr, MultiWayEquiJoin, OutputExpr, ProjectionExpr, RelExpr, + RelExprVisitorMut, SelectionExpr, SourceExpr, UnionExpr, }, util::{Named, Resolvable}, }; @@ -388,6 +388,33 @@ impl RelExprVisitorMut> for Resolver { .and_then(|()| self.visit_projection_attributes(expr.attributes.as_mut(), ctx)) } + fn visit_multi_way_equi_join_expr( + &mut self, + expr: &mut MultiWayEquiJoin, + ctx: VisitorCtx<'_, '_>, + ) -> VisitorResult { + // TODO: Typecheck: A union is valid if the column types match and + // the amount of columns is the same. + if expr.relations.len() < 2 { + return Err(SyntaxError::new( + "A multi way equi join requires at least two relations", + )); + } + + expr.relations + .iter_mut() + .try_for_each(|relation| self.visit_expr(relation, ctx)) + .and_then(|()| { + expr.on.iter_mut().flatten().flatten().try_for_each(|expr| { + ctx.begin_tuple_context(); + let ret = self.visit_expr(expr, ctx); + ctx.end_tuple_context(); + ret + }) + }) + .and_then(|()| self.visit_projection_attributes(expr.attributes.as_mut(), ctx)) + } + fn visit_anti_join_expr(&mut self, expr: &mut AntiJoinExpr, ctx: VisitorCtx) -> VisitorResult { self.visit_expr(&mut expr.left, ctx) .and_then(|()| self.visit_expr(&mut expr.right, ctx)) diff --git a/packages/coln-query/src/optimizer/mod.rs b/packages/coln-query/src/optimizer/mod.rs index ba57c3c3..c39447c4 100644 --- a/packages/coln-query/src/optimizer/mod.rs +++ b/packages/coln-query/src/optimizer/mod.rs @@ -7,6 +7,12 @@ use crate::{error::OptimizationError, host::Code}; /// An optimizer does a series of transformations on the relational algebra IR /// which retain the semantics of the program but intend to improve performance. /// For instance, it could do: +/// - Collapse a PROJECT(JOIN(R, S, ON, []), ATTRS) into a JOIN(R, S, ON, ATTRS). +/// - Some Projections can be turned into a simple schema operations, e.g.: +/// - column rename +/// - column omission/reordering (beware that operations which require schemas +/// to be equal may fail in a "dirty" state, e.g., set difference and union) +/// - Decide a binary join ordering of an NWayJoin /// - predicate pushdown /// - expression simplification pub trait Optimizer: Clone { diff --git a/packages/coln-query/src/relational/expr.rs b/packages/coln-query/src/relational/expr.rs index a36de295..b87e6179 100644 --- a/packages/coln-query/src/relational/expr.rs +++ b/packages/coln-query/src/relational/expr.rs @@ -29,12 +29,16 @@ pub enum RelExpr { Output(Box), Alias(Box), Distinct(Box), + // A union can also be expressed with a full outer join and a projection. Union(Box), + // As the antijoin is a generalization of the set difference, this may be + // removed in the future. Difference(Box), Selection(Box), Projection(Box), CartesianProduct(Box), EquiJoin(Box), + MultiWayEquiJoin(Box), AntiJoin(Box), FixedPointIter(Box), } @@ -250,14 +254,40 @@ pub struct EquiJoinExpr { pub left: Expr, /// Must evaluate to a relation. pub right: Expr, - /// The attributes to join on. The first element of any pair belongs to the - /// left relation, and the second element of any pair belongs to right relation. - /// Each attribute pair should produce the same type. + /// The attribute(s) to join on. The first element of any pair is evaluated + /// in the context of the left relation, and the second element of any pair + /// is evaluated in the context of the right relation. + /// + /// If `on` is empty, a [`CartesianProduct`](CartesianProductExpr) is computed. pub on: Vec<(Expr, Expr)>, /// An optional projection step. See documentation of [`ProjectionExpr`]. pub attributes: Option>, } +/// An equijoin involving `N` relations. A better input than a folded sequence +/// of [binary `EquiJoin`s](EquiJoinExpr) for worst-case optimal join algorithms +/// (such as the leapfrog triejoin). +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MultiWayEquiJoin { + /// The `N` relations which participate in the join. Each [`Expr`] must + /// evaluate to a relation. + pub relations: Vec, + /// Each entry in the outer vector corresponds to a variable which must be + /// equal among all its occurrences. The inner vector vector tracks the + /// occurrences for each variable. The inner vector is _guaranteed_ to have + /// the same arity as the [`relations`](Self::relations) vector. An entry + /// at index `i` in the inner vector with value `None` indicates that the + /// corresponding relation ([`relations[i]`](Self::relations)) does _not_ + /// bind the variable, whereas a value of [`Some(Expr)`](Expr) binds the + /// variable to the value of the `Expr` evaluated in the context of the + /// corresponding relation (which is again [`relations[i]`](Self::relations)). + /// + /// If `on` is empty, a [`CartesianProduct`](CartesianProductExpr) is computed. + pub on: Vec>>, + /// An optional projection step. See documentation of [`ProjectionExpr`]. + pub attributes: Option>, +} + /// This is not a commutative operation, that is, swapping the `left` and `right` /// relations may alter the result. This computes `left` setminus `right` while /// only considering the columns specified in `on`. @@ -275,6 +305,7 @@ pub struct AntiJoinExpr { #[derive(Clone, Debug, Eq, PartialEq)] pub struct ThetaJoinExpr { + // Can be subsumed by the EquiJoin/MultiWayJoin, but better /// Must evaluate to a relation. pub left: Expr, /// Must evaluate to a relation. @@ -334,6 +365,7 @@ pub trait RelExprVisitor { RelExpr::Projection(expr) => self.visit_projection_expr(expr, ctx), RelExpr::CartesianProduct(expr) => self.visit_cartesian_product_expr(expr, ctx), RelExpr::EquiJoin(expr) => self.visit_equi_join_expr(expr, ctx), + RelExpr::MultiWayEquiJoin(expr) => self.visit_multi_way_equi_join_expr(expr, ctx), RelExpr::AntiJoin(expr) => self.visit_anti_join_expr(expr, ctx), RelExpr::FixedPointIter(expr) => self.visit_fixed_point_iter_expr(expr, ctx), } @@ -348,6 +380,7 @@ pub trait RelExprVisitor { fn visit_projection_expr(&mut self, expr: &ProjectionExpr, ctx: C) -> T; fn visit_cartesian_product_expr(&mut self, expr: &CartesianProductExpr, ctx: C) -> T; fn visit_equi_join_expr(&mut self, expr: &EquiJoinExpr, ctx: C) -> T; + fn visit_multi_way_equi_join_expr(&mut self, expr: &MultiWayEquiJoin, ctx: C) -> T; fn visit_anti_join_expr(&mut self, expr: &AntiJoinExpr, ctx: C) -> T; fn visit_fixed_point_iter_expr(&mut self, expr: &FixedPointIterExpr, ctx: C) -> T; } @@ -365,6 +398,7 @@ pub trait RelExprVisitorMut { RelExpr::Projection(expr) => self.visit_projection_expr(expr, ctx), RelExpr::CartesianProduct(expr) => self.visit_cartesian_product_expr(expr, ctx), RelExpr::EquiJoin(expr) => self.visit_equi_join_expr(expr, ctx), + RelExpr::MultiWayEquiJoin(expr) => self.visit_multi_way_equi_join_expr(expr, ctx), RelExpr::AntiJoin(expr) => self.visit_anti_join_expr(expr, ctx), RelExpr::FixedPointIter(expr) => self.visit_fixed_point_iter_expr(expr, ctx), } @@ -379,6 +413,7 @@ pub trait RelExprVisitorMut { fn visit_projection_expr(&mut self, expr: &mut ProjectionExpr, ctx: C) -> T; fn visit_cartesian_product_expr(&mut self, expr: &mut CartesianProductExpr, ctx: C) -> T; fn visit_equi_join_expr(&mut self, expr: &mut EquiJoinExpr, ctx: C) -> T; + fn visit_multi_way_equi_join_expr(&mut self, expr: &mut MultiWayEquiJoin, ctx: C) -> T; fn visit_anti_join_expr(&mut self, expr: &mut AntiJoinExpr, ctx: C) -> T; fn visit_fixed_point_iter_expr(&mut self, expr: &mut FixedPointIterExpr, ctx: C) -> T; } @@ -396,6 +431,7 @@ pub trait RelExprVisitorOwn { RelExpr::Projection(expr) => self.visit_projection_expr(*expr, ctx), RelExpr::CartesianProduct(expr) => self.visit_cartesian_product_expr(*expr, ctx), RelExpr::EquiJoin(expr) => self.visit_equi_join_expr(*expr, ctx), + RelExpr::MultiWayEquiJoin(expr) => self.visit_multi_way_equi_join_expr(*expr, ctx), RelExpr::AntiJoin(expr) => self.visit_anti_join_expr(*expr, ctx), RelExpr::FixedPointIter(expr) => self.visit_fixed_point_iter_expr(*expr, ctx), } @@ -410,6 +446,7 @@ pub trait RelExprVisitorOwn { fn visit_projection_expr(&mut self, expr: ProjectionExpr, ctx: C) -> T; fn visit_cartesian_product_expr(&mut self, expr: CartesianProductExpr, ctx: C) -> T; fn visit_equi_join_expr(&mut self, expr: EquiJoinExpr, ctx: C) -> T; + fn visit_multi_way_equi_join_expr(&mut self, expr: MultiWayEquiJoin, ctx: C) -> T; fn visit_anti_join_expr(&mut self, expr: AntiJoinExpr, ctx: C) -> T; fn visit_fixed_point_iter_expr(&mut self, expr: FixedPointIterExpr, ctx: C) -> T; } @@ -425,6 +462,7 @@ impl MemAddr for SelectionExpr {} impl MemAddr for ProjectionExpr {} impl MemAddr for CartesianProductExpr {} impl MemAddr for EquiJoinExpr {} +impl MemAddr for MultiWayEquiJoin {} impl MemAddr for AntiJoinExpr {} impl MemAddr for ThetaJoinExpr {} impl MemAddr for FixedPointIterExpr {} diff --git a/packages/coln-query/src/relational/incremental/interpreter.rs b/packages/coln-query/src/relational/incremental/interpreter.rs index 5f8855b3..1563f423 100644 --- a/packages/coln-query/src/relational/incremental/interpreter.rs +++ b/packages/coln-query/src/relational/incremental/interpreter.rs @@ -11,6 +11,7 @@ use super::operators::{ reindex::reindex_helper, }; use crate::relational::RelationSchema; +use crate::relational::expr::MultiWayEquiJoin; use crate::relational::incremental::dbsp::{ DbspInput, OrdIndexedStreamInputHandle, new_ord_indexed_stream, }; @@ -100,10 +101,24 @@ fn collect_source_exprs<'a>(stmts: &'a [Stmt], out: &mut Vec<&'a SourceExpr>) { } RelExpr::Projection(expr) => { walk_expr(&expr.relation, out); - expr.attributes.iter().for_each(|(_, e)| walk_expr(e, out)); + expr.attributes + .iter() + .for_each(|(_, expr)| walk_expr(expr, out)); } RelExpr::CartesianProduct(expr) => walk_equi_join(&expr.inner, out), RelExpr::EquiJoin(expr) => walk_equi_join(expr, out), + RelExpr::MultiWayEquiJoin(expr) => { + expr.relations.iter().for_each(|rel| walk_expr(rel, out)); + expr.on + .iter() + .flatten() + .flatten() + .for_each(|expr| walk_expr(expr, out)); + expr.attributes + .iter() + .flatten() + .for_each(|(_, expr)| walk_expr(expr, out)); + } RelExpr::AntiJoin(expr) => { walk_expr(&expr.left, out); walk_expr(&expr.right, out); @@ -125,9 +140,10 @@ fn collect_source_exprs<'a>(stmts: &'a [Stmt], out: &mut Vec<&'a SourceExpr>) { walk_expr(l, out); walk_expr(r, out); }); - if let Some(attributes) = &expr.attributes { - attributes.iter().for_each(|(_, e)| walk_expr(e, out)); - } + expr.attributes + .iter() + .flatten() + .for_each(|(_, expr)| walk_expr(expr, out)); } stmts.iter().for_each(|stmt| walk_stmt(stmt, out)); } @@ -586,6 +602,16 @@ impl RelExprVisitor> Ok(Value::Relation(new_relation(schema, joined))) } + fn visit_multi_way_equi_join_expr( + &mut self, + expr: &MultiWayEquiJoin, + ctx: VisitorCtx<'_, '_>, + ) -> ExprVisitorResult { + unimplemented!( + "Multi way equi joins are not supported by DBSP. Fold it into a sequence of binary equi joins prior to handing off to the DBSP backend." + ) + } + fn visit_anti_join_expr(&mut self, expr: &AntiJoinExpr, ctx: VisitorCtx) -> ExprVisitorResult { let left = self .visit_expr(&expr.left, ctx) diff --git a/packages/coln-query/src/relational/relation.rs b/packages/coln-query/src/relational/relation.rs index 797619c3..f1496d83 100644 --- a/packages/coln-query/src/relational/relation.rs +++ b/packages/coln-query/src/relational/relation.rs @@ -587,6 +587,8 @@ impl Display for RelationType { } impl RelationType { + // TODO: Maybe the relation type should be position-aware and allow for + // duplicated columns sharing the same name.. pub fn join(self, other: Self) -> Self { // We start with other to have duplicate fields' types be taken from self. let mut fields = other.fields; diff --git a/packages/coln-query/src/typing/type_resolver.rs b/packages/coln-query/src/typing/type_resolver.rs index c2ae37c9..141e144d 100644 --- a/packages/coln-query/src/typing/type_resolver.rs +++ b/packages/coln-query/src/typing/type_resolver.rs @@ -16,8 +16,8 @@ use crate::{ }, relational::expr::{ AliasExpr, AntiJoinExpr, CartesianProductExpr, DifferenceExpr, DistinctExpr, EquiJoinExpr, - FixedPointIterExpr, OutputExpr, ProjectionExpr, RelExpr, RelExprVisitor, SelectionExpr, - SourceExpr, UnionExpr, + FixedPointIterExpr, MultiWayEquiJoin, OutputExpr, ProjectionExpr, RelExpr, RelExprVisitor, + SelectionExpr, SourceExpr, UnionExpr, }, }; pub use crate::{ @@ -308,7 +308,7 @@ impl RelExprVisitor> for TypeResolver { let first = expr .relations .first() - .ok_or_else(|| SyntaxError::new("Union expr with only no operands"))?; + .ok_or_else(|| SyntaxError::new("Union expr with no operands"))?; self.visit_expr(first, ctx) } @@ -349,6 +349,28 @@ impl RelExprVisitor> for TypeResolver { self.visit_projection_attributes(joined, expr.attributes.as_ref(), ctx) } + fn visit_multi_way_equi_join_expr( + &mut self, + expr: &MultiWayEquiJoin, + ctx: VisitorCtx<'_, '_>, + ) -> VisitorResult { + let mut relations = expr.relations.iter(); + let first_relation_type = relations + .next() + .ok_or_else(|| SyntaxError::new("Multi way equi join with no operands")) + .and_then(|first_relation| { + self.visit_expr(first_relation, ctx) + .and_then(|expr_type| assert_type!(expr_type, ExprType::Relation)) + })?; + let joined = relations.try_fold(first_relation_type, |fold, relation| { + let relation_type = self + .visit_expr(relation, ctx) + .and_then(|expr_type| assert_type!(expr_type, ExprType::Relation))?; + Ok(fold.join(relation_type)) + })?; + self.visit_projection_attributes(joined, expr.attributes.as_ref(), ctx) + } + fn visit_anti_join_expr(&mut self, expr: &AntiJoinExpr, ctx: VisitorCtx) -> VisitorResult { self.visit_expr(&expr.left, ctx) } From ecb367881f390321428a03f9ea75fb8041f75669 Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Tue, 18 Aug 2026 16:33:52 +0200 Subject: [PATCH 08/58] [chore] lint fixes --- packages/coln-flir-rs/src/ir/path.rs | 6 +++--- packages/coln-flir-rs/src/schema.rs | 8 ++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/coln-flir-rs/src/ir/path.rs b/packages/coln-flir-rs/src/ir/path.rs index 0a86c648..8410d917 100644 --- a/packages/coln-flir-rs/src/ir/path.rs +++ b/packages/coln-flir-rs/src/ir/path.rs @@ -26,9 +26,9 @@ impl Path { } } -impl Into for Path { - fn into(self) -> String { - self.to_string() +impl From for String { + fn from(value: Path) -> Self { + value.to_string() } } diff --git a/packages/coln-flir-rs/src/schema.rs b/packages/coln-flir-rs/src/schema.rs index d41819fb..b2f3216b 100644 --- a/packages/coln-flir-rs/src/schema.rs +++ b/packages/coln-flir-rs/src/schema.rs @@ -110,11 +110,7 @@ impl From<&ir::TableEntry> for Option { .iter() .position(|column| column.path == *primary_key_column) .map(|idx| CompilerColIdx::Column(idx as u64)) - .expect( - &format!( - "Primary key column {primary_key_column} not found in base table {path}", - ) - ) + .unwrap_or_else(|| panic!("Primary key column {primary_key_column} not found in base table {path}")) }) .collect::>() }); @@ -320,7 +316,7 @@ impl From<&[CompilerCol]> for StoreEngineCols { let name = col.name.clone(); let (first, second) = match &col.ty { ir::ColType::RowId { path } => { - let [hash_col, ctr_col] = StoreEngineCols::foreign_key_cols(&name, &path); + let [hash_col, ctr_col] = StoreEngineCols::foreign_key_cols(&name, path); (hash_col, Some(ctr_col)) } ir::ColType::BuiltinTy { builtin_ty } => ( From 8f040fcbf4f35a656cb6aec2faa3fe8f136910e0 Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Thu, 20 Aug 2026 13:09:53 +0200 Subject: [PATCH 09/58] Backup --- packages/coln-flir-rs/src/ir/mod.rs | 5 +- packages/coln-flir-rs/src/ir/path.rs | 6 + packages/coln-query/src/api/query.rs | 497 ++++++++++++++++++--- packages/coln-query/src/api/schema.rs | 45 +- packages/coln-query/src/relational/expr.rs | 1 + 5 files changed, 463 insertions(+), 91 deletions(-) diff --git a/packages/coln-flir-rs/src/ir/mod.rs b/packages/coln-flir-rs/src/ir/mod.rs index f0c8860c..119ce232 100644 --- a/packages/coln-flir-rs/src/ir/mod.rs +++ b/packages/coln-flir-rs/src/ir/mod.rs @@ -156,8 +156,6 @@ pub enum Term { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ValueEntry { pub column: ColumnIdx, - /// Note: A [`Term::Lit`] together with a [`ColumnIdx`](Self::column) does - /// not make sense, I suppose. pub term: Term, } @@ -170,7 +168,8 @@ pub struct Atom { pub entity: Path, /// To bring the `row_id` of the [`Entity`](Self::entity) into scope. /// - /// Note: A [`Some(Term::Lit)`](Term::Lit) does not make sense, I suppose. + /// Note: A [`Some(Term::Lit)`](Term::Lit) does not make sense in this + /// context, as we do not support a row id literal at the moment, I suppose. pub row_id: Option, /// To bring some columns of the [`Entity`](Self::entity) into scope. pub values: Vec, diff --git a/packages/coln-flir-rs/src/ir/path.rs b/packages/coln-flir-rs/src/ir/path.rs index 8410d917..9347a6e0 100644 --- a/packages/coln-flir-rs/src/ir/path.rs +++ b/packages/coln-flir-rs/src/ir/path.rs @@ -32,6 +32,12 @@ impl From for String { } } +impl From<&Path> for String { + fn from(value: &Path) -> Self { + value.to_string() + } +} + impl Display for Path { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for (i, qname) in self.0.iter().enumerate() { diff --git a/packages/coln-query/src/api/query.rs b/packages/coln-query/src/api/query.rs index 37281cb2..fb29fc19 100644 --- a/packages/coln-query/src/api/query.rs +++ b/packages/coln-query/src/api/query.rs @@ -6,12 +6,21 @@ //! (FLIR) into a query program expressed in [`Statements`](crate::host::stmt::Stmt), //! using [`HostExprs`](crate::host::expr::Expr) and [`RelExprs`](crate::relational::expr::RelExpr). -use crate::api::schema::{Column, TableRef, TableSchema}; +#![allow(unreachable_code)] // Temporary due to todo!() annotations. +use crate::api::schema::{TableRef, TableSchema}; use crate::error::SyntaxError; use crate::host::Code; +use crate::host::expr::{BinaryExpr, Expr, Literal, LiteralExpr, VarExpr}; +use crate::host::operator::Operator; +use crate::host::stmt::{Stmt, VarStmt}; +use crate::relational::RelationSchema; +use crate::relational::expr::{ + AntiJoinExpr, EquiJoinExpr, MultiWayEquiJoin, ProjectionExpr, SelectionExpr, SourceExpr, +}; use coln_flir_rs::ir::{ - self, Atom, EntityVariant, FlatRealm, Prop, Rule, RuleEntry, TableEntry, Term, + self, Atom, EntityVariant, Equality, FlatRealm, Path, Prop, RuleEntry, TableEntry, Term, }; +use coln_flir_rs::schema::{BaseTableSchema, CompilerColIdx, StoreEngineCols}; use std::collections::HashMap; type BaseTableName = TableRef; @@ -33,9 +42,40 @@ impl> From for RuleName { struct QueryProgramBuilder { program: Code, - base_tables: HashMap, + base_tables: HashMap, derived_views: HashMap, - rules: HashMap, + rules: HashMap, +} + +struct RuleMeta { + kind: ir::RuleVariant, + output_schema: TableSchema, +} + +impl RuleMeta { + fn new(kind: ir::RuleVariant, output_schema: TableSchema) -> Self { + RuleMeta { + kind, + output_schema, + } + } +} + +impl From<&BaseTableSchema> for RelationSchema { + fn from(value: &BaseTableSchema) -> Self { + RelationSchema::new( + value.name().to_string(), + value.query_cols().iter().map(|col| col.name().to_string()), + value + .query_cols() + .iter() + // The first two columns are the row id columns and can act as + // the key for now. + .take(2) + .map(|col| col.name().to_string()), + ) + .expect("Actually infallible") + } } impl QueryProgramBuilder { @@ -53,17 +93,19 @@ impl QueryProgramBuilder { builder.table_declaration(table)?; } for rule in &flat_realm.rules { + if rule.rule.consequents.is_empty() { + // The compiler does not clean up after the lowering and emits + // useless rules after lowering, so we vacuum-clean here instead. + continue; + } builder.rule_declaration(rule)?; } Ok(builder) } fn table_declaration(&mut self, table_entry: &TableEntry) -> Result<(), SyntaxError> { - let name = table_entry.path.to_string(); - let schema = &table_entry.table; - - match &schema.entity_variant { - EntityVariant::Table => self.base_table(BaseTableName::from(name), schema), + match &table_entry.table.entity_variant { + EntityVariant::Table => self.base_table(table_entry), EntityVariant::View(materialization) => { unimplemented!("[Initial models] Materialized views defined through a query"); } @@ -72,28 +114,10 @@ impl QueryProgramBuilder { } } } - fn base_table(&mut self, name: BaseTableName, schema: &ir::Schema) -> Result<(), SyntaxError> { - let columns = schema.columns.iter().map(Column::from).collect(); - let primary_key = schema - .primary_key - .as_ref() - .map_or(Ok(Vec::new()), |compound_primary_key| { - compound_primary_key - .iter() - .map(|primary_key_column| { - schema - .columns - .iter() - .position(|column| column.path == *primary_key_column) - .ok_or_else(|| { - SyntaxError::new(format!( - "Primary key column {primary_key_column} not found in base table {name}", - )) - }) - }) - .collect::, SyntaxError>>() - })?; - let table_schema = TableSchema::new(name.clone(), columns, vec![primary_key]); + fn base_table(&mut self, table_entry: &ir::TableEntry) -> Result<(), SyntaxError> { + let name = BaseTableName::from(&table_entry.path); + let table_schema = + Option::::from(table_entry).expect("Broken precondition"); self.base_tables .insert(name.clone(), table_schema) .ok_or_else(|| SyntaxError::new(format!("Base table {name} defined multiple times"))) @@ -102,34 +126,405 @@ impl QueryProgramBuilder { fn rule_declaration(&mut self, rule_entry: &RuleEntry) -> Result<(), SyntaxError> { let name = rule_entry.path.to_string(); - let rule = &rule_entry.rule; + let Some(rule) = FriendlyRule::from(&rule_entry.rule) else { + // The rule is filtered out but not an error case. + return Ok(()); + }; + let stmt = self.rule(name, &rule)?; + self.program.push(stmt); + let rule_meta = RuleMeta::new(rule.kind, todo!("Table schema from rule declaration")); self.rules - .insert(RuleName::from(&name), rule.rule_variant.clone()) + .insert(RuleName::from(&name), rule_meta) .ok_or_else(|| SyntaxError::new(format!("Rule {name} defined multiple times")))?; - match &rule.rule_variant { - ir::RuleVariant::Enforced => { - todo!() + Ok(()) + } + fn rule(&mut self, name: String, rule: &FriendlyRule) -> Result { + let left = self.conjunctive_query(&rule.lhs, &rule.vars)?; + let right = self.conjunctive_query(&rule.rhs, &rule.vars)?; + let rule_as_stmt = Stmt::from(VarStmt { + name, + initializer: Some(Expr::from(AntiJoinExpr { + left, + right, + on: todo!("Take intersection of vars"), + })), + }); + Ok(rule_as_stmt) + } + fn conjunctive_query( + &mut self, + query: &ConjunctiveQuery, + vars: &Vec, + ) -> Result { + if query.atoms.is_empty() { + return Err(SyntaxError::new( + "FLIR emits conjunctive query with no atom", + )); + } + + let joined_atoms = Expr::from(MultiWayEquiJoin { + relations: query + .atoms + .iter() + .map(|atom| self.atom(atom, vars)) + .collect::, _>>()?, + // TODO: + on: vec![], + attributes: None, + }); + + let with_conditions = query + .conditions + .iter() + .map(|condition| self.selection(condition, vars)) + // // All conditions get compiled into one condition by ANDing them. + .try_reduce(|acc, conditions| { + Ok(Expr::from(BinaryExpr { + operator: Operator::And, + left: acc, + right: conditions, + })) + })? + .into_iter() + // We fold the Option: If there are no conditions at all, we return + // `joined_atoms` as is and otherwise, we wrap it in a SelectionExpr + // whose condition embodies all conditions. + .fold(joined_atoms, |joined_atoms, conditions| { + Expr::from(SelectionExpr { + relation: joined_atoms, + condition: conditions, + }) + }); + + Ok(with_conditions) + } + /// Generates a condition which possibly expands to two ANDed conditions + /// due to row ids being flattening to two variables. + /// + /// Currently, the compiler only supports equality conditions. + fn selection( + &mut self, + condition: &Equality, + vars: &Vec, + ) -> Result { + let left = self.term(&condition.left, vars)?; + let right = self.term(&condition.right, vars)?; + // Things get a bit ugly unfortunately due to the flattening of row ids. + let conditions: Box> = match (left.len(), right.len()) { + (2, 2) => { + // This case compares two row ids which expand to two variables + // each and thus we have to create two conditions. + // The underlying condition has to be true for both the hash + // _and_ the counter. In code that translates to the diagonal of + // the terms. + let diagonal = left.into_iter().zip(right.into_iter()); + Box::new(diagonal) + } + _ => { + // This case deals with comparing: + // 1. An already flat variable with a literal. + // But also covers two nonsense cases at the moment: + // 1. A row id with a literal or an already flat variable. + // 2. Two literals. + // In code this boils down to computing all pairs of the terms. + let cartesian_product = left + .into_iter() + .flat_map(|left| right.iter().map(move |right| (left.clone(), right.clone()))); + Box::new(cartesian_product) + } + }; + + Ok(conditions + .map(|(left, right)| { + Expr::from(BinaryExpr { + operator: Operator::Equal, + left, + right, + }) + }) + .reduce(|acc, condition| { + Expr::from(BinaryExpr { + operator: Operator::And, + left: acc, + right: condition, + }) + }) + .expect("A FLIR condition must produce at least one condition")) + } + fn atom(&mut self, atom: &Atom, vars: &Vec) -> Result { + let (source, schema): (Expr, &BaseTableSchema) = + if let Some((source_expr, schema)) = self.base_table_source_expr(&atom.entity) { + (Expr::from(source_expr), schema) + } else if let Some(var_expr) = self.derived_view_var_expr(&atom.entity) { + ( + Expr::from(var_expr), + todo!("Generic schema representation for derived views"), + ) + } else { + return Err(SyntaxError::new(format!( + "Atom references undeclared entity '{}'", + atom.entity + ))); + }; + + let attributes: Vec<(String, Expr)> = if let Some(row_id) = &atom.row_id { + // TODO: find abstraction for this Term dance. + match row_id { + ir::Term::Var { index } => { + let var = vars + .get(*index as usize) + .ok_or_else(|| SyntaxError::new("FLIR var idx out of bounds"))?; + match &var.ty { + ir::ColType::RowId { path } => { + let mut columns = + schema.resolve_query_cols(CompilerColIdx::for_row_id()); + let column_hash = columns.next().expect("No row id hash column"); + let column_ctr = columns.next().expect("No row id counter column"); + vec![ + ( + var.name + .clone() + .append(StoreEngineCols::HASH_COL_SUFFIX) + .to_string(), + Expr::from(VarExpr::new(column_hash.name())), + ), + ( + var.name + .clone() + .append(StoreEngineCols::CTR_COL_SUFFIX) + .to_string(), + Expr::from(VarExpr::new(column_ctr.name())), + ), + ] + } + ir::ColType::BuiltinTy { builtin_ty } => { + return Err(SyntaxError::new( + "FLIR wants to assign a row id to a variable of native scalar type", + )); + } + } + } + ir::Term::Lit { lit } => { + return Err(SyntaxError::new( + "FLIR expects row id to be equal to a literal", + )); + } + } + } else { + vec![] + }; + + let (conditions, attributes) = atom.values.iter().try_fold((vec![], attributes), |(mut conditions, mut attributes), value| { + let mut columns = schema.resolve_query_cols(CompilerColIdx::from(value.column)); + match &value.term { + ir::Term::Lit { lit } => { + let column = columns.next().expect("A literal can only ever be compared to a single column because it cannot store a row id"); + conditions.push(Expr::from(BinaryExpr { + operator: Operator::Equal, + left: Expr::from(VarExpr::new(column.name())), + right: Expr::from(LiteralExpr::from(Literal::from(lit))), + })) + }, + ir::Term::Var { index } => { + let var = vars + .get(*index as usize) + .ok_or_else(|| SyntaxError::new("FLIR var idx out of bounds"))?; + match &var.ty { + ir::ColType::BuiltinTy { builtin_ty } => { + let column = columns.next().expect("A var of a native scalar type can only ever reference a single value column"); + attributes.push((var.name.to_string(), Expr::from( + VarExpr::new(column.name()), + ))) + }, + ir::ColType::RowId { path } => { + let column_hash = columns.next().expect("No row id hash column"); + let column_ctr = columns.next().expect("No row id counter column"); + attributes.push((var.name.clone().append(StoreEngineCols::HASH_COL_SUFFIX).to_string(), Expr::from( + VarExpr::new(column_hash.name()) + ))); + attributes.push((var.name.clone().append(StoreEngineCols::CTR_COL_SUFFIX).to_string(), Expr::from( + VarExpr::new(column_ctr.name()) + ))); + }, + } + }, + }; + Ok((conditions, attributes)) + })?; + + let with_selection = conditions + .into_iter() + .reduce(|acc, condition| { + Expr::from(BinaryExpr { + operator: Operator::And, + left: acc, + right: condition, + }) + }) + .into_iter() + .fold(source, |source, root_condition| { + Expr::from(SelectionExpr { + relation: source, + condition: root_condition, + }) + }); + + let with_projection = Expr::from(ProjectionExpr { + relation: with_selection, + attributes, + }); + + Ok(with_projection) + } + fn term(&mut self, term: &Term, vars: &Vec) -> Result, SyntaxError> { + match term { + Term::Lit { lit } => Ok(vec![Expr::from(LiteralExpr::from(Literal::from(lit)))]), + Term::Var { index } => { + let var = vars + .get(*index as usize) + .ok_or_else(|| SyntaxError::new("FLIR var idx out of bounds"))?; + match &var.ty { + ir::ColType::BuiltinTy { builtin_ty: _ } => { + Ok(vec![Expr::from(VarExpr::new(var.name.to_string()))]) + } + ir::ColType::RowId { path: _ } => Ok(vec![ + Expr::from(VarExpr::new( + var.name.clone().append(StoreEngineCols::HASH_COL_SUFFIX), + )), + Expr::from(VarExpr::new( + var.name.clone().append(StoreEngineCols::CTR_COL_SUFFIX), + )), + ]), + } } - ir::RuleVariant::Monitored => todo!(), - ir::RuleVariant::Chased => todo!( - // TODO: clarify - "Chased rules produce a materialized view; how are they different from a materialized view defined in the table section?" - ), } } - fn rule(&mut self, rule: &Rule) -> Result<(), SyntaxError> { - todo!() + /// If the entity referenced by `Path` is part of the extensional database + /// (EDB) and present in the base tables, the function returns a + /// [`SourceExpr`] referencing that entity. Otherwise, [`None`] is returned. + fn base_table_source_expr(&mut self, name: &Path) -> Option<(SourceExpr, &BaseTableSchema)> { + self.base_tables + .get(&BaseTableName::from(name)) + .map(|base_table_schema| { + ( + SourceExpr { + schema: RelationSchema::from(base_table_schema), + }, + base_table_schema, + ) + }) + } + /// If the entity referenced by `Path` is part of the intensional database + /// (IDB) and present in the derived views, the function returns a + /// [`VarExpr`] referencing that entity. Due to coln-compiler declaring + /// tables and views prior to the rules, said entity must be known at this + /// point. Otherwise, [`None`] is returned. + fn derived_view_var_expr(&mut self, name: &Path) -> Option { + self.derived_views + .get(&DerivedViewName::from(name)) + .map(|_derived_view_schema| VarExpr::new(name.to_string())) } - fn prop(&mut self, prop: &Prop) -> Result<(), SyntaxError> { - match prop { - Prop::Atom { atom } => todo!(), - Prop::Eq { left, right } => todo!(), +} + +/// Just like [`ir::Rule`] but friendlier because: +/// +/// 1. Meaningless rules with an empty [consequent](Rule::consequents) are +/// skipped and chased rules panic at the moment due to open questions. +/// 2. It zips the [`Rule::var_names`] and the [`Rule::var_types`] into one +/// array of [`FriendlyVar`]s. +/// 3. It converts [`Rule::antecedents`] and [`Rule::consequents`] into a +/// [`ConjunctiveQuery`], each. +struct FriendlyRule { + kind: ir::RuleVariant, + vars: Vec, + lhs: ConjunctiveQuery, + rhs: ConjunctiveQuery, +} + +impl FriendlyRule { + fn from(rule: &ir::Rule) -> Option { + if rule.consequents.is_empty() { + return None; } + if matches!(rule.rule_variant, ir::RuleVariant::Chased) { + unimplemented!( + "[Unclear] Chased rules produce a materialized view; how are they different from a materialized view defined in the table/entities section?" + ); + } + assert!( + rule.var_names.len() == rule.var_types.len(), + "var_names and var_types arrays do not size match" + ); + let vars = rule + .var_names + .iter() + .zip(rule.var_types.iter()) + .map(|(path, col_type)| FriendlyVar { + name: path.clone(), + ty: col_type.clone(), + }) + .collect(); + let lhs = ConjunctiveQuery::from(&rule.antecedents); + let rhs = ConjunctiveQuery::from(&rule.consequents); + Some(FriendlyRule { + kind: rule.rule_variant, + vars, + lhs, + rhs, + }) } - fn atom(&mut self, atom: &Atom) -> Result<(), SyntaxError> { - todo!() +} + +/// Prepares either a [left-hand side](Rule::antecedents) or a +/// [right-hand side](Rule::consequents) of a [`Rule`] for inclusion in an +/// antijoin by partitioning a `Vec` into atoms and conditions. This is +/// useful because applying all atoms first, guarantees that every variable a +/// condition may refer to is in scope already. +struct ConjunctiveQuery { + atoms: Vec, + // Currently, only equality conditions are part of the FLIR. + conditions: Vec, +} + +impl ConjunctiveQuery { + fn from(props: &Vec) -> Self { + let (atoms, conditions) = + props + .iter() + .fold((vec![], vec![]), |(mut atoms, mut conditions), prop| { + match prop { + ir::Prop::Atom { atom } => atoms.push(atom.clone()), + ir::Prop::Eq { equality } => conditions.push(equality.clone()), + } + (atoms, conditions) + }); + Self { atoms, conditions } } - fn term(&mut self, term: &Term) -> Result<(), SyntaxError> { - todo!() +} + +/// All information from [`ir::Rule::var_names`] and [`ir::Rule::var_types`] but +/// _zipped_. +struct FriendlyVar { + name: ir::Path, + ty: ir::ColType, // either a row id or a builtin type +} + +pub trait TryReduceResultExt: Iterator> { + /// Reduces an iterator yielding `Result`, short-circuiting if an `Err` + /// is yielded or if the reduction closure returns an `Err`. + fn try_reduce(mut self, mut f: F) -> Result, E> + where + Self: Sized, + F: FnMut(T, T) -> Result, + { + let first = match self.next() { + Some(Ok(v)) => v, + Some(Err(e)) => return Err(e), + None => return Ok(None), + }; + + self.try_fold(first, |acc, item| f(acc, item?)).map(Some) } } + +impl TryReduceResultExt for I where I: Iterator> {} diff --git a/packages/coln-query/src/api/schema.rs b/packages/coln-query/src/api/schema.rs index f817aa91..f1000210 100644 --- a/packages/coln-query/src/api/schema.rs +++ b/packages/coln-query/src/api/schema.rs @@ -7,7 +7,7 @@ use std::fmt::Display; -use crate::scalarial::{ScalarType, ScalarTypedValue}; +use crate::{host::expr::Literal, scalarial::ScalarType}; /// An identifier that uniquely identifies a table (globally across the store). #[derive(Eq, PartialEq, Hash, Debug, Clone)] @@ -21,10 +21,10 @@ impl Display for TableRef { } } -impl> From for TableRef { - fn from(value: T) -> Self { +impl From<&ir::Path> for TableRef { + fn from(value: &ir::Path) -> Self { TableRef { - inner: value.into(), + inner: value.to_string(), } } } @@ -35,7 +35,7 @@ pub struct TableSchema { /// All fields of the table in their physical order. columns: Vec, /// The list of (possibly compound) primary keys into the table, specified - /// as indices into the [`columns`](Self::columns). + /// as indexes into the schema's [`columns`](Self::columns). primary_keys: Vec>, } @@ -58,40 +58,11 @@ pub struct Column { use coln_flir_rs::ir::{self}; -impl From<&ir::ColumnEntry> for Column { - fn from(value: &ir::ColumnEntry) -> Self { - // For now we use the flattened path representation in the query engine. - let name = value.path.to_string(); - let scalar_type = ScalarType::from(&value.col_type); - Column { name, scalar_type } - } -} - -impl From<&ir::ColType> for ScalarType { - fn from(value: &ir::ColType) -> Self { - match value { - ir::ColType::BuiltinTy { builtin_ty } => ScalarType::from(*builtin_ty), - // We assume that row ids will be sent as unsigned integers by coln-store. - ir::ColType::RowId { path } => ScalarType::Uint, - } - } -} - -impl From for ScalarType { - fn from(value: ir::BuiltinTy) -> Self { - match value { - // TODO: Discuss scalar types and their mappings. - ir::BuiltinTy::BuiltinInt => ScalarType::Iint, - ir::BuiltinTy::BuiltinStr => ScalarType::String, - } - } -} - -impl From<&ir::Lit> for ScalarTypedValue { +impl From<&ir::Lit> for Literal { fn from(value: &ir::Lit) -> Self { match value { - ir::Lit::Int { value } => ScalarTypedValue::Iint(*value), - ir::Lit::String { value } => ScalarTypedValue::String(value.clone()), + ir::Lit::Int { value } => Literal::Iint(*value), + ir::Lit::String { value } => Literal::String(value.clone()), } } } diff --git a/packages/coln-query/src/relational/expr.rs b/packages/coln-query/src/relational/expr.rs index b87e6179..be00f0a7 100644 --- a/packages/coln-query/src/relational/expr.rs +++ b/packages/coln-query/src/relational/expr.rs @@ -75,6 +75,7 @@ impl_rel_and_expr_from! { (RelExpr::Projection, ProjectionExpr), (RelExpr::CartesianProduct, CartesianProductExpr), (RelExpr::EquiJoin, EquiJoinExpr), + (RelExpr::MultiWayEquiJoin, MultiWayEquiJoin), (RelExpr::AntiJoin, AntiJoinExpr), (RelExpr::FixedPointIter, FixedPointIterExpr), } From c8ca3e3d78d5286572feacadeef2607c28d1dbfb Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Thu, 20 Aug 2026 15:04:14 +0200 Subject: [PATCH 10/58] Initial support for mapping coln-flir to a query program --- packages/coln-flir-rs/src/schema.rs | 4 + packages/coln-query/src/api/query.rs | 1201 ++++++++++++++--- packages/coln-query/src/api/schema.rs | 46 + packages/coln-query/src/host/resolver.rs | 18 +- packages/coln-query/src/relational/expr.rs | 307 ++++- .../src/relational/incremental/interpreter.rs | 10 +- .../coln-query/src/typing/type_resolver.rs | 6 +- 7 files changed, 1385 insertions(+), 207 deletions(-) diff --git a/packages/coln-flir-rs/src/schema.rs b/packages/coln-flir-rs/src/schema.rs index b2f3216b..ac8a2fb0 100644 --- a/packages/coln-flir-rs/src/schema.rs +++ b/packages/coln-flir-rs/src/schema.rs @@ -192,6 +192,10 @@ impl Col { pub fn name(&self) -> &ir::ColName { &self.name } + /// The column's (scalar) type, in whichever engine's view `T` belongs to. + pub fn ty(&self) -> &T { + &self.ty + } } /// Column metadata from the perspective of the compiler. diff --git a/packages/coln-query/src/api/query.rs b/packages/coln-query/src/api/query.rs index fb29fc19..03a9ac2d 100644 --- a/packages/coln-query/src/api/query.rs +++ b/packages/coln-query/src/api/query.rs @@ -6,8 +6,7 @@ //! (FLIR) into a query program expressed in [`Statements`](crate::host::stmt::Stmt), //! using [`HostExprs`](crate::host::expr::Expr) and [`RelExprs`](crate::relational::expr::RelExpr). -#![allow(unreachable_code)] // Temporary due to todo!() annotations. -use crate::api::schema::{TableRef, TableSchema}; +use crate::api::schema::{Column, TableRef, TableSchema}; use crate::error::SyntaxError; use crate::host::Code; use crate::host::expr::{BinaryExpr, Expr, Literal, LiteralExpr, VarExpr}; @@ -15,36 +14,31 @@ use crate::host::operator::Operator; use crate::host::stmt::{Stmt, VarStmt}; use crate::relational::RelationSchema; use crate::relational::expr::{ - AntiJoinExpr, EquiJoinExpr, MultiWayEquiJoin, ProjectionExpr, SelectionExpr, SourceExpr, + AntiJoinExpr, JoinVariable, MultiWayEquiJoinExpr, ProjectionExpr, RelationIdx, SelectionExpr, + SourceExpr, }; +use crate::scalarial::ScalarType; use coln_flir_rs::ir::{ self, Atom, EntityVariant, Equality, FlatRealm, Path, Prop, RuleEntry, TableEntry, Term, }; -use coln_flir_rs::schema::{BaseTableSchema, CompilerColIdx, StoreEngineCols}; -use std::collections::HashMap; +use coln_flir_rs::schema::{BaseTableSchema, CompilerColIdx, QueryEngineCol, StoreEngineCols}; +use std::collections::hash_map::Entry; +use std::collections::{BTreeMap, HashMap}; type BaseTableName = TableRef; type DerivedViewName = TableRef; -/// An identifier that uniquely identifies a table (globally across the store). -#[derive(Eq, PartialEq, Hash, Debug, Clone)] -pub struct RuleName { - inner: String, -} - -impl> From for RuleName { - fn from(value: T) -> Self { - RuleName { - inner: value.into(), - } - } -} - struct QueryProgramBuilder { program: Code, base_tables: HashMap, - derived_views: HashMap, - rules: HashMap, + /// The relations the program itself defines, that is, one per declared rule. + /// + /// This doubles as the set of derived views an [`Atom`] may reference, so + /// that what [`rule_declaration`](Self::rule_declaration) writes is exactly + /// what [`derived_view_var_expr`](Self::derived_view_var_expr) reads. Two + /// maps for the two roles previously meant nothing ever populated the one + /// being read. + derived_views: HashMap, } struct RuleMeta { @@ -84,7 +78,6 @@ impl QueryProgramBuilder { program: Vec::new(), base_tables: HashMap::new(), derived_views: HashMap::new(), - rules: HashMap::new(), } } pub fn from_flat_realm(flat_realm: &FlatRealm) -> Result { @@ -118,72 +111,111 @@ impl QueryProgramBuilder { let name = BaseTableName::from(&table_entry.path); let table_schema = Option::::from(table_entry).expect("Broken precondition"); - self.base_tables + if self + .base_tables .insert(name.clone(), table_schema) - .ok_or_else(|| SyntaxError::new(format!("Base table {name} defined multiple times"))) - .map(|_old_entry| ()) + .is_some() + { + return Err(SyntaxError::new(format!( + "Base table {name} defined multiple times" + ))); + } + Ok(()) } fn rule_declaration(&mut self, rule_entry: &RuleEntry) -> Result<(), SyntaxError> { - let name = rule_entry.path.to_string(); + let name = DerivedViewName::from(&rule_entry.path); let Some(rule) = FriendlyRule::from(&rule_entry.rule) else { // The rule is filtered out but not an error case. return Ok(()); }; - let stmt = self.rule(name, &rule)?; + let (stmt, output_bindings) = self.rule(name.to_string(), &rule)?; self.program.push(stmt); - let rule_meta = RuleMeta::new(rule.kind, todo!("Table schema from rule declaration")); - self.rules - .insert(RuleName::from(&name), rule_meta) - .ok_or_else(|| SyntaxError::new(format!("Rule {name} defined multiple times")))?; + let rule_meta = RuleMeta::new(rule.kind, rule_output_schema(&name, &output_bindings)); + // See `base_table` on the direction of this check. + if self.derived_views.insert(name.clone(), rule_meta).is_some() { + return Err(SyntaxError::new(format!( + "Rule {name} defined multiple times" + ))); + } Ok(()) } - fn rule(&mut self, name: String, rule: &FriendlyRule) -> Result { - let left = self.conjunctive_query(&rule.lhs, &rule.vars)?; - let right = self.conjunctive_query(&rule.rhs, &rule.vars)?; + /// Lowers one rule into the statement that binds its name, and reports the + /// [`Binding`]s of the relation that statement evaluates to, so the caller + /// can describe the rule's output schema. + fn rule( + &mut self, + name: String, + rule: &FriendlyRule, + ) -> Result<(Stmt, Vec), SyntaxError> { + let (left, left_bindings) = self.conjunctive_query(&rule.lhs, &rule.vars)?; + let (right, right_bindings) = self.conjunctive_query(&rule.rhs, &rule.vars)?; let rule_as_stmt = Stmt::from(VarStmt { name, initializer: Some(Expr::from(AntiJoinExpr { left, right, - on: todo!("Take intersection of vars"), + on: antijoin_key(&left_bindings, &right_bindings), })), }); - Ok(rule_as_stmt) + // An antijoin carries the left relation's tuple through unchanged, so + // the rule's output is shaped by its antecedents. + Ok((rule_as_stmt, left_bindings)) } + /// Lowers one side of a rule into a relational expression, and reports which + /// variable parts that expression binds so the enclosing [`AntiJoinExpr`] + /// can work out what to compare on. fn conjunctive_query( &mut self, query: &ConjunctiveQuery, - vars: &Vec, - ) -> Result { + vars: &[FriendlyVar], + ) -> Result<(Expr, Vec), SyntaxError> { if query.atoms.is_empty() { return Err(SyntaxError::new( "FLIR emits conjunctive query with no atom", )); } - let joined_atoms = Expr::from(MultiWayEquiJoin { - relations: query - .atoms - .iter() - .map(|atom| self.atom(atom, vars)) - .collect::, _>>()?, - // TODO: - on: vec![], - attributes: None, - }); + let plans = query + .atoms + .iter() + .map(|atom| self.atom(atom, vars)) + .collect::, _>>()?; + + // A part bound by several atoms is a single binding of the conjunctive + // query as a whole, because the join keeps one active copy of it. We + // keep the first, matching the join's left-to-right shadowing. + let mut bindings: BTreeMap<(ir::VarIdx, VarPart), Binding> = BTreeMap::new(); + for plan in &plans { + for binding in &plan.bindings { + bindings + .entry((binding.var, binding.part)) + .or_insert_with(|| binding.clone()); + } + } + let bindings = bindings.into_values().collect(); + + let on = join_variables(&plans); + let mut relations: Vec = plans.into_iter().map(|plan| plan.relation).collect(); + let joined_atoms = if relations.len() == 1 { + // A single atom has nothing to join against, and the join operators + // require at least two relations. + relations.pop().expect("Length checked") + } else { + Expr::from(MultiWayEquiJoinExpr::new(relations, on, None)?) + }; let with_conditions = query .conditions .iter() .map(|condition| self.selection(condition, vars)) - // // All conditions get compiled into one condition by ANDing them. - .try_reduce(|acc, conditions| { - Ok(Expr::from(BinaryExpr { + // All conditions get compiled into one condition by ANDing them. + .try_reduce(|left, right| { + Expr::from(BinaryExpr { operator: Operator::And, - left: acc, - right: conditions, - })) + left, + right, + }) })? .into_iter() // We fold the Option: If there are no conditions at all, we return @@ -196,7 +228,7 @@ impl QueryProgramBuilder { }) }); - Ok(with_conditions) + Ok((with_conditions, bindings)) } /// Generates a condition which possibly expands to two ANDed conditions /// due to row ids being flattening to two variables. @@ -205,7 +237,7 @@ impl QueryProgramBuilder { fn selection( &mut self, condition: &Equality, - vars: &Vec, + vars: &[FriendlyVar], ) -> Result { let left = self.term(&condition.left, vars)?; let right = self.term(&condition.right, vars)?; @@ -217,7 +249,7 @@ impl QueryProgramBuilder { // The underlying condition has to be true for both the hash // _and_ the counter. In code that translates to the diagonal of // the terms. - let diagonal = left.into_iter().zip(right.into_iter()); + let diagonal = left.into_iter().zip(right); Box::new(diagonal) } _ => { @@ -251,7 +283,10 @@ impl QueryProgramBuilder { }) .expect("A FLIR condition must produce at least one condition")) } - fn atom(&mut self, atom: &Atom, vars: &Vec) -> Result { + // Scoped to this function because of the derived-view `todo!()` below; + // the rest of the module is checked for unreachable code again. + #[allow(unreachable_code)] + fn atom(&mut self, atom: &Atom, vars: &[FriendlyVar]) -> Result { let (source, schema): (Expr, &BaseTableSchema) = if let Some((source_expr, schema)) = self.base_table_source_expr(&atom.entity) { (Expr::from(source_expr), schema) @@ -267,92 +302,56 @@ impl QueryProgramBuilder { ))); }; - let attributes: Vec<(String, Expr)> = if let Some(row_id) = &atom.row_id { - // TODO: find abstraction for this Term dance. + let mut binder = AtomBinder::default(); + + // The row id, if this atom brings it into scope. + if let Some(row_id) = &atom.row_id { match row_id { ir::Term::Var { index } => { - let var = vars - .get(*index as usize) - .ok_or_else(|| SyntaxError::new("FLIR var idx out of bounds"))?; - match &var.ty { - ir::ColType::RowId { path } => { - let mut columns = - schema.resolve_query_cols(CompilerColIdx::for_row_id()); - let column_hash = columns.next().expect("No row id hash column"); - let column_ctr = columns.next().expect("No row id counter column"); - vec![ - ( - var.name - .clone() - .append(StoreEngineCols::HASH_COL_SUFFIX) - .to_string(), - Expr::from(VarExpr::new(column_hash.name())), - ), - ( - var.name - .clone() - .append(StoreEngineCols::CTR_COL_SUFFIX) - .to_string(), - Expr::from(VarExpr::new(column_ctr.name())), - ), - ] - } - ir::ColType::BuiltinTy { builtin_ty } => { - return Err(SyntaxError::new( - "FLIR wants to assign a row id to a variable of native scalar type", - )); - } + let var = friendly_var(vars, *index)?; + if !var.is_row_id() { + return Err(SyntaxError::new( + "FLIR wants to assign a row id to a variable of native scalar type", + )); } + binder.bind( + *index, + var, + schema.resolve_query_cols(CompilerColIdx::for_row_id()), + )?; } - ir::Term::Lit { lit } => { + ir::Term::Lit { lit: _ } => { + // Matching [`ir::Atom::row_id`]'s own note: a literal row id + // is not something we can express. return Err(SyntaxError::new( - "FLIR expects row id to be equal to a literal", + "FLIR equates a row id with a literal, which is not supported", )); } } - } else { - vec![] - }; + } - let (conditions, attributes) = atom.values.iter().try_fold((vec![], attributes), |(mut conditions, mut attributes), value| { + // The value columns this atom constrains or brings into scope. + for value in &atom.values { let mut columns = schema.resolve_query_cols(CompilerColIdx::from(value.column)); match &value.term { ir::Term::Lit { lit } => { - let column = columns.next().expect("A literal can only ever be compared to a single column because it cannot store a row id"); - conditions.push(Expr::from(BinaryExpr { + let column = columns.next().ok_or_else(|| { + SyntaxError::new("FLIR compares a literal against a column that does not resolve to any query column") + })?; + binder.conditions.push(Expr::from(BinaryExpr { operator: Operator::Equal, left: Expr::from(VarExpr::new(column.name())), right: Expr::from(LiteralExpr::from(Literal::from(lit))), - })) - }, + })); + } ir::Term::Var { index } => { - let var = vars - .get(*index as usize) - .ok_or_else(|| SyntaxError::new("FLIR var idx out of bounds"))?; - match &var.ty { - ir::ColType::BuiltinTy { builtin_ty } => { - let column = columns.next().expect("A var of a native scalar type can only ever reference a single value column"); - attributes.push((var.name.to_string(), Expr::from( - VarExpr::new(column.name()), - ))) - }, - ir::ColType::RowId { path } => { - let column_hash = columns.next().expect("No row id hash column"); - let column_ctr = columns.next().expect("No row id counter column"); - attributes.push((var.name.clone().append(StoreEngineCols::HASH_COL_SUFFIX).to_string(), Expr::from( - VarExpr::new(column_hash.name()) - ))); - attributes.push((var.name.clone().append(StoreEngineCols::CTR_COL_SUFFIX).to_string(), Expr::from( - VarExpr::new(column_ctr.name()) - ))); - }, - } - }, - }; - Ok((conditions, attributes)) - })?; + binder.bind(*index, friendly_var(vars, *index)?, columns)?; + } + } + } - let with_selection = conditions + let with_selection = binder + .conditions .into_iter() .reduce(|acc, condition| { Expr::from(BinaryExpr { @@ -369,34 +368,23 @@ impl QueryProgramBuilder { }) }); - let with_projection = Expr::from(ProjectionExpr { + let relation = Expr::from(ProjectionExpr { relation: with_selection, - attributes, + attributes: binder.attributes, }); - Ok(with_projection) + Ok(AtomPlan { + relation, + bindings: binder.bindings, + }) } - fn term(&mut self, term: &Term, vars: &Vec) -> Result, SyntaxError> { + fn term(&mut self, term: &Term, vars: &[FriendlyVar]) -> Result, SyntaxError> { match term { Term::Lit { lit } => Ok(vec![Expr::from(LiteralExpr::from(Literal::from(lit)))]), - Term::Var { index } => { - let var = vars - .get(*index as usize) - .ok_or_else(|| SyntaxError::new("FLIR var idx out of bounds"))?; - match &var.ty { - ir::ColType::BuiltinTy { builtin_ty: _ } => { - Ok(vec![Expr::from(VarExpr::new(var.name.to_string()))]) - } - ir::ColType::RowId { path: _ } => Ok(vec![ - Expr::from(VarExpr::new( - var.name.clone().append(StoreEngineCols::HASH_COL_SUFFIX), - )), - Expr::from(VarExpr::new( - var.name.clone().append(StoreEngineCols::CTR_COL_SUFFIX), - )), - ]), - } - } + Term::Var { index } => Ok(friendly_var(vars, *index)? + .parts() + .map(|(_part, name)| Expr::from(VarExpr::new(name))) + .collect()), } } /// If the entity referenced by `Path` is part of the extensional database @@ -487,7 +475,7 @@ struct ConjunctiveQuery { } impl ConjunctiveQuery { - fn from(props: &Vec) -> Self { + fn from(props: &[Prop]) -> Self { let (atoms, conditions) = props .iter() @@ -509,22 +497,911 @@ struct FriendlyVar { ty: ir::ColType, // either a row id or a builtin type } -pub trait TryReduceResultExt: Iterator> { - /// Reduces an iterator yielding `Result`, short-circuiting if an `Err` - /// is yielded or if the reduction closure returns an `Err`. - fn try_reduce(mut self, mut f: F) -> Result, E> +impl FriendlyVar { + fn is_row_id(&self) -> bool { + matches!(self.ty, ir::ColType::RowId { path: _ }) + } + /// The attribute name(s) this variable expands to in an atom's projected + /// schema: one for a builtin scalar type, two for a row id, which flattens + /// into a commit hash and a counter column. + /// + /// This is the single place that flattening happens, and its order matches + /// the order + /// [`resolve_query_cols`](BaseTableSchema::resolve_query_cols) yields the + /// corresponding columns in, so the two can be zipped. + fn parts(&self) -> impl Iterator { + match &self.ty { + ir::ColType::BuiltinTy { builtin_ty: _ } => { + vec![(VarPart::Scalar, self.name.to_string())] + } + ir::ColType::RowId { path: _ } => vec![ + ( + VarPart::RowIdHash, + self.name + .clone() + .append(StoreEngineCols::HASH_COL_SUFFIX) + .to_string(), + ), + ( + VarPart::RowIdCtr, + self.name + .clone() + .append(StoreEngineCols::CTR_COL_SUFFIX) + .to_string(), + ), + ], + } + .into_iter() + } +} + +/// Which of the attributes a FLIR variable expands to, see +/// [`FriendlyVar::parts`]. The derived ordering keeps a row id's two halves +/// adjacent and in flattening order. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +enum VarPart { + /// The variable in its entirety. It is of a builtin scalar type. + Scalar, + /// The commit-hash half of a row id. + RowIdHash, + /// The counter half of a row id. + RowIdCtr, +} + +/// One attribute an atom's projection exposes, tagged with the FLIR variable it +/// originates from. +/// +/// The join condition of a conjunctive query is derived by grouping these on +/// [`var`](Self::var) and [`part`](Self::part) rather than on +/// [`name`](Self::name): the variable index is exact and free, whereas grouping +/// by name would additionally assume that rendering [`ir::Path`]s into strings +/// is injective, and would have to tell a row id's two halves apart by parsing +/// their suffixes back off. +#[derive(Clone, Debug)] +struct Binding { + var: ir::VarIdx, + part: VarPart, + /// The attribute's name in the atom's projected schema. + name: String, + /// The type of the query column this part is bound to. Taken from the + /// column rather than from the FLIR variable, because that is where a row + /// id's halves have already been resolved to their query-engine types. + scalar_type: ScalarType, +} + +/// The relational plan for one FLIR [`Atom`], together with the [`Binding`]s +/// its projection exposes. +/// +/// Reporting the bindings is what lets the enclosing conjunctive query derive +/// its join variables without re-deriving them from the projection it just +/// built. +struct AtomPlan { + /// Essentially, a `Projection(Selection(atom's source relation))`. + relation: Expr, + /// The [`Binding`]s of the plan. + bindings: Vec, +} + +/// Accumulates what one atom contributes while its row id and value terms are +/// walked. +#[derive(Default)] +struct AtomBinder { + /// Conditions local to this atom: literal comparisons, plus the equalities + /// that a variable repeated within this one atom gives rise to, that is, + /// `atom(x, x)`. + conditions: Vec, + /// The atom's projection, mapping each bound variable part onto the query + /// column carrying it. + attributes: Vec<(String, Expr)>, + bindings: Vec, + /// The query column each variable part was *first* bound to in this atom, so + /// a repeated occurrence can be turned into an equality against it. + bound: HashMap<(ir::VarIdx, VarPart), String>, +} + +impl AtomBinder { + /// Binds `var`'s parts to `columns`, which must resolve to one query column + /// per part. + fn bind<'a>( + &mut self, + index: ir::VarIdx, + var: &FriendlyVar, + columns: impl Iterator, + ) -> Result<(), SyntaxError> { + let parts: Vec<(VarPart, String)> = var.parts().collect(); + let columns: Vec<&QueryEngineCol> = columns.collect(); + if parts.len() != columns.len() { + return Err(SyntaxError::new(format!( + "FLIR binds variable '{}', which flattens into {} column(s), to a \ + column resolving to {} query column(s)", + var.name, + parts.len(), + columns.len() + ))); + } + for ((part, name), column) in parts.into_iter().zip(columns) { + let scalar_type = ScalarType::from(*column.ty()); + let column = column.name().to_string(); + match self.bound.entry((index, part)) { + Entry::Vacant(slot) => { + self.attributes + .push((name.clone(), Expr::from(VarExpr::new(column.clone())))); + self.bindings.push(Binding { + var: index, + part, + name, + scalar_type, + }); + slot.insert(column); + } + Entry::Occupied(first) => { + // The variable is repeated within this single atom, as in + // `R(x, x)`. That is a local equality condition on this one + // relation rather than a join condition, and the projection + // has to expose the attribute exactly once — two attributes + // of the same name would collide in the projected schema. + self.conditions.push(Expr::from(BinaryExpr { + operator: Operator::Equal, + left: Expr::from(VarExpr::new(first.get().clone())), + right: Expr::from(VarExpr::new(column)), + })); + } + } + } + Ok(()) + } +} + +/// The schema of the relation a rule evaluates to. +/// +/// Its columns are the parts the rule's output binds, in the +/// `(VarIdx, VarPart)` order [`QueryProgramBuilder::conjunctive_query`] reports +/// them in, and their types come from the query columns those parts resolve to +/// rather than from the FLIR variables — a row id's two halves reach the query +/// engine as plain unsigned integers, which the variable's [`ir::ColType`] does +/// not say. +fn rule_output_schema(name: &TableRef, bindings: &[Binding]) -> TableSchema { + TableSchema::new( + name.clone(), + bindings + .iter() + .map(|binding| Column::new(binding.name.clone(), binding.scalar_type)) + .collect(), + // A rule declares no key of its own, and nothing consumes the primary + // keys of a derived relation yet. The row id parts it binds would be the + // candidate once something does. + vec![], + ) +} + +fn friendly_var(vars: &[FriendlyVar], index: ir::VarIdx) -> Result<&FriendlyVar, SyntaxError> { + vars.get(index as usize) + .ok_or_else(|| SyntaxError::new(format!("FLIR var idx {index} out of bounds"))) +} + +/// Derives the join condition of a conjunctive query: one [`JoinVariable`] per +/// variable part that more than one atom binds. +/// +/// A part bound by a single atom is dropped. It is not an equality class, so it +/// is not part of a join condition — it reaches the output through its atom's +/// schema, which is also how it stays available to an enclosing antijoin. +/// +/// Grouping runs through a [`BTreeMap`] keyed on `(VarIdx, VarPart)`, so the +/// resulting order follows the FLIR variable indices instead of a hash order. +/// Plans have to be reproducible for a given input. +fn join_variables(plans: &[AtomPlan]) -> Vec { + let mut occurrences: BTreeMap<(ir::VarIdx, VarPart), Vec<(RelationIdx, String)>> = + BTreeMap::new(); + for (relation, plan) in plans.iter().enumerate() { + for binding in &plan.bindings { + occurrences + .entry((binding.var, binding.part)) + .or_default() + .push((relation, binding.name.clone())); + } + } + occurrences + .into_values() + .filter(|occurrences| occurrences.len() > 1) + .map(|occurrences| JoinVariable { + // Every atom projects a given part onto the same name, so the first + // occurrence's name is the shared output name — and it is the copy + // the join keeps active, since shadowing favours the earlier + // relation. + name: occurrences[0].1.clone(), + occurrences: occurrences + .into_iter() + .map(|(relation, name)| (relation, Expr::from(VarExpr::new(name)))) + .collect(), + }) + .collect() +} + +/// The key an [`AntiJoinExpr`] between the two sides of a rule compares on: +/// every variable part that both sides bind. +/// +/// Note that a part occurring only once *within* a side belongs here all the +/// same. It is not a join variable of that side's conjunctive query, but it is +/// bound by that side's schema, and the antijoin does have to compare on it. +fn antijoin_key(left: &[Binding], right: &[Binding]) -> Vec<(Expr, Expr)> { + let right: BTreeMap<(ir::VarIdx, VarPart), &str> = right + .iter() + .map(|binding| ((binding.var, binding.part), binding.name.as_str())) + .collect(); + left.iter() + .filter_map(|binding| { + right + .get(&(binding.var, binding.part)) + .map(|counterpart| (binding.name.as_str(), *counterpart)) + }) + .map(|(left, right)| { + ( + Expr::from(VarExpr::new(left)), + Expr::from(VarExpr::new(right)), + ) + }) + .collect() +} + +pub trait TryReduce: Iterator> { + /// Reduces to a single item, short-circuiting on the first [`Err`]. + /// + /// Unlike collecting into a `Vec` first, nothing is allocated, and unlike + /// [`Iterator::reduce`] the items may fail. Note that `f` itself is + /// infallible: the fallibility belongs to the items, not to the step that + /// combines two of them. + fn try_reduce(mut self, mut f: impl FnMut(T, T) -> T) -> Result, E> where Self: Sized, - F: FnMut(T, T) -> Result, { - let first = match self.next() { - Some(Ok(v)) => v, - Some(Err(e)) => return Err(e), - None => return Ok(None), + let Some(first) = self.next().transpose()? else { + return Ok(None); }; - - self.try_fold(first, |acc, item| f(acc, item?)).map(Some) + self.try_fold(first, |acc, item| Ok(f(acc, item?))) + .map(Some) } } -impl TryReduceResultExt for I where I: Iterator> {} +impl TryReduce for I where I: Iterator> {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::relational::expr::RelExpr; + + /// A builder with one base table `t` whose columns are given as + /// `(name, type)` pairs, so [`QueryProgramBuilder::atom`] can be driven + /// directly. + fn builder_with_table(columns: Vec<(&str, ir::ColType)>) -> QueryProgramBuilder { + let mut builder = QueryProgramBuilder::new(); + builder + .table_declaration(&table_entry(columns)) + .expect("A single base table declaration must succeed"); + builder + } + + fn table_entry(columns: Vec<(&str, ir::ColType)>) -> ir::TableEntry { + ir::TableEntry { + path: ir::Path::from("t"), + table: ir::Schema { + entity_variant: ir::EntityVariant::Table, + columns: columns + .into_iter() + .map(|(name, col_type)| ir::ColumnEntry { + path: ir::Path::from(name), + col_type, + }) + .collect(), + primary_key: None, + }, + } + } + + fn builtin() -> ir::ColType { + ir::ColType::BuiltinTy { + builtin_ty: ir::BuiltinTy::BuiltinInt, + } + } + + fn atom_over_t(row_id: Option, values: Vec<(ir::ColumnIdx, ir::Term)>) -> ir::Atom { + ir::Atom { + entity: ir::Path::from("t"), + row_id, + values: values + .into_iter() + .map(|(column, term)| ir::ValueEntry { column, term }) + .collect(), + } + } + + fn var_term(index: ir::VarIdx) -> ir::Term { + ir::Term::Var { index } + } + + /// Destructures the `Projection(Selection?(Source))` shape an atom lowers to. + fn projection(expr: &Expr) -> &ProjectionExpr { + match expr { + Expr::Relational(rel) => match rel.as_ref() { + RelExpr::Projection(projection) => projection, + other => panic!("Expected an atom to lower to a projection, got {other:?}"), + }, + other => panic!("Expected a relational expression, got {other:?}"), + } + } + + /// The selection an atom's local conditions produce, if it has any. + fn selection(expr: &Expr) -> Option<&SelectionExpr> { + match expr { + Expr::Relational(rel) => match rel.as_ref() { + RelExpr::Selection(selection) => Some(selection), + _ => None, + }, + _ => None, + } + } + + fn attribute_names(projection: &ProjectionExpr) -> Vec<&str> { + projection + .attributes + .iter() + .map(|(name, _)| name.as_str()) + .collect() + } + + #[test] + fn declaring_the_same_base_table_twice_is_an_error() { + // `HashMap::insert` returns the previous value, so the check's direction + // matters: the first declaration must pass and the second must not. + let mut builder = QueryProgramBuilder::new(); + let entry = table_entry(vec![("a", builtin())]); + builder + .table_declaration(&entry) + .expect("The first declaration of a base table must succeed"); + assert!( + builder.table_declaration(&entry).is_err(), + "A second declaration of the same base table must be rejected" + ); + } + + #[test] + fn an_atom_projects_each_bound_variable_onto_its_column() { + let mut builder = builder_with_table(vec![("a", builtin()), ("b", builtin())]); + let vars = vec![scalar_var("x"), scalar_var("y")]; + let plan = builder + .atom( + &atom_over_t(None, vec![(0, var_term(0)), (1, var_term(1))]), + &vars, + ) + .expect("A well-formed atom lowers"); + + assert_eq!(attribute_names(projection(&plan.relation)), vec!["x", "y"]); + assert_eq!(plan.bindings.len(), 2); + // No local conditions, so no selection between projection and source. + assert!(selection(&projection(&plan.relation).relation).is_none()); + } + + #[test] + fn a_literal_becomes_a_local_condition_rather_than_a_binding() { + let mut builder = builder_with_table(vec![("a", builtin())]); + let plan = builder + .atom( + &atom_over_t( + None, + vec![( + 0, + ir::Term::Lit { + lit: ir::Lit::Int { value: 42 }, + }, + )], + ), + &[], + ) + .expect("An atom comparing a column to a literal lowers"); + + assert!(plan.bindings.is_empty()); + assert!(attribute_names(projection(&plan.relation)).is_empty()); + assert!( + selection(&projection(&plan.relation).relation).is_some(), + "The literal must become a selection beneath the projection" + ); + } + + #[test] + fn a_variable_repeated_within_one_atom_is_bound_once_and_equated() { + // `t(x, x)` must not project two attributes called `x` — they would + // collide in the projected schema. The repetition is a local equality + // condition on this one relation instead, which is also what keeps the + // join's relation indices distinct per variable. + let mut builder = builder_with_table(vec![("a", builtin()), ("b", builtin())]); + let vars = vec![scalar_var("x")]; + let plan = builder + .atom( + &atom_over_t(None, vec![(0, var_term(0)), (1, var_term(0))]), + &vars, + ) + .expect("A repeated variable lowers"); + + assert_eq!(attribute_names(projection(&plan.relation)), vec!["x"]); + assert_eq!(plan.bindings.len(), 1); + + let selection = selection(&projection(&plan.relation).relation) + .expect("The repetition must produce a selection"); + match &selection.condition { + Expr::Binary(binary) => { + assert_eq!(binary.operator, Operator::Equal); + assert_ne!( + binary.left, binary.right, + "The equality must compare the two distinct columns" + ); + } + other => panic!("Expected an equality condition, got {other:?}"), + } + } + + #[test] + fn a_row_id_variable_binds_both_of_its_halves() { + let mut builder = builder_with_table(vec![( + "a", + ir::ColType::RowId { + path: ir::Path::from("other"), + }, + )]); + let vars = vec![row_id_var("x")]; + let plan = builder + .atom(&atom_over_t(None, vec![(0, var_term(0))]), &vars) + .expect("A row id valued column lowers"); + + assert_eq!(plan.bindings.len(), 2); + assert_eq!(attribute_names(projection(&plan.relation)).len(), 2); + assert_eq!( + plan.bindings + .iter() + .map(|binding| binding.part) + .collect::>(), + vec![VarPart::RowIdHash, VarPart::RowIdCtr] + ); + } + + #[test] + fn binding_a_row_id_to_a_scalar_variable_is_an_error() { + let mut builder = builder_with_table(vec![("a", builtin())]); + let vars = vec![scalar_var("x")]; + assert!( + builder + .atom(&atom_over_t(Some(var_term(0)), vec![]), &vars) + .is_err() + ); + } + + /// A rule `antecedents => consequents` over the given variables. + fn rule_entry( + name: &str, + vars: Vec, + antecedents: Vec, + consequents: Vec, + ) -> ir::RuleEntry { + let atoms = |atoms: Vec| { + atoms + .into_iter() + .map(|atom| ir::Prop::Atom { atom }) + .collect() + }; + ir::RuleEntry { + path: ir::Path::from(name), + rule: ir::Rule { + rule_variant: ir::RuleVariant::Enforced, + var_names: vars.iter().map(|var| var.name.clone()).collect(), + var_types: vars.iter().map(|var| var.ty.clone()).collect(), + antecedents: atoms(antecedents), + consequents: atoms(consequents), + }, + } + } + + #[test] + fn a_flat_realm_lowers_into_a_program() { + let realm = FlatRealm { + tables: vec![table_entry(vec![("a", builtin()), ("b", builtin())])], + rules: vec![rule_entry( + "r", + vec![scalar_var("x"), scalar_var("y")], + // t(x, y) and t(x, _) share `x`, so the body is a real join. + vec![ + atom_over_t(None, vec![(0, var_term(0)), (1, var_term(1))]), + atom_over_t(None, vec![(0, var_term(0))]), + ], + vec![atom_over_t(None, vec![(0, var_term(0))])], + )], + }; + + let builder = QueryProgramBuilder::from_flat_realm(&realm).expect("The realm lowers"); + + assert_eq!(builder.program.len(), 1, "One rule is one statement"); + let schema = &builder + .derived_views + .get(&TableRef::from(&ir::Path::from("r"))) + .expect("The rule must be registered under its own name") + .output_schema; + // The antecedents bind `x` and `y`, so both are output columns, with the + // types of the query columns they resolve to. + assert_eq!( + schema + .columns() + .iter() + .map(|column| (column.name(), column.scalar_type())) + .collect::>(), + vec![("x", ScalarType::Iint), ("y", ScalarType::Iint)] + ); + } + + #[test] + fn a_lowered_program_passes_the_resolver() { + // Whatever the lowering emits has to be a well-formed program: every + // variable resolves, and every relational operator's invariants hold. + // This is what actually reaches `MultiWayEquiJoinExpr::validate`. + // + // It stops short of `Pipeline::runtime`, which would go on to build the + // DBSP circuit and hit the backend's `unimplemented!` for multi way + // joins — that needs the fold-into-binary-joins pass. + let realm = FlatRealm { + tables: vec![table_entry(vec![("a", builtin()), ("b", builtin())])], + rules: vec![rule_entry( + "r", + vec![scalar_var("x"), scalar_var("y")], + vec![ + atom_over_t(None, vec![(0, var_term(0)), (1, var_term(1))]), + atom_over_t(None, vec![(0, var_term(0))]), + ], + vec![atom_over_t(None, vec![(0, var_term(0))])], + )], + }; + let builder = QueryProgramBuilder::from_flat_realm(&realm).expect("The realm lowers"); + + crate::host::resolver::ResolvedCode::from(builder.program) + .expect("The lowered program must resolve"); + } + + #[test] + fn declaring_the_same_rule_twice_is_an_error() { + let rule = rule_entry( + "r", + vec![scalar_var("x")], + vec![atom_over_t(None, vec![(0, var_term(0))])], + vec![atom_over_t(None, vec![(0, var_term(0))])], + ); + let realm = FlatRealm { + tables: vec![table_entry(vec![("a", builtin())])], + rules: vec![rule.clone(), rule], + }; + assert!(QueryProgramBuilder::from_flat_realm(&realm).is_err()); + } + + fn multi_way_join(expr: &Expr) -> &MultiWayEquiJoinExpr { + match expr { + Expr::Relational(rel) => match rel.as_ref() { + RelExpr::MultiWayEquiJoin(join) => join, + other => panic!("Expected a multi way equi join, got {other:?}"), + }, + other => panic!("Expected a relational expression, got {other:?}"), + } + } + + fn conjunctive_query(atoms: Vec) -> ConjunctiveQuery { + ConjunctiveQuery { + atoms, + conditions: vec![], + } + } + + #[test] + fn a_single_atom_conjunctive_query_needs_no_join() { + // There is nothing to equate across atoms, and the join operators reject + // fewer than two relations, so the atom must come through as-is. + let mut builder = builder_with_table(vec![("a", builtin())]); + let query = conjunctive_query(vec![atom_over_t(None, vec![(0, var_term(0))])]); + let (expr, bindings) = builder + .conjunctive_query(&query, &[scalar_var("x")]) + .expect("A one-atom conjunctive query lowers"); + + assert_eq!(attribute_names(projection(&expr)), vec!["x"]); + assert_eq!(bindings.len(), 1); + } + + #[test] + fn two_atoms_sharing_a_variable_lower_to_a_join_on_that_variable() { + let mut builder = builder_with_table(vec![("a", builtin()), ("b", builtin())]); + // t(x, y) and t(x, z): `x` is shared, `y` and `z` are not. + let query = conjunctive_query(vec![ + atom_over_t(None, vec![(0, var_term(0)), (1, var_term(1))]), + atom_over_t(None, vec![(0, var_term(0)), (1, var_term(2))]), + ]); + let vars = vec![scalar_var("x"), scalar_var("y"), scalar_var("z")]; + let (expr, bindings) = builder + .conjunctive_query(&query, &vars) + .expect("A two-atom conjunctive query lowers"); + + let join = multi_way_join(&expr); + assert_eq!(join.relations.len(), 2); + assert_eq!( + summary(&join.on), + vec![("x".to_string(), vec![0, 1])], + "Only the shared variable may appear in the join condition" + ); + // `y` and `z` are still bound by the query as a whole, which is what + // keeps them available to an enclosing antijoin. + assert_eq!(bindings.len(), 3); + assert!(join.validate().is_ok()); + } + + #[test] + fn two_atoms_sharing_nothing_lower_to_a_cartesian_product() { + let mut builder = builder_with_table(vec![("a", builtin())]); + let query = conjunctive_query(vec![ + atom_over_t(None, vec![(0, var_term(0))]), + atom_over_t(None, vec![(0, var_term(1))]), + ]); + let vars = vec![scalar_var("x"), scalar_var("y")]; + let (expr, _bindings) = builder + .conjunctive_query(&query, &vars) + .expect("Atoms sharing no variable still lower"); + + assert!( + multi_way_join(&expr).on.is_empty(), + "An empty join condition is how a cartesian product is expressed" + ); + } + + #[test] + fn a_conjunctive_query_without_atoms_is_an_error() { + let mut builder = builder_with_table(vec![("a", builtin())]); + assert!( + builder + .conjunctive_query(&conjunctive_query(vec![]), &[]) + .is_err() + ); + } + + #[test] + fn an_atom_over_an_undeclared_entity_is_an_error() { + let mut builder = builder_with_table(vec![("a", builtin())]); + let atom = ir::Atom { + entity: ir::Path::from("nonexistent"), + row_id: None, + values: vec![], + }; + assert!(builder.atom(&atom, &[]).is_err()); + } + + fn scalar_var(name: &str) -> FriendlyVar { + FriendlyVar { + name: ir::Path::from(name), + ty: ir::ColType::BuiltinTy { + builtin_ty: ir::BuiltinTy::BuiltinInt, + }, + } + } + + fn row_id_var(name: &str) -> FriendlyVar { + FriendlyVar { + name: ir::Path::from(name), + ty: ir::ColType::RowId { + path: ir::Path::from("some_table"), + }, + } + } + + fn binding(var: ir::VarIdx, part: VarPart, name: &str) -> Binding { + Binding { + var, + part, + name: name.to_string(), + // Irrelevant to join-variable and antijoin-key derivation; the + // schema tests below assert on types via `atom` instead. + scalar_type: ScalarType::Null, + } + } + + /// An [`AtomPlan`] whose relation is a stand-in: only the bindings matter to + /// [`join_variables`] and [`antijoin_key`]. + fn plan(bindings: Vec) -> AtomPlan { + AtomPlan { + relation: Expr::from(VarExpr::new("atom")), + bindings, + } + } + + /// The relation indices and output name of each derived join variable. + fn summary(variables: &[JoinVariable]) -> Vec<(String, Vec)> { + variables + .iter() + .map(|variable| { + ( + variable.name.clone(), + variable + .occurrences + .iter() + .map(|(relation, _)| *relation) + .collect(), + ) + }) + .collect() + } + + #[test] + fn a_scalar_variable_flattens_into_one_part() { + let parts: Vec<(VarPart, String)> = scalar_var("x").parts().collect(); + assert_eq!(parts, vec![(VarPart::Scalar, "x".to_string())]); + } + + #[test] + fn a_row_id_variable_flattens_into_a_hash_and_a_counter_part() { + let var = row_id_var("x"); + let parts: Vec = var.parts().map(|(part, _)| part).collect(); + assert_eq!(parts, vec![VarPart::RowIdHash, VarPart::RowIdCtr]); + // The names have to differ, or the projection would collide with itself. + let names: Vec = var.parts().map(|(_, name)| name).collect(); + assert_ne!(names[0], names[1]); + assert!(names.iter().all(|name| name.contains('x'))); + } + + #[test] + fn a_variable_shared_by_two_atoms_becomes_one_join_variable() { + let plans = vec![ + plan(vec![binding(0, VarPart::Scalar, "x")]), + plan(vec![binding(0, VarPart::Scalar, "x")]), + ]; + assert_eq!( + summary(&join_variables(&plans)), + vec![("x".to_string(), vec![0, 1])] + ); + } + + #[test] + fn a_variable_bound_by_a_single_atom_is_not_a_join_variable() { + // It constrains nothing, and it still reaches the output through its + // atom's schema — which is what keeps it available to the antijoin. + let plans = vec![ + plan(vec![ + binding(0, VarPart::Scalar, "x"), + binding(1, VarPart::Scalar, "lonely"), + ]), + plan(vec![binding(0, VarPart::Scalar, "x")]), + ]; + assert_eq!( + summary(&join_variables(&plans)), + vec![("x".to_string(), vec![0, 1])] + ); + } + + #[test] + fn atoms_sharing_no_variable_yield_an_empty_join_condition() { + let plans = vec![ + plan(vec![binding(0, VarPart::Scalar, "x")]), + plan(vec![binding(1, VarPart::Scalar, "y")]), + ]; + assert!(join_variables(&plans).is_empty()); + } + + #[test] + fn a_shared_row_id_variable_yields_one_join_variable_per_half() { + // Equality on a row id is equality on the hash *and* the counter, so the + // two halves are two independent equality classes. + let plans = vec![ + plan(vec![ + binding(0, VarPart::RowIdHash, "xRowIdHash"), + binding(0, VarPart::RowIdCtr, "xRowIdCtr"), + ]), + plan(vec![ + binding(0, VarPart::RowIdHash, "xRowIdHash"), + binding(0, VarPart::RowIdCtr, "xRowIdCtr"), + ]), + ]; + assert_eq!( + summary(&join_variables(&plans)), + vec![ + ("xRowIdHash".to_string(), vec![0, 1]), + ("xRowIdCtr".to_string(), vec![0, 1]), + ] + ); + } + + #[test] + fn a_variable_shared_by_three_atoms_has_three_occurrences() { + let plans = vec![ + plan(vec![binding(0, VarPart::Scalar, "x")]), + plan(vec![binding(0, VarPart::Scalar, "x")]), + plan(vec![binding(0, VarPart::Scalar, "x")]), + ]; + assert_eq!( + summary(&join_variables(&plans)), + vec![("x".to_string(), vec![0, 1, 2])] + ); + } + + #[test] + fn join_variables_are_ordered_by_flir_variable_index() { + // Not by the order the atoms happened to bind them in, so that the same + // input always lowers to the same plan. + let plans = vec![ + plan(vec![ + binding(2, VarPart::Scalar, "c"), + binding(0, VarPart::Scalar, "a"), + ]), + plan(vec![ + binding(0, VarPart::Scalar, "a"), + binding(2, VarPart::Scalar, "c"), + binding(1, VarPart::Scalar, "b"), + ]), + plan(vec![binding(1, VarPart::Scalar, "b")]), + ]; + let names: Vec = join_variables(&plans) + .into_iter() + .map(|variable| variable.name) + .collect(); + assert_eq!(names, vec!["a", "b", "c"]); + } + + #[test] + fn derived_join_variables_satisfy_the_join_operator_invariants() { + // The two halves of this change have to fit: whatever `join_variables` + // derives must be constructible, which is the check that no singleton + // and no repeated relation index slips through. + let plans = vec![ + plan(vec![ + binding(0, VarPart::Scalar, "x"), + binding(1, VarPart::Scalar, "only_here"), + ]), + plan(vec![ + binding(0, VarPart::Scalar, "x"), + binding(2, VarPart::Scalar, "y"), + ]), + plan(vec![binding(2, VarPart::Scalar, "y")]), + ]; + let on = join_variables(&plans); + let relations = plans.into_iter().map(|plan| plan.relation).collect(); + MultiWayEquiJoinExpr::new(relations, on, None) + .expect("Derived join variables must satisfy the operator's invariants"); + } + + #[test] + fn the_antijoin_key_is_the_intersection_of_both_sides() { + let left = vec![ + binding(0, VarPart::Scalar, "shared"), + binding(1, VarPart::Scalar, "left_only"), + ]; + let right = vec![ + binding(0, VarPart::Scalar, "shared"), + binding(2, VarPart::Scalar, "right_only"), + ]; + let key = antijoin_key(&left, &right); + assert_eq!(key.len(), 1); + assert_eq!( + key[0], + ( + Expr::from(VarExpr::new("shared")), + Expr::from(VarExpr::new("shared")) + ) + ); + } + + #[test] + fn the_antijoin_key_matches_parts_rather_than_variables() { + // Both sides bind variable 0, but the hash half only appears on the + // left, so only the counter half may be compared. + let left = vec![ + binding(0, VarPart::RowIdHash, "xRowIdHash"), + binding(0, VarPart::RowIdCtr, "xRowIdCtr"), + ]; + let right = vec![binding(0, VarPart::RowIdCtr, "xRowIdCtr")]; + let key = antijoin_key(&left, &right); + assert_eq!(key.len(), 1); + assert_eq!(key[0].0, Expr::from(VarExpr::new("xRowIdCtr"))); + } + + #[test] + fn disjoint_sides_produce_an_empty_antijoin_key() { + let left = vec![binding(0, VarPart::Scalar, "x")]; + let right = vec![binding(1, VarPart::Scalar, "y")]; + assert!(antijoin_key(&left, &right).is_empty()); + } +} diff --git a/packages/coln-query/src/api/schema.rs b/packages/coln-query/src/api/schema.rs index f1000210..f03b12be 100644 --- a/packages/coln-query/src/api/schema.rs +++ b/packages/coln-query/src/api/schema.rs @@ -49,6 +49,15 @@ impl TableSchema { } } +impl TableSchema { + pub fn name(&self) -> &TableRef { + &self.name + } + pub fn columns(&self) -> &[Column] { + &self.columns + } +} + pub struct Column { /// The column's name. name: String, @@ -56,7 +65,23 @@ pub struct Column { scalar_type: ScalarType, } +impl Column { + pub fn new>(name: T, scalar_type: ScalarType) -> Self { + Self { + name: name.into(), + scalar_type, + } + } + pub fn name(&self) -> &str { + &self.name + } + pub fn scalar_type(&self) -> ScalarType { + self.scalar_type + } +} + use coln_flir_rs::ir::{self}; +use coln_flir_rs::schema::{NativeScalarType, QueryEngineScalarType}; impl From<&ir::Lit> for Literal { fn from(value: &ir::Lit) -> Self { @@ -66,3 +91,24 @@ impl From<&ir::Lit> for Literal { } } } + +impl From for ScalarType { + fn from(value: NativeScalarType) -> Self { + match value { + NativeScalarType::Iint => ScalarType::Iint, + NativeScalarType::Uint => ScalarType::Uint, + NativeScalarType::String => ScalarType::String, + } + } +} + +impl From for ScalarType { + fn from(value: QueryEngineScalarType) -> Self { + match value { + // A row id's two halves reach the query engine as plain unsigned + // integers, so every query-engine type is a native one by this + // point. + QueryEngineScalarType::Native(native) => ScalarType::from(native), + } + } +} diff --git a/packages/coln-query/src/host/resolver.rs b/packages/coln-query/src/host/resolver.rs index 422a6994..c19a5ee8 100644 --- a/packages/coln-query/src/host/resolver.rs +++ b/packages/coln-query/src/host/resolver.rs @@ -15,7 +15,7 @@ use crate::{ }, relational::expr::{ AliasExpr, AntiJoinExpr, CartesianProductExpr, DifferenceExpr, DistinctExpr, EquiJoinExpr, - FixedPointIterExpr, MultiWayEquiJoin, OutputExpr, ProjectionExpr, RelExpr, + FixedPointIterExpr, MultiWayEquiJoinExpr, OutputExpr, ProjectionExpr, RelExpr, RelExprVisitorMut, SelectionExpr, SourceExpr, UnionExpr, }, util::{Named, Resolvable}, @@ -390,22 +390,20 @@ impl RelExprVisitorMut> for Resolver { fn visit_multi_way_equi_join_expr( &mut self, - expr: &mut MultiWayEquiJoin, + expr: &mut MultiWayEquiJoinExpr, ctx: VisitorCtx<'_, '_>, ) -> VisitorResult { - // TODO: Typecheck: A union is valid if the column types match and - // the amount of columns is the same. - if expr.relations.len() < 2 { - return Err(SyntaxError::new( - "A multi way equi join requires at least two relations", - )); - } + // The structural invariants (arity, in-bounds and distinct relation + // indices, at least two occurrences per join variable) are checked here + // rather than re-derived by every consumer, because the fields are + // public and a plan may be assembled or rewritten by hand. + expr.validate()?; expr.relations .iter_mut() .try_for_each(|relation| self.visit_expr(relation, ctx)) .and_then(|()| { - expr.on.iter_mut().flatten().flatten().try_for_each(|expr| { + expr.on_exprs_mut().try_for_each(|expr| { ctx.begin_tuple_context(); let ret = self.visit_expr(expr, ctx); ctx.end_tuple_context(); diff --git a/packages/coln-query/src/relational/expr.rs b/packages/coln-query/src/relational/expr.rs index be00f0a7..850bef7e 100644 --- a/packages/coln-query/src/relational/expr.rs +++ b/packages/coln-query/src/relational/expr.rs @@ -11,10 +11,12 @@ //! preserves relation-valued variables, nested operators, and tuple-of-relations. use crate::{ + error::SyntaxError, host::{expr::Expr, stmt::BlockStmt}, relational::RelationSchema, util::MemAddr, }; +use std::collections::HashSet; /// Relational-algebra operator = backend-neutral query-plan vocabulary. /// @@ -38,7 +40,7 @@ pub enum RelExpr { Projection(Box), CartesianProduct(Box), EquiJoin(Box), - MultiWayEquiJoin(Box), + MultiWayEquiJoin(Box), AntiJoin(Box), FixedPointIter(Box), } @@ -75,7 +77,7 @@ impl_rel_and_expr_from! { (RelExpr::Projection, ProjectionExpr), (RelExpr::CartesianProduct, CartesianProductExpr), (RelExpr::EquiJoin, EquiJoinExpr), - (RelExpr::MultiWayEquiJoin, MultiWayEquiJoin), + (RelExpr::MultiWayEquiJoin, MultiWayEquiJoinExpr), (RelExpr::AntiJoin, AntiJoinExpr), (RelExpr::FixedPointIter, FixedPointIterExpr), } @@ -265,30 +267,160 @@ pub struct EquiJoinExpr { pub attributes: Option>, } -/// An equijoin involving `N` relations. A better input than a folded sequence -/// of [binary `EquiJoin`s](EquiJoinExpr) for worst-case optimal join algorithms -/// (such as the leapfrog triejoin). +/// The position of a relation within [`MultiWayEquiJoinExpr::relations`]. +pub type RelationIdx = usize; + +/// One equality class of a [`MultiWayEquiJoinExpr`]: every listed occurrence +/// must produce the same value for a tuple to enter the join's output. +/// +/// A variable bound by only *one* relation is deliberately not representable +/// here — it constrains nothing, so it is not part of a join condition. Such a +/// variable still reaches the output, carried by its relation's schema like any +/// other non-join attribute. Keeping them out is what makes +/// [`MultiWayEquiJoinExpr::on`]`.is_empty()` an exact test for "nothing to join +/// on". +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct JoinVariable { + /// The name the joined attribute carries in the output schema. + /// + /// The lowering from coln's FLIR projects every atom onto the names of the + /// variables it binds, so there the occurrences are plain column picks that + /// already agree on this name, and the schema fold described on + /// [`MultiWayEquiJoinExpr::on`] keeps exactly one active copy of it. When + /// the occurrences do *not* agree on a name (`l.a = r.b`), producing this + /// name is the job of whoever lowers the join. + pub name: String, + /// Which relations bind this variable, and how: the [`RelationIdx`] indexes + /// into [`MultiWayEquiJoinExpr::relations`], and the [`Expr`] is evaluated + /// in the context of that relation. + /// + /// Invariants, enforced by [`MultiWayEquiJoinExpr::new`]: at least two + /// occurrences, every index in bounds, indices pairwise distinct, and + /// ordered by index. + pub occurrences: Vec<(RelationIdx, Expr)>, +} + +/// An equijoin involving `N >= 2` relations. A better input than a folded +/// sequence of [binary `EquiJoin`s](EquiJoinExpr) for worst-case optimal join +/// algorithms (such as the leapfrog triejoin), which are variable-oriented: +/// they iterate a variable ordering, which is what [`on`](Self::on) spells out. #[derive(Clone, Debug, Eq, PartialEq)] -pub struct MultiWayEquiJoin { - /// The `N` relations which participate in the join. Each [`Expr`] must +pub struct MultiWayEquiJoinExpr { + /// The `N >= 2` relations which participate in the join. Each [`Expr`] must /// evaluate to a relation. pub relations: Vec, - /// Each entry in the outer vector corresponds to a variable which must be - /// equal among all its occurrences. The inner vector vector tracks the - /// occurrences for each variable. The inner vector is _guaranteed_ to have - /// the same arity as the [`relations`](Self::relations) vector. An entry - /// at index `i` in the inner vector with value `None` indicates that the - /// corresponding relation ([`relations[i]`](Self::relations)) does _not_ - /// bind the variable, whereas a value of [`Some(Expr)`](Expr) binds the - /// variable to the value of the `Expr` evaluated in the context of the - /// corresponding relation (which is again [`relations[i]`](Self::relations)). + /// The join condition, as one [`JoinVariable`] per equality class of + /// attributes that have to agree. /// - /// If `on` is empty, a [`CartesianProduct`](CartesianProductExpr) is computed. - pub on: Vec>>, + /// If `on` is empty, a [`CartesianProduct`](CartesianProductExpr) over + /// [`relations`](Self::relations) is computed. Since a variable bound by a + /// single relation cannot be a [`JoinVariable`], that test is exact rather + /// than approximate. + /// + /// **Output schema.** Joining folds + /// [`RelationSchema::join`](crate::relational::RelationSchema::join) left to + /// right, which deactivates an attribute of a later relation when an earlier + /// one already contributes an active attribute of the same name. A join + /// variable whose occurrences agree on their name therefore appears **once** + /// in the output, carried by the first relation that binds it — no + /// de-duplicating projection is required, and no join column is silently + /// duplicated. + pub on: Vec, /// An optional projection step. See documentation of [`ProjectionExpr`]. pub attributes: Option>, } +impl MultiWayEquiJoinExpr { + /// The only constructor that cannot produce a malformed join: it normalizes + /// each [`JoinVariable`]'s occurrences into relation order and then applies + /// [`validate`](Self::validate). + pub fn new( + relations: Vec, + on: Vec, + attributes: Option>, + ) -> Result { + let mut joined = Self { + relations, + on, + attributes, + }; + for variable in &mut joined.on { + variable.occurrences.sort_by_key(|(relation, _)| *relation); + } + joined.validate()?; + Ok(joined) + } + + /// Checks the invariants documented on [`Self::relations`] and + /// [`JoinVariable::occurrences`]. [`Self::new`] applies this to everything + /// it builds; the resolver re-applies it because the fields are public and + /// a plan may also be assembled or rewritten by hand. + pub fn validate(&self) -> Result<(), SyntaxError> { + if self.relations.len() < 2 { + return Err(SyntaxError::new(format!( + "A multi way equi join requires at least two relations, got {}", + self.relations.len() + ))); + } + let mut names = HashSet::with_capacity(self.on.len()); + for variable in &self.on { + if !names.insert(&variable.name) { + return Err(SyntaxError::new(format!( + "Join variable '{}' is declared twice", + variable.name + ))); + } + if variable.occurrences.len() < 2 { + return Err(SyntaxError::new(format!( + "Join variable '{}' has {} occurrence(s): below two it constrains \ + nothing, and a variable bound by a single relation reaches the \ + output through that relation's schema instead", + variable.name, + variable.occurrences.len() + ))); + } + let mut relations = HashSet::with_capacity(variable.occurrences.len()); + for (relation, _) in &variable.occurrences { + if *relation >= self.relations.len() { + return Err(SyntaxError::new(format!( + "Join variable '{}' refers to relation {relation} but the join \ + has only {} relations", + variable.name, + self.relations.len() + ))); + } + if !relations.insert(relation) { + return Err(SyntaxError::new(format!( + "Join variable '{}' occurs twice in relation {relation}: a \ + variable repeated within one relation is a local equality \ + condition on that relation, not a join condition", + variable.name + ))); + } + } + } + Ok(()) + } + + /// Every [`Expr`] nested in the join condition, in [`on`](Self::on) order. + /// Each one is evaluated in the context of *its own* relation, so a consumer + /// that needs to know which relation must iterate [`on`](Self::on) directly. + pub fn on_exprs(&self) -> impl Iterator { + self.on + .iter() + .flat_map(|variable| variable.occurrences.iter().map(|(_, expr)| expr)) + } + + /// The [`on_exprs`](Self::on_exprs) counterpart for rewriting passes. + /// Handing out `&mut Expr` cannot break any invariant, as those constrain + /// the arity and the relation indices rather than the expressions. + pub fn on_exprs_mut(&mut self) -> impl Iterator { + self.on + .iter_mut() + .flat_map(|variable| variable.occurrences.iter_mut().map(|(_, expr)| expr)) + } +} + /// This is not a commutative operation, that is, swapping the `left` and `right` /// relations may alter the result. This computes `left` setminus `right` while /// only considering the columns specified in `on`. @@ -298,9 +430,16 @@ pub struct AntiJoinExpr { pub left: Expr, /// Must evaluate to a relation. pub right: Expr, - /// The attributes to _not_ join on. The first element of any pair belongs to the - /// left relation, and the second element of any pair belongs to right relation. - /// Each attribute pair should produce the same type. + /// The attributes the two relations are compared on: a `left` row is + /// suppressed exactly when some `right` row agrees with it on all of them. + /// The first element of any pair is evaluated in the context of the left + /// relation, the second in the context of the right one, and each pair + /// should produce the same type. + /// + /// Note that this is the key to match *on*, in the same sense as + /// [`EquiJoinExpr::on`] — the columns that survive into the output are not + /// expressed here at all, since the output carries the left relation's + /// schema unchanged. pub on: Vec<(Expr, Expr)>, } @@ -381,7 +520,7 @@ pub trait RelExprVisitor { fn visit_projection_expr(&mut self, expr: &ProjectionExpr, ctx: C) -> T; fn visit_cartesian_product_expr(&mut self, expr: &CartesianProductExpr, ctx: C) -> T; fn visit_equi_join_expr(&mut self, expr: &EquiJoinExpr, ctx: C) -> T; - fn visit_multi_way_equi_join_expr(&mut self, expr: &MultiWayEquiJoin, ctx: C) -> T; + fn visit_multi_way_equi_join_expr(&mut self, expr: &MultiWayEquiJoinExpr, ctx: C) -> T; fn visit_anti_join_expr(&mut self, expr: &AntiJoinExpr, ctx: C) -> T; fn visit_fixed_point_iter_expr(&mut self, expr: &FixedPointIterExpr, ctx: C) -> T; } @@ -414,7 +553,7 @@ pub trait RelExprVisitorMut { fn visit_projection_expr(&mut self, expr: &mut ProjectionExpr, ctx: C) -> T; fn visit_cartesian_product_expr(&mut self, expr: &mut CartesianProductExpr, ctx: C) -> T; fn visit_equi_join_expr(&mut self, expr: &mut EquiJoinExpr, ctx: C) -> T; - fn visit_multi_way_equi_join_expr(&mut self, expr: &mut MultiWayEquiJoin, ctx: C) -> T; + fn visit_multi_way_equi_join_expr(&mut self, expr: &mut MultiWayEquiJoinExpr, ctx: C) -> T; fn visit_anti_join_expr(&mut self, expr: &mut AntiJoinExpr, ctx: C) -> T; fn visit_fixed_point_iter_expr(&mut self, expr: &mut FixedPointIterExpr, ctx: C) -> T; } @@ -447,7 +586,7 @@ pub trait RelExprVisitorOwn { fn visit_projection_expr(&mut self, expr: ProjectionExpr, ctx: C) -> T; fn visit_cartesian_product_expr(&mut self, expr: CartesianProductExpr, ctx: C) -> T; fn visit_equi_join_expr(&mut self, expr: EquiJoinExpr, ctx: C) -> T; - fn visit_multi_way_equi_join_expr(&mut self, expr: MultiWayEquiJoin, ctx: C) -> T; + fn visit_multi_way_equi_join_expr(&mut self, expr: MultiWayEquiJoinExpr, ctx: C) -> T; fn visit_anti_join_expr(&mut self, expr: AntiJoinExpr, ctx: C) -> T; fn visit_fixed_point_iter_expr(&mut self, expr: FixedPointIterExpr, ctx: C) -> T; } @@ -463,7 +602,125 @@ impl MemAddr for SelectionExpr {} impl MemAddr for ProjectionExpr {} impl MemAddr for CartesianProductExpr {} impl MemAddr for EquiJoinExpr {} -impl MemAddr for MultiWayEquiJoin {} +impl MemAddr for MultiWayEquiJoinExpr {} impl MemAddr for AntiJoinExpr {} impl MemAddr for ThetaJoinExpr {} impl MemAddr for FixedPointIterExpr {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::host::expr::VarExpr; + + /// A stand-in relation operand. [`MultiWayEquiJoinExpr::validate`] only ever + /// counts these, so their content is irrelevant. + fn relations(count: usize) -> Vec { + (0..count) + .map(|idx| Expr::from(VarExpr::new(format!("r{idx}")))) + .collect() + } + + fn join_variable(name: &str, occurrences: &[RelationIdx]) -> JoinVariable { + JoinVariable { + name: name.to_string(), + occurrences: occurrences + .iter() + .map(|relation| (*relation, Expr::from(VarExpr::new(name)))) + .collect(), + } + } + + #[test] + fn accepts_a_join_variable_shared_by_two_relations() { + let joined = + MultiWayEquiJoinExpr::new(relations(2), vec![join_variable("x", &[0, 1])], None) + .expect("A variable bound by two relations is a join variable"); + assert_eq!(joined.on.len(), 1); + assert_eq!(joined.on_exprs().count(), 2); + } + + #[test] + fn accepts_an_empty_join_condition_as_a_cartesian_product() { + let joined = MultiWayEquiJoinExpr::new(relations(3), vec![], None) + .expect("An empty join condition is a cartesian product, not an error"); + assert!(joined.on.is_empty()); + } + + #[test] + fn rejects_fewer_than_two_relations() { + for count in 0..2 { + assert!( + MultiWayEquiJoinExpr::new(relations(count), vec![], None).is_err(), + "A join over {count} relation(s) should be rejected" + ); + } + } + + #[test] + fn rejects_a_single_occurrence_because_it_constrains_nothing() { + // The whole point of the `on` representation: a variable bound by only + // one relation is not an equality class. It reaches the output through + // that relation's schema instead, which is why rejecting it here is safe + // and keeps `on.is_empty()` an exact cartesian-product test. + let error = MultiWayEquiJoinExpr::new(relations(2), vec![join_variable("x", &[0])], None) + .expect_err("A single occurrence must not be representable"); + assert!(error.to_string().contains("occurrence")); + } + + #[test] + fn rejects_an_out_of_bounds_relation_index() { + assert!( + MultiWayEquiJoinExpr::new(relations(2), vec![join_variable("x", &[0, 2])], None) + .is_err() + ); + } + + #[test] + fn rejects_a_variable_occurring_twice_in_one_relation() { + // Such a repetition is a local equality condition on that one relation, + // so it belongs in a `SelectionExpr` beneath the join. + assert!( + MultiWayEquiJoinExpr::new(relations(2), vec![join_variable("x", &[0, 0])], None) + .is_err() + ); + } + + #[test] + fn rejects_two_join_variables_claiming_the_same_output_name() { + assert!( + MultiWayEquiJoinExpr::new( + relations(3), + vec![join_variable("x", &[0, 1]), join_variable("x", &[1, 2])], + None + ) + .is_err() + ); + } + + #[test] + fn normalizes_occurrences_into_relation_order() { + // Plans have to be reproducible: the occurrence order must not depend on + // the order the producer happened to discover the occurrences in. + let joined = + MultiWayEquiJoinExpr::new(relations(3), vec![join_variable("x", &[2, 0, 1])], None) + .expect("Out-of-order occurrences are normalized, not rejected"); + let order: Vec = joined.on[0] + .occurrences + .iter() + .map(|(relation, _)| *relation) + .collect(); + assert_eq!(order, vec![0, 1, 2]); + } + + #[test] + fn validate_agrees_with_new_on_hand_assembled_joins() { + // The fields are public, so a hand-built or rewritten plan can violate + // the invariants; the resolver relies on `validate` catching that. + let malformed = MultiWayEquiJoinExpr { + relations: relations(2), + on: vec![join_variable("x", &[0])], + attributes: None, + }; + assert!(malformed.validate().is_err()); + } +} diff --git a/packages/coln-query/src/relational/incremental/interpreter.rs b/packages/coln-query/src/relational/incremental/interpreter.rs index 1563f423..8d859b4c 100644 --- a/packages/coln-query/src/relational/incremental/interpreter.rs +++ b/packages/coln-query/src/relational/incremental/interpreter.rs @@ -11,7 +11,7 @@ use super::operators::{ reindex::reindex_helper, }; use crate::relational::RelationSchema; -use crate::relational::expr::MultiWayEquiJoin; +use crate::relational::expr::MultiWayEquiJoinExpr; use crate::relational::incremental::dbsp::{ DbspInput, OrdIndexedStreamInputHandle, new_ord_indexed_stream, }; @@ -109,11 +109,7 @@ fn collect_source_exprs<'a>(stmts: &'a [Stmt], out: &mut Vec<&'a SourceExpr>) { RelExpr::EquiJoin(expr) => walk_equi_join(expr, out), RelExpr::MultiWayEquiJoin(expr) => { expr.relations.iter().for_each(|rel| walk_expr(rel, out)); - expr.on - .iter() - .flatten() - .flatten() - .for_each(|expr| walk_expr(expr, out)); + expr.on_exprs().for_each(|expr| walk_expr(expr, out)); expr.attributes .iter() .flatten() @@ -604,7 +600,7 @@ impl RelExprVisitor> fn visit_multi_way_equi_join_expr( &mut self, - expr: &MultiWayEquiJoin, + expr: &MultiWayEquiJoinExpr, ctx: VisitorCtx<'_, '_>, ) -> ExprVisitorResult { unimplemented!( diff --git a/packages/coln-query/src/typing/type_resolver.rs b/packages/coln-query/src/typing/type_resolver.rs index 141e144d..9dcd2923 100644 --- a/packages/coln-query/src/typing/type_resolver.rs +++ b/packages/coln-query/src/typing/type_resolver.rs @@ -16,8 +16,8 @@ use crate::{ }, relational::expr::{ AliasExpr, AntiJoinExpr, CartesianProductExpr, DifferenceExpr, DistinctExpr, EquiJoinExpr, - FixedPointIterExpr, MultiWayEquiJoin, OutputExpr, ProjectionExpr, RelExpr, RelExprVisitor, - SelectionExpr, SourceExpr, UnionExpr, + FixedPointIterExpr, MultiWayEquiJoinExpr, OutputExpr, ProjectionExpr, RelExpr, + RelExprVisitor, SelectionExpr, SourceExpr, UnionExpr, }, }; pub use crate::{ @@ -351,7 +351,7 @@ impl RelExprVisitor> for TypeResolver { fn visit_multi_way_equi_join_expr( &mut self, - expr: &MultiWayEquiJoin, + expr: &MultiWayEquiJoinExpr, ctx: VisitorCtx<'_, '_>, ) -> VisitorResult { let mut relations = expr.relations.iter(); From aff0a5fa0619af31d0068d38f27bad6a91272deb Mon Sep 17 00:00:00 2001 From: Leo Stewen Date: Thu, 20 Aug 2026 19:10:11 +0200 Subject: [PATCH 11/58] Backup --- packages/coln-query/src/api/query.rs | 27 +- packages/coln-query/src/host/mod.rs | 4 + packages/coln-query/src/host/operator.rs | 52 + packages/coln-query/src/host/print.rs | 1035 +++++++++++++++++ packages/coln-query/src/host/walk.rs | 730 ++++++++++++ .../src/relational/incremental/interpreter.rs | 118 +- 6 files changed, 1842 insertions(+), 124 deletions(-) create mode 100644 packages/coln-query/src/host/print.rs create mode 100644 packages/coln-query/src/host/walk.rs diff --git a/packages/coln-query/src/api/query.rs b/packages/coln-query/src/api/query.rs index 03a9ac2d..35537660 100644 --- a/packages/coln-query/src/api/query.rs +++ b/packages/coln-query/src/api/query.rs @@ -28,8 +28,10 @@ use std::collections::{BTreeMap, HashMap}; type BaseTableName = TableRef; type DerivedViewName = TableRef; -struct QueryProgramBuilder { +struct QueryProgram { + /// The (raw, that is, unresolved, unoptimized) query program itself. program: Code, + /// The declared base tables. base_tables: HashMap, /// The relations the program itself defines, that is, one per declared rule. /// @@ -72,16 +74,19 @@ impl From<&BaseTableSchema> for RelationSchema { } } -impl QueryProgramBuilder { - fn new() -> Self { +impl QueryProgram { + fn empty() -> Self { Self { program: Vec::new(), base_tables: HashMap::new(), derived_views: HashMap::new(), } } + pub fn program(&self) -> &Code { + &self.program + } pub fn from_flat_realm(flat_realm: &FlatRealm) -> Result { - let mut builder = QueryProgramBuilder::new(); + let mut builder = QueryProgram::empty(); for table in &flat_realm.tables { builder.table_declaration(table)?; } @@ -772,8 +777,8 @@ mod tests { /// A builder with one base table `t` whose columns are given as /// `(name, type)` pairs, so [`QueryProgramBuilder::atom`] can be driven /// directly. - fn builder_with_table(columns: Vec<(&str, ir::ColType)>) -> QueryProgramBuilder { - let mut builder = QueryProgramBuilder::new(); + fn builder_with_table(columns: Vec<(&str, ir::ColType)>) -> QueryProgram { + let mut builder = QueryProgram::empty(); builder .table_declaration(&table_entry(columns)) .expect("A single base table declaration must succeed"); @@ -852,7 +857,7 @@ mod tests { fn declaring_the_same_base_table_twice_is_an_error() { // `HashMap::insert` returns the previous value, so the check's direction // matters: the first declaration must pass and the second must not. - let mut builder = QueryProgramBuilder::new(); + let mut builder = QueryProgram::empty(); let entry = table_entry(vec![("a", builtin())]); builder .table_declaration(&entry) @@ -1014,9 +1019,9 @@ mod tests { )], }; - let builder = QueryProgramBuilder::from_flat_realm(&realm).expect("The realm lowers"); + let builder = QueryProgram::from_flat_realm(&realm).expect("The realm lowers"); - assert_eq!(builder.program.len(), 1, "One rule is one statement"); + assert_eq!(builder.program().len(), 1, "One rule is one statement"); let schema = &builder .derived_views .get(&TableRef::from(&ir::Path::from("r"))) @@ -1055,7 +1060,7 @@ mod tests { vec![atom_over_t(None, vec![(0, var_term(0))])], )], }; - let builder = QueryProgramBuilder::from_flat_realm(&realm).expect("The realm lowers"); + let builder = QueryProgram::from_flat_realm(&realm).expect("The realm lowers"); crate::host::resolver::ResolvedCode::from(builder.program) .expect("The lowered program must resolve"); @@ -1073,7 +1078,7 @@ mod tests { tables: vec![table_entry(vec![("a", builtin())])], rules: vec![rule.clone(), rule], }; - assert!(QueryProgramBuilder::from_flat_realm(&realm).is_err()); + assert!(QueryProgram::from_flat_realm(&realm).is_err()); } fn multi_way_join(expr: &Expr) -> &MultiWayEquiJoinExpr { diff --git a/packages/coln-query/src/host/mod.rs b/packages/coln-query/src/host/mod.rs index 6bb7dc5e..cbea2430 100644 --- a/packages/coln-query/src/host/mod.rs +++ b/packages/coln-query/src/host/mod.rs @@ -11,11 +11,15 @@ pub mod expr; pub mod function; pub mod interpreter; pub mod operator; +pub mod print; pub mod resolver; pub mod stmt; pub mod tuple; pub mod variable; +pub mod walk; pub use interpreter::{HostInterpreter, InterpreterContext, ScalarHost}; +pub use print::{AsSource, AsTree}; +pub use walk::{Node, post_order, pre_order, walk}; pub type Code = Vec; diff --git a/packages/coln-query/src/host/operator.rs b/packages/coln-query/src/host/operator.rs index e99798e4..e7f8dce6 100644 --- a/packages/coln-query/src/host/operator.rs +++ b/packages/coln-query/src/host/operator.rs @@ -36,3 +36,55 @@ pub enum Operator { /// Eagerly-evaluated binary operation. Division, } + +/// The binding power of a prefix operator ([`Operator::Not`], +/// [`Operator::Subtraction`] applied to one operand). Above every binary +/// operator, so `-a + b` needs no parentheses. +pub const UNARY_PRECEDENCE: u8 = 7; + +/// The binding power of a postfix form (a call, an index) and of any atom that +/// can never need parentheses. +pub const PRIMARY_PRECEDENCE: u8 = 8; + +impl Operator { + /// This operator's binding power as a *binary* operator: higher binds + /// tighter. Only a printer needs it — evaluation order is already fixed by + /// the tree's shape — but a tree built by hand (or by a lowering) carries no + /// [`GroupingExpr`](super::expr::GroupingExpr), so rendering it back to + /// readable text has to re-derive where parentheses belong. + pub fn precedence(self) -> u8 { + match self { + Operator::Or => 1, + Operator::And => 2, + Operator::Equal | Operator::NotEqual => 3, + Operator::Less | Operator::LessEqual | Operator::Greater | Operator::GreaterEqual => 4, + Operator::Addition | Operator::Subtraction => 5, + Operator::Multiplication | Operator::Division => 6, + Operator::Not => UNARY_PRECEDENCE, + } + } + + pub fn symbol(self) -> &'static str { + match self { + Operator::Equal => "==", + Operator::NotEqual => "!=", + Operator::Less => "<", + Operator::LessEqual => "<=", + Operator::Greater => ">", + Operator::GreaterEqual => ">=", + Operator::And => "&&", + Operator::Or => "||", + Operator::Not => "!", + Operator::Addition => "+", + Operator::Subtraction => "-", + Operator::Multiplication => "*", + Operator::Division => "/", + } + } +} + +impl std::fmt::Display for Operator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.symbol()) + } +} diff --git a/packages/coln-query/src/host/print.rs b/packages/coln-query/src/host/print.rs new file mode 100644 index 00000000..7435adbb --- /dev/null +++ b/packages/coln-query/src/host/print.rs @@ -0,0 +1,1035 @@ +// SPDX-FileCopyrightText: 2026 Coln contributors +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Rendering a [`Code`](super::Code) program back into text, in two flavors. +//! +//! - [`AsSource`] reads like a program: infix host expressions, relational +//! operators as named calls. What you want in order to *review* a plan, e.g. +//! to check what a `coln-flir` lowering produced. +//! - [`AsTree`] reads like the data structure: one line per node, tagged with +//! the role it plays in its parent, plus the payloads [`AsSource`] leaves out +//! (schemas, resolved variable slots). What you want in order to *debug* which +//! nodes a pass actually built. +//! +//! The split mirrors the one in [`walk`](mod@super::walk): the tree rendering is a +//! scan and rides on [`Walk`](super::walk::Walk), while the source rendering is +//! a fold — parenthesization flows *down* from the enclosing operator and text +//! is assembled *up* from the operands — so it is a visitor, like the +//! interpreter and the type resolver. A flat node stream cannot express it. + +use super::{ + expr::{ + AssignExpr, BinaryExpr, CallExpr, Expr, ExprVisitor, FunctionExpr, GetIndexExpr, + GroupingExpr, Literal, LiteralExpr, TupleExpr, UnaryExpr, VarExpr, + }, + operator::{PRIMARY_PRECEDENCE, UNARY_PRECEDENCE}, + stmt::{BlockStmt, ExprStmt, Stmt, StmtVisitor, VarStmt}, + walk::{Child, Event, Node, ROOT, walk}, +}; +use crate::relational::expr::{ + AliasExpr, AntiJoinExpr, CartesianProductExpr, DifferenceExpr, DistinctExpr, EquiJoinExpr, + FixedPointIterExpr, JoinVariable, MultiWayEquiJoinExpr, OutputExpr, OutputKind, ProjectionExpr, + RelExpr, RelExprVisitor, SelectionExpr, SourceExpr, UnionExpr, +}; +use std::fmt::{self, Display, Write}; + +/// One level of indentation. +const INDENT: &str = " "; + +/// The width a rendered line aims to stay within. +const MAX_WIDTH: usize = 80; + +/// The precedence of an expression in a position where nothing can need +/// parentheses (a statement, an argument, inside brackets). +const OUTERMOST: u8 = 0; + +/// Writing into a [`String`] cannot fail, which is why every method of the +/// printers below returns `()` instead of a [`fmt::Result`]. +macro_rules! emit { + ($printer:ident, $($arg:tt)*) => { + write!($printer.out, $($arg)*).expect("writing into a String cannot fail") + }; +} + +/// `code` rendered as host-language source text. See [`AsSource`]. +pub fn to_source(code: &[Stmt]) -> String { + AsSource(code).to_string() +} + +/// `code` rendered as an indented node tree. See [`AsTree`]. +pub fn to_tree(code: &[Stmt]) -> String { + AsTree(code).to_string() +} + +/// A program rendered as source text, e.g. `println!("{}", AsSource(&code))`. +/// +/// The rendering is meant to be *read*, not parsed back: there is no parser for +/// the host language, so nothing round-trips and no attempt is made to keep the +/// output unambiguous under re-parsing. It does keep the output *faithful*, +/// which for a tree built programmatically (as every `coln-flir` lowering builds +/// it) means re-deriving parentheses from operator precedence, since such a tree +/// carries no [`GroupingExpr`]. +pub struct AsSource<'a>(pub &'a [Stmt]); + +impl Display for AsSource<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut printer = SourcePrinter { + out: String::new(), + indent: 0, + }; + printer.stmts(self.0); + f.write_str(&printer.out) + } +} + +struct SourcePrinter { + out: String, + /// Indentation of the line currently being written, in levels. + indent: usize, +} + +impl SourcePrinter { + /// The statements of one block, one per line, without a trailing newline. + /// Every `visit_*_stmt` starts at the cursor the caller left and ends + /// without a newline, so nesting composes. + fn stmts(&mut self, stmts: &[Stmt]) { + for (index, stmt) in stmts.iter().enumerate() { + if index > 0 { + self.newline(); + } + self.visit_stmt(stmt, ()); + } + } + + /// `{ … }` with the body one level deeper. Shared by blocks, function bodies + /// and fixed-point steps. + fn block(&mut self, stmts: &[Stmt]) { + if stmts.is_empty() { + self.out.push_str("{}"); + return; + } + self.out.push('{'); + self.indent += 1; + self.newline(); + self.stmts(stmts); + self.indent -= 1; + self.newline(); + self.out.push('}'); + } + + fn newline(&mut self) { + self.out.push('\n'); + for _ in 0..self.indent { + self.out.push_str(INDENT); + } + } + + /// Render whatever `render` writes into a detached buffer, so a caller can + /// lay the pieces out only once they exist — which is how an operator + /// decides between one line and one argument per line. + fn captured(&mut self, render: impl FnOnce(&mut Self)) -> String { + let enclosing = std::mem::take(&mut self.out); + render(self); + std::mem::replace(&mut self.out, enclosing) + } + + /// Wrap what `render` writes in parentheses if `own` binds looser than the + /// enclosing position `ctx` demands. + fn parenthesized(&mut self, own: u8, ctx: u8, render: impl FnOnce(&mut Self)) { + let parens = own < ctx; + if parens { + self.out.push('('); + } + render(self); + if parens { + self.out.push(')'); + } + } + + /// Emit `name(…)`: on one line if that fits within [`MAX_WIDTH`], otherwise + /// one argument per line. + /// + /// The arguments are rendered *before* the decision, so it is made on their + /// real width rather than on a structural guess. This is a greedy layout + /// with no backtracking: enough to keep an atom lowering + /// (`project(select(source("edge"), …), …)`) on one line while a plan that + /// genuinely does not fit still nests readably, and far short of a real + /// layout algorithm. + fn operator(&mut self, name: &str, arguments: impl FnOnce(&mut Self) -> Vec) { + // Rendered one level deeper, so the newlines *inside* a multi-line + // argument are already indented for the broken layout below. A single- + // line argument holds no newline, so the deeper level cannot leak into + // the compact layout. + self.indent += 1; + let arguments = arguments(self); + self.indent -= 1; + let compact = arguments.join(", "); + if !compact.contains('\n') && self.start() + name.len() + compact.len() + 2 <= MAX_WIDTH { + emit!(self, "{name}({compact})"); + return; + } + emit!(self, "{name}("); + self.indent += 1; + for argument in arguments { + self.newline(); + self.out.push_str(&argument); + self.out.push(','); + } + self.indent -= 1; + self.newline(); + self.out.push(')'); + } + + /// The column the text written next will start at — approximately. + /// + /// Exact for text appended to the buffer being emitted, but a piece being + /// [`captured`](Self::captured) has no line to measure yet, so its + /// indentation stands in: that *is* where its first line will be placed. + /// Both under-estimate a keyword prefix a caller adds afterwards + /// (`where: …`), which only ever makes the layout more compact than asked. + fn start(&self) -> usize { + let column = self.out.len() - self.out.rfind('\n').map_or(0, |index| index + 1); + column.max(INDENT.len() * self.indent) + } + + /// A rendered operand, for [`Self::operator`]. + fn operand(&mut self, expr: &Expr) -> String { + self.captured(|printer| printer.visit_expr(expr, OUTERMOST)) + } + + /// `{ a, b: }`. + /// + /// An attribute whose expression is just the variable of the same name is a + /// plain column pick, so it prints as the bare name — which is the shape + /// every atom lowered from coln's FLIR has, and spelling it `a: a` would + /// bury the interesting attributes among the trivial ones. + fn attributes(&mut self, attributes: &[(String, Expr)]) -> String { + if attributes.is_empty() { + return "{}".to_string(); + } + let rendered: Vec = attributes + .iter() + .map(|(name, expr)| match expr { + Expr::Var(var) if var.name == *name => name.clone(), + expr => format!("{name}: {}", self.operand(expr)), + }) + .collect(); + format!("{{ {} }}", rendered.join(", ")) + } + + /// `[a == b, …]`, one entry per attribute pair to match on. The two sides + /// are evaluated against different relations, so equal-looking sides are + /// normal rather than redundant. + fn on_pairs(&mut self, on: &[(Expr, Expr)]) -> String { + let rendered: Vec = on + .iter() + .map(|(left, right)| format!("{} == {}", self.operand(left), self.operand(right))) + .collect(); + format!("[{}]", rendered.join(", ")) + } + + /// `[x: 0.a == 2.b, …]`, one entry per equality class, each occurrence + /// prefixed with the index of the relation it is evaluated against. + fn join_variables(&mut self, on: &[JoinVariable]) -> String { + let rendered: Vec = on + .iter() + .map(|variable| { + let occurrences: Vec = variable + .occurrences + .iter() + .map(|(relation, expr)| format!("{relation}.{}", self.operand(expr))) + .collect(); + format!("{}: {}", variable.name, occurrences.join(" == ")) + }) + .collect(); + format!("[{}]", rendered.join(", ")) + } + + /// The optional projection an equi join carries, as a trailing argument. + fn select(&mut self, attributes: Option<&Vec<(String, Expr)>>) -> Option { + attributes.map(|attributes| format!("select: {}", self.attributes(attributes))) + } +} + +/// A string literal's text, with the escapes it needs to stay one token. +fn escaped(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} + +impl StmtVisitor<(), ()> for SourcePrinter { + fn visit_var_stmt(&mut self, stmt: &VarStmt, ctx: ()) { + emit!(self, "var {}", stmt.name); + if let Some(initializer) = &stmt.initializer { + self.out.push_str(" = "); + self.visit_expr(initializer, OUTERMOST); + } + self.out.push(';'); + } + + fn visit_expr_stmt(&mut self, stmt: &ExprStmt, ctx: ()) { + self.visit_expr(&stmt.expr, OUTERMOST); + self.out.push(';'); + } + + fn visit_block_stmt(&mut self, stmt: &BlockStmt, ctx: ()) { + self.block(&stmt.stmts); + } +} + +/// The context of a host expression is the precedence its position demands: +/// render parentheses if the expression itself binds looser than that. +impl ExprVisitor<(), u8> for SourcePrinter { + fn visit_literal_expr(&mut self, expr: &LiteralExpr, ctx: u8) { + match &expr.value { + // `Literal`'s own `Display` prints a string bare, which would make + // it indistinguishable from a variable here. + Literal::String(value) => emit!(self, "\"{}\"", escaped(value)), + value => emit!(self, "{value}"), + } + } + + fn visit_tuple_expr(&mut self, expr: &TupleExpr, ctx: u8) { + let elements: Vec = expr + .elements + .iter() + .map(|element| self.operand(element)) + .collect(); + // A trailing comma is what distinguishes a one-element tuple from a + // grouping. + let trailing = if elements.len() == 1 { "," } else { "" }; + emit!(self, "({}{trailing})", elements.join(", ")); + } + + fn visit_get_index_expr(&mut self, expr: &GetIndexExpr, ctx: u8) { + self.parenthesized(PRIMARY_PRECEDENCE, ctx, |printer| { + printer.visit_expr(&expr.target, PRIMARY_PRECEDENCE); + printer.out.push('['); + printer.visit_expr(&expr.index, OUTERMOST); + printer.out.push(']'); + }); + } + + fn visit_grouping_expr(&mut self, expr: &GroupingExpr, ctx: u8) { + // An explicit grouping is kept even where precedence makes it + // redundant: it is a node the tree actually contains. + self.out.push('('); + self.visit_expr(&expr.expr, OUTERMOST); + self.out.push(')'); + } + + fn visit_binary_expr(&mut self, expr: &BinaryExpr, ctx: u8) { + let own = expr.operator.precedence(); + self.parenthesized(own, ctx, |printer| { + printer.visit_expr(&expr.left, own); + emit!(printer, " {} ", expr.operator); + // Left-associative, so an equally tight right operand needs + // parentheses to keep its shape. + printer.visit_expr(&expr.right, own + 1); + }); + } + + fn visit_unary_expr(&mut self, expr: &UnaryExpr, ctx: u8) { + self.parenthesized(UNARY_PRECEDENCE, ctx, |printer| { + emit!(printer, "{}", expr.operator); + printer.visit_expr(&expr.operand, UNARY_PRECEDENCE); + }); + } + + fn visit_var_expr(&mut self, expr: &VarExpr, ctx: u8) { + self.out.push_str(&expr.name); + } + + fn visit_assign_expr(&mut self, expr: &AssignExpr, ctx: u8) { + self.parenthesized(OUTERMOST, ctx, |printer| { + emit!(printer, "{} = ", expr.name); + printer.visit_expr(&expr.value, OUTERMOST); + }); + } + + fn visit_function_expr(&mut self, expr: &FunctionExpr, ctx: u8) { + emit!(self, "fn({}) ", expr.parameters.join(", ")); + self.block(&expr.body.stmts); + } + + fn visit_call_expr(&mut self, expr: &CallExpr, ctx: u8) { + self.parenthesized(PRIMARY_PRECEDENCE, ctx, |printer| { + printer.visit_expr(&expr.callee, PRIMARY_PRECEDENCE); + let arguments: Vec = expr + .arguments + .iter() + .map(|argument| printer.operand(argument)) + .collect(); + emit!(printer, "({})", arguments.join(", ")); + }); + } + + fn visit_relational_expr(&mut self, expr: &RelExpr, ctx: u8) { + self.visit_rel(expr, ctx); + } +} + +impl RelExprVisitor<(), u8> for SourcePrinter { + fn visit_source_expr(&mut self, expr: &SourceExpr, ctx: u8) { + // The name *is* the source's identity (see `SourceExpr::to_id`); the + // rest of the schema is derived, and shown by `AsTree` instead. + emit!(self, "source(\"{}\")", escaped(expr.as_id())); + } + + fn visit_output_expr(&mut self, expr: &OutputExpr, ctx: u8) { + let kind = match expr.kind { + OutputKind::Cli => "cli", + OutputKind::Channel => "channel", + }; + self.operator("output", |printer| { + vec![ + printer.operand(&expr.relation), + format!("as: \"{}\"", escaped(expr.id.as_str())), + format!("to: {kind}"), + ] + }); + } + + fn visit_alias_expr(&mut self, expr: &AliasExpr, ctx: u8) { + self.operator("alias", |printer| { + vec![ + printer.operand(&expr.relation), + format!("as: {}", expr.alias), + ] + }); + } + + fn visit_distinct_expr(&mut self, expr: &DistinctExpr, ctx: u8) { + self.operator("distinct", |printer| vec![printer.operand(&expr.relation)]); + } + + fn visit_union_expr(&mut self, expr: &UnionExpr, ctx: u8) { + self.operator("union", |printer| { + expr.relations + .iter() + .map(|relation| printer.operand(relation)) + .collect() + }); + } + + fn visit_difference_expr(&mut self, expr: &DifferenceExpr, ctx: u8) { + self.operator("difference", |printer| { + vec![printer.operand(&expr.left), printer.operand(&expr.right)] + }); + } + + fn visit_selection_expr(&mut self, expr: &SelectionExpr, ctx: u8) { + self.operator("select", |printer| { + vec![ + printer.operand(&expr.relation), + format!("where: {}", printer.operand(&expr.condition)), + ] + }); + } + + fn visit_projection_expr(&mut self, expr: &ProjectionExpr, ctx: u8) { + self.operator("project", |printer| { + vec![ + printer.operand(&expr.relation), + format!("select: {}", printer.attributes(&expr.attributes)), + ] + }); + } + + fn visit_cartesian_product_expr(&mut self, expr: &CartesianProductExpr, ctx: u8) { + let inner = &expr.inner; + // The `on` clause of the delegate is empty by construction, so printing + // it would only add noise. + self.operator("product", |printer| { + [printer.operand(&inner.left), printer.operand(&inner.right)] + .into_iter() + .chain(printer.select(inner.attributes.as_ref())) + .collect() + }); + } + + fn visit_equi_join_expr(&mut self, expr: &EquiJoinExpr, ctx: u8) { + self.operator("join", |printer| { + [ + printer.operand(&expr.left), + printer.operand(&expr.right), + format!("on: {}", printer.on_pairs(&expr.on)), + ] + .into_iter() + .chain(printer.select(expr.attributes.as_ref())) + .collect() + }); + } + + fn visit_multi_way_equi_join_expr(&mut self, expr: &MultiWayEquiJoinExpr, ctx: u8) { + self.operator("multijoin", |printer| { + let relations: Vec = expr + .relations + .iter() + .map(|relation| printer.operand(relation)) + .collect(); + // The relations are one bracketed argument rather than N, because + // `on` addresses them by index. + [ + format!("[{}]", relations.join(", ")), + format!("on: {}", printer.join_variables(&expr.on)), + ] + .into_iter() + .chain(printer.select(expr.attributes.as_ref())) + .collect() + }); + } + + fn visit_anti_join_expr(&mut self, expr: &AntiJoinExpr, ctx: u8) { + self.operator("antijoin", |printer| { + vec![ + printer.operand(&expr.left), + printer.operand(&expr.right), + format!("on: {}", printer.on_pairs(&expr.on)), + ] + }); + } + + fn visit_fixed_point_iter_expr(&mut self, expr: &FixedPointIterExpr, ctx: u8) { + // A binding form rather than a call: the step body can only be read + // with the accumulator's name in scope. + emit!(self, "fix {} = ", expr.accumulator.0); + self.visit_expr(&expr.accumulator.1, OUTERMOST); + self.out.push(' '); + self.block(&expr.step.stmts); + } +} + +/// A program rendered as an indented node tree, e.g. +/// `println!("{}", AsTree(&code))`. +/// +/// One line per node: the role it plays in its parent, its kind, and the +/// payloads that are not children — an operator, a schema, whether a variable +/// has been resolved. Built on [`walk`], so it needs no knowledge of the tree's +/// shape beyond [`Node::push_children`]. +pub struct AsTree<'a>(pub &'a [Stmt]); + +impl Display for AsTree<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // How many children each currently open ancestor still has to enter. + // A node is its parent's last child exactly when that count is 1 as the + // node is entered, which is what the elbow and the guide lines need. + let mut remaining: Vec = Vec::new(); + let mut children = Vec::new(); + let mut first = true; + for event in walk(self.0) { + let child = match event { + Event::Enter(child) => child, + Event::Leave(_) => { + remaining.pop(); + continue; + } + }; + if !first { + f.write_char('\n')?; + } + first = false; + if let Some((parent, ancestors)) = remaining.split_last_mut() { + for ancestor in ancestors { + f.write_str(if *ancestor > 0 { "│ " } else { " " })?; + } + f.write_str(if *parent > 1 { "├─ " } else { "└─ " })?; + *parent -= 1; + } + child.node.push_children(&mut children); + remaining.push(children.len()); + children.clear(); + if child.role != ROOT { + write!(f, "{}: ", child.label())?; + } + f.write_str(&describe(child))?; + } + Ok(()) + } +} + +/// One node as a single line: its kind, plus every payload that is not a child +/// and would therefore be invisible in the tree. +fn describe(child: Child<'_>) -> String { + match child.node { + Node::Stmt(stmt) => match stmt { + Stmt::Var(stmt) => format!("VarStmt {}", stmt.name), + Stmt::Expr(_) => "ExprStmt".to_string(), + Stmt::Block(stmt) => format!("Block ({} stmts)", stmt.stmts.len()), + }, + Node::Expr(expr) => match expr { + Expr::Literal(expr) => match &expr.value { + Literal::String(value) => format!("Literal \"{}\"", escaped(value)), + value => format!("Literal {value}"), + }, + Expr::Tuple(expr) => format!("Tuple ({} elements)", expr.elements.len()), + Expr::GetIndex(_) => "GetIndex".to_string(), + Expr::Grouping(_) => "Grouping".to_string(), + Expr::Binary(expr) => format!("Binary {}", expr.operator), + Expr::Unary(expr) => format!("Unary {}", expr.operator), + // Whether the resolver has been here is invisible in the source + // rendering but is exactly what one debugs a resolution with. + Expr::Var(expr) => format!("Var {}{}", expr.name, slot(expr.resolved)), + Expr::Assign(expr) => format!("Assign {}{}", expr.name, slot(expr.resolved)), + Expr::Function(expr) => format!("Function ({})", expr.parameters.join(", ")), + Expr::Call(_) => "Call".to_string(), + // Normalized away by the walk, see `Node`. + Expr::Relational(_) => "Relational".to_string(), + }, + Node::Rel(rel) => match rel { + RelExpr::Source(expr) => format!( + "Source \"{}\" tuple={} key={}", + expr.as_id(), + expr.schema.tuple, + expr.schema.key + ), + RelExpr::Output(expr) => format!( + "Output \"{}\" {}", + expr.id.as_str(), + match expr.kind { + OutputKind::Cli => "cli", + OutputKind::Channel => "channel", + } + ), + RelExpr::Alias(expr) => format!("Alias {}", expr.alias), + RelExpr::Distinct(_) => "Distinct".to_string(), + RelExpr::Union(_) => "Union".to_string(), + RelExpr::Difference(_) => "Difference".to_string(), + RelExpr::Selection(_) => "Selection".to_string(), + RelExpr::Projection(expr) => { + format!("Projection {}", attribute_names(&expr.attributes)) + } + // A cartesian product delegates to an equi join, so its projection + // lives on that delegate — and its `select` children come from + // there too, which is why the names have to be read off it as well. + RelExpr::CartesianProduct(expr) => { + format!("CartesianProduct{}", select_tag(&expr.inner.attributes)) + } + RelExpr::EquiJoin(expr) => format!("EquiJoin{}", select_tag(&expr.attributes)), + RelExpr::MultiWayEquiJoin(expr) => format!( + "MultiWayEquiJoin on={}{}", + join_variable_classes(&expr.on), + select_tag(&expr.attributes) + ), + RelExpr::AntiJoin(_) => "AntiJoin".to_string(), + RelExpr::FixedPointIter(expr) => format!("FixedPointIter {}", expr.accumulator.0), + }, + } +} + +/// The names a projection produces. The expressions behind them are children, +/// so only the names belong on the node's own line. +fn attribute_names(attributes: &[(String, Expr)]) -> String { + let names: Vec<&str> = attributes.iter().map(|(name, _)| name.as_str()).collect(); + format!("[{}]", names.join(", ")) +} + +/// The ` select=[…]` tag an *optional* projection contributes to a node's line, +/// or nothing when the operator carries none. +/// +/// Every join-shaped operator holds its projection as an `Option`, so they all +/// need the same conditional tag; a [`ProjectionExpr`] always has one and prints +/// it through [`attribute_names`] directly. +fn select_tag(attributes: &Option>) -> String { + attributes + .as_ref() + .map(|attributes| format!(" select={}", attribute_names(attributes))) + .unwrap_or_default() +} + +/// The equality classes of a multi-way join, as `[y: 0=2, z: 1=2]`: which +/// relations each join variable equates. +/// +/// The occurrence expressions are children, so only the relation indices belong +/// here — and they *have* to be here, because a +/// [`RelationIdx`](crate::relational::expr::RelationIdx) is a payload rather +/// than a node. Without them the drawing could not say whether `y` is bound by +/// relations 0 and 2 or by 1 and 2, which is the whole content of the join. +fn join_variable_classes(on: &[JoinVariable]) -> String { + let classes: Vec = on + .iter() + .map(|variable| { + let relations: Vec = variable + .occurrences + .iter() + .map(|(relation, _)| relation.to_string()) + .collect(); + format!("{}: {}", variable.name, relations.join("=")) + }) + .collect(); + format!("[{}]", classes.join(", ")) +} + +/// The resolved slot of a variable reference, as `@scope:index`, or a marker +/// that the resolver has not reached it. +fn slot(resolved: Option<(usize, usize)>) -> String { + match resolved { + Some((scope, index)) => format!(" @{scope}:{index}"), + None => " (unresolved)".to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::relational::{RelationSchema, expr::SinkId}; + + fn schema(name: &str) -> RelationSchema { + RelationSchema::new(name, ["x", "y"], ["x"]).expect("Correct schema definition") + } + + fn var(name: &str) -> Expr { + Expr::from(VarExpr::new(name)) + } + + fn expr_stmt(expr: Expr) -> Stmt { + Stmt::from(ExprStmt { expr }) + } + + /// The plan a transitive-closure lowering produces: a fixed point whose step + /// joins the accumulator against a source and unions the result back in. + /// Parameterized by the names, which is what decides whether the source + /// rendering fits on one line — the tree rendering is unaffected by them. + fn transitive_closure(edge: &str, acc: &str, from: &str, to: &str) -> Vec { + vec![ + Stmt::from(VarStmt { + name: edge.to_string(), + initializer: Some(Expr::from(SourceExpr::new(schema(edge)))), + }), + Stmt::from(VarStmt { + name: "reach".to_string(), + initializer: Some(Expr::from(FixedPointIterExpr { + accumulator: (acc.to_string(), var(edge)), + step: BlockStmt { + stmts: vec![expr_stmt(Expr::from(DistinctExpr { + relation: Expr::from(UnionExpr { + relations: vec![ + var(acc), + Expr::from(EquiJoinExpr { + left: var(acc), + right: var(edge), + on: vec![(var(to), var(from))], + attributes: Some(vec![ + (from.to_string(), var(from)), + (to.to_string(), var(to)), + ]), + }), + ], + }), + }))], + }, + })), + }), + expr_stmt(Expr::from(OutputExpr { + relation: var("reach"), + id: SinkId::from("reach"), + kind: OutputKind::Channel, + })), + ] + } + + /// [`AsSource`]: is the program readable, and does it stay faithful while + /// getting there — parentheses, quoting, and where the lines break. + mod source { + use super::*; + use crate::host::operator::Operator; + + fn binary(operator: Operator, left: Expr, right: Expr) -> Expr { + Expr::from(BinaryExpr { + operator, + left, + right, + }) + } + + fn literal(value: u64) -> Expr { + Expr::from(LiteralExpr::from(value)) + } + + #[test] + fn renders_precedence_without_grouping_nodes() { + // A programmatically built tree carries no `GroupingExpr`, so the + // parentheses have to come from the operators themselves. + let code = vec![expr_stmt(binary( + Operator::Multiplication, + binary(Operator::Addition, literal(1), literal(2)), + literal(3), + ))]; + assert_eq!(to_source(&code), "(1 + 2) * 3;"); + } + + #[test] + fn omits_parentheses_where_precedence_already_agrees() { + let code = vec![expr_stmt(binary( + Operator::Addition, + binary(Operator::Multiplication, literal(1), literal(2)), + literal(3), + ))]; + assert_eq!(to_source(&code), "1 * 2 + 3;"); + } + + #[test] + fn keeps_a_right_nested_operand_of_equal_precedence_parenthesized() { + // `-` is left-associative, so `1 - (2 - 3)` must not print as `1 - 2 - 3`. + let code = vec![expr_stmt(binary( + Operator::Subtraction, + literal(1), + binary(Operator::Subtraction, literal(2), literal(3)), + ))]; + assert_eq!(to_source(&code), "1 - (2 - 3);"); + } + + #[test] + fn quotes_string_literals_so_they_are_not_variables() { + let code = vec![expr_stmt(binary( + Operator::Equal, + var("name"), + Expr::from(LiteralExpr::from("a\"b")), + ))]; + assert_eq!(to_source(&code), "name == \"a\\\"b\";"); + } + + #[test] + fn renders_an_atom_lowering_on_one_line() { + // `project(select(source))` is what one FLIR atom lowers to. Its only + // nested operator is the source leaf, so it stays compact — and the + // column picks print as bare names. + let code = vec![expr_stmt(Expr::from(ProjectionExpr { + relation: Expr::from(SelectionExpr { + relation: Expr::from(SourceExpr::new(schema("edge"))), + condition: binary(Operator::Greater, var("x"), literal(1)), + }), + attributes: vec![("x".to_string(), var("x")), ("z".to_string(), var("y"))], + }))]; + assert_eq!( + to_source(&code), + "project(select(source(\"edge\"), where: x > 1), select: { x, z: y });" + ); + } + + #[test] + fn keeps_a_whole_fixed_point_step_on_one_line_when_it_fits() { + // Three nested operators, still well inside the width budget: breaking + // them would cost eight lines and buy nothing. + assert_eq!( + to_source(&transitive_closure("edge", "acc", "x", "y")), + "\ +var edge = source(\"edge\"); +var reach = fix acc = edge { + distinct(union(acc, join(acc, edge, on: [y == x], select: { x, y }))); +}; +output(reach, as: \"reach\", to: channel);" + ); + } + + #[test] + fn breaks_a_nested_plan_that_does_not_fit() { + // The same plan with realistic names no longer fits, so each operator + // takes one argument per line — and the indentation of a broken + // argument's own inner lines has to follow. + assert_eq!( + to_source(&transitive_closure( + "transitive_edge", + "accumulated", + "source_node", + "target_node" + )), + "\ +var transitive_edge = source(\"transitive_edge\"); +var reach = fix accumulated = transitive_edge { + distinct( + union( + accumulated, + join( + accumulated, + transitive_edge, + on: [target_node == source_node], + select: { source_node, target_node }, + ), + ), + ); +}; +output(reach, as: \"reach\", to: channel);" + ); + } + + #[test] + fn renders_a_multi_way_join_by_relation_index() { + let code = vec![expr_stmt(Expr::from( + MultiWayEquiJoinExpr::new( + vec![var("r0"), var("r1"), var("r2")], + vec![JoinVariable { + name: "y".to_string(), + occurrences: vec![(0, var("y")), (2, var("y"))], + }], + None, + ) + .expect("a variable bound by two relations is a join variable"), + ))]; + assert_eq!( + to_source(&code), + "multijoin([r0, r1, r2], on: [y: 0.y == 2.y]);" + ); + } + } + + /// [`AsTree`]: does every node land at the right place in the drawing, and + /// does it carry the payloads the source rendering has no room for. + mod tree { + use super::*; + + #[test] + fn renders_a_whole_plan_the_source_rendering_compresses() { + // The plan whose entire fixed-point step the source rendering puts + // on one line. One node per line instead, which is what pins the + // shape down: the join's six operands told apart only by their role, + // the crossing back into statements at `step`, and the schema of the + // source leaf. Every variable is still `(unresolved)` because this + // plan has not been through the resolver. + assert_eq!( + to_tree(&transitive_closure("edge", "acc", "x", "y")), + "\ +VarStmt edge +└─ init: Source \"edge\" tuple=| x | y | key=| x | +VarStmt reach +└─ init: FixedPointIter acc + ├─ init: Var edge (unresolved) + └─ step[0]: ExprStmt + └─ expr: Distinct + └─ relation: Union + ├─ relation[0]: Var acc (unresolved) + └─ relation[1]: EquiJoin select=[x, y] + ├─ left: Var acc (unresolved) + ├─ right: Var edge (unresolved) + ├─ on[0].left: Var y (unresolved) + ├─ on[0].right: Var x (unresolved) + ├─ select[0]: Var x (unresolved) + └─ select[1]: Var y (unresolved) +ExprStmt +└─ expr: Output \"reach\" channel + └─ relation: Var reach (unresolved)" + ); + } + + #[test] + fn names_the_projection_a_cartesian_product_carries_on_its_delegate() { + // A `CartesianProductExpr` keeps its projection on the `EquiJoinExpr` + // it delegates to, and the walk reports that delegate's `select` + // children — so a node line that looked only at the product itself + // would show the attribute expressions with no names to bind them to. + let code = vec![expr_stmt(Expr::from(CartesianProductExpr::new( + var("l"), + var("r"), + Some(vec![ + ("out".to_string(), var("a")), + ("keep".to_string(), var("b")), + ]), + )))]; + assert_eq!( + to_tree(&code), + "\ +ExprStmt +└─ expr: CartesianProduct select=[out, keep] + ├─ left: Var l (unresolved) + ├─ right: Var r (unresolved) + ├─ select[0]: Var a (unresolved) + └─ select[1]: Var b (unresolved)" + ); + } + + #[test] + fn spells_out_which_relations_each_join_variable_equates() { + // The occurrence expressions arrive as flat `on` children, so the + // relation indices are the one part of a multi-way join that only + // the parent line can carry. `y` joins relations 0 and 2, `z` joins + // 1 and 2 — and relation 1 is reachable *only* through `z`. + let code = vec![expr_stmt(Expr::from( + MultiWayEquiJoinExpr::new( + vec![var("r0"), var("r1"), var("r2")], + vec![ + JoinVariable { + name: "y".to_string(), + occurrences: vec![(0, var("y")), (2, var("y"))], + }, + JoinVariable { + name: "z".to_string(), + occurrences: vec![(1, var("z")), (2, var("z"))], + }, + ], + None, + ) + .expect("every variable is bound by two relations"), + ))]; + assert_eq!( + to_tree(&code), + "\ +ExprStmt +└─ expr: MultiWayEquiJoin on=[y: 0=2, z: 1=2] + ├─ relation[0]: Var r0 (unresolved) + ├─ relation[1]: Var r1 (unresolved) + ├─ relation[2]: Var r2 (unresolved) + ├─ on[0]: Var y (unresolved) + ├─ on[0]: Var y (unresolved) + ├─ on[1]: Var z (unresolved) + └─ on[1]: Var z (unresolved)" + ); + } + + #[test] + fn tags_every_child_with_its_role() { + // What the source rendering cannot show: which operand a child is, and + // that the resolver has not run yet. + let code = vec![expr_stmt(Expr::from(EquiJoinExpr { + left: var("l"), + right: var("r"), + on: vec![(var("a"), var("b"))], + attributes: None, + }))]; + assert_eq!( + to_tree(&code), + "\ +ExprStmt +└─ expr: EquiJoin + ├─ left: Var l (unresolved) + ├─ right: Var r (unresolved) + ├─ on[0].left: Var a (unresolved) + └─ on[0].right: Var b (unresolved)" + ); + } + + #[test] + fn shows_the_payloads_the_source_rendering_omits() { + let code = vec![Stmt::from(VarStmt { + name: "edge".to_string(), + initializer: Some(Expr::from(DistinctExpr { + relation: Expr::from(SourceExpr::new(schema("edge"))), + })), + })]; + assert_eq!( + to_tree(&code), + "\ +VarStmt edge +└─ init: Distinct + └─ relation: Source \"edge\" tuple=| x | y | key=| x |" + ); + } + + #[test] + fn guides_lines_through_deeper_siblings() { + // The `│` of the outer union has to continue past the whole first + // branch, otherwise a deep tree cannot be read. + let code = vec![expr_stmt(Expr::from(UnionExpr { + relations: vec![Expr::from(DistinctExpr { relation: var("a") }), var("b")], + }))]; + assert_eq!( + to_tree(&code), + "\ +ExprStmt +└─ expr: Union + ├─ relation[0]: Distinct + │ └─ relation: Var a (unresolved) + └─ relation[1]: Var b (unresolved)" + ); + } + } +} diff --git a/packages/coln-query/src/host/walk.rs b/packages/coln-query/src/host/walk.rs new file mode 100644 index 00000000..9256d14b --- /dev/null +++ b/packages/coln-query/src/host/walk.rs @@ -0,0 +1,730 @@ +// SPDX-FileCopyrightText: 2026 Coln contributors +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Generic depth-first traversal of the AST, for consumers that only need to +//! *scan* it. +//! +//! # Why this exists next to the visitor traits +//! +//! The AST has two kinds of consumer, and they need opposite things: +//! +//! - A **fold** computes a value per node out of its children's values, and the +//! parent decides whether and in which order children are visited at all +//! (`TypeResolver::visit_unary_expr` answers `Not` without ever looking at the +//! operand; `DbspInterpreter::visit_equi_join_expr` must consume the left +//! alias *between* its two operands). No traversal can be factored out of +//! those — they stay on the [`ExprVisitor`](super::expr::ExprVisitor) family. +//! - A **scan** reads nodes, produces no per-node result, and does not care +//! about order. Everything a scan needs from the AST's shape is "what are this +//! node's children", which is exactly what this module states *once*. +//! +//! So this is not a replacement for the visitors, it is the other half: scans +//! stop restating the tree's shape, and a potential future flatter +//! representation only has to re-implement [`Node::push_children`] to keep +//! every scan working. +//! +//! # Traversal orders +//! +//! [`Walk`] yields [`Enter`](Event::Enter)/[`Leave`](Event::Leave) events, from +//! which every order follows: [`pre_order`] keeps the enters, [`post_order`] +//! keeps the leaves. A static order is not the real execution order either way: +//! A [`CallExpr`](super::expr::CallExpr) jumps into a function body and a +//! [`FixedPointIterExpr`](crate::relational::expr::FixedPointIterExpr) repeats +//! its step. +//! +//! There is deliberately no `&mut` counterpart. A `&mut` to a parent and to its +//! children cannot be held at once, so a mutable walk could only ever be +//! pre-order and could never emit [`Leave`](Event::Leave) — which is precisely +//! the half `Resolver` needs, to pop the scopes and tuple contexts it pushes on +//! the way down. Rewriting passes stay visitors. + +use crate::{ + host::{expr::Expr, stmt::Stmt}, + relational::expr::{EquiJoinExpr, RelExpr, SourceExpr}, +}; + +/// A borrowed pointer to a node of any of the three mutually recursive node +/// kinds. The kinds interleave in both directions — a +/// [`FunctionExpr`](super::expr::FunctionExpr) body and a +/// [`FixedPointIterExpr`](crate::relational::expr::FixedPointIterExpr) step are +/// statements — so a traversal has to be able to hold any of them. +/// +/// [`Expr::Relational`] never appears as a [`Node::Expr`]: it is a pure wrapper +/// carrying no data of its own, so the walk yields the [`RelExpr`] inside it +/// directly rather than both. +#[derive(Clone, Copy, Debug)] +pub enum Node<'a> { + Stmt(&'a Stmt), + Expr(&'a Expr), + Rel(&'a RelExpr), +} + +impl<'a> From<&'a Stmt> for Node<'a> { + fn from(stmt: &'a Stmt) -> Self { + Node::Stmt(stmt) + } +} + +impl<'a> From<&'a Expr> for Node<'a> { + fn from(expr: &'a Expr) -> Self { + match expr { + // Unwrap the bridge into the relational layer, see [`Node`]. + Expr::Relational(rel) => Node::Rel(rel), + expr => Node::Expr(expr), + } + } +} + +impl<'a> From<&'a RelExpr> for Node<'a> { + fn from(rel: &'a RelExpr) -> Self { + Node::Rel(rel) + } +} + +/// A child of a node, tagged with the position it occupies in its parent. +/// +/// A scan ignores all of it; a rendering needs it, because a bare node cannot +/// say *which* operand it is: an [`EquiJoinExpr`]'s left relation and one of its +/// join keys are both plain [`Expr`]s. The three parts spell out one position +/// exactly — see [`label`](Self::label) for how they read together — so a +/// consumer never has to recover it by counting children and knowing which +/// field is a `Vec` of what. +#[derive(Clone, Copy, Debug)] +pub struct Child<'a> { + /// The field this child came from. + pub role: &'static str, + /// Which occurrence within [`role`](Self::role), when that field holds a + /// collection. `None` for a field holding a single child. + pub index: Option, + /// Which part of that occurrence, when the collection holds tuples — the + /// two sides of a join's `on` pair. `None` otherwise. + pub part: Option<&'static str>, + pub node: Node<'a>, +} + +/// The role reported for the node a walk was started from. +pub const ROOT: &str = "root"; + +impl<'a> Child<'a> { + /// The only child of a field: `relation`, `condition`, `callee`. + fn new(role: &'static str, node: impl Into>) -> Self { + Self { + role, + index: None, + part: None, + node: node.into(), + } + } + + /// One element of a collection field: `argument[1]`, `select[0]`. + fn at(role: &'static str, index: usize, node: impl Into>) -> Self { + Self { + index: Some(index), + ..Self::new(role, node) + } + } + + /// One part of a tuple element of a collection field: `on[0].left`. + fn part( + role: &'static str, + index: usize, + part: &'static str, + node: impl Into>, + ) -> Self { + Self { + part: Some(part), + ..Self::at(role, index, node) + } + } + + fn root(node: impl Into>) -> Self { + Self::new(ROOT, node) + } + + /// This child's position as a path — `relation`, `select[1]`, + /// `on[0].left` — which is what a rendering puts in front of the node. + pub fn label(&self) -> String { + let index = self + .index + .map(|index| format!("[{index}]")) + .unwrap_or_default(); + let part = self.part.map(|part| format!(".{part}")).unwrap_or_default(); + format!("{}{index}{part}", self.role) + } +} + +impl<'a> Node<'a> { + /// Append this node's node-typed children to `out`, in source order, each + /// tagged with the position it occupies here. + /// + /// **This is the only place the AST's shape is spelled out for traversal + /// purposes.** Non-node payloads (an operator, an attribute's name, a + /// [`SinkId`](crate::relational::expr::SinkId)) are not children and are + /// reached by matching on the node itself — which is also why a + /// [`Child`]'s position is worth stating: it is what lets a consumer tie a + /// child back to the payload that names it, without re-deriving the field + /// layout it is trying not to depend on. + pub fn push_children(self, out: &mut Vec>) { + match self { + Node::Stmt(stmt) => Self::push_stmt_children(stmt, out), + Node::Expr(expr) => Self::push_expr_children(expr, out), + Node::Rel(rel) => Self::push_rel_children(rel, out), + } + } + + fn push_stmt_children(stmt: &'a Stmt, out: &mut Vec>) { + match stmt { + // At most one initializer, so there is nothing to index. + Stmt::Var(stmt) => { + out.extend(stmt.initializer.iter().map(|expr| Child::new("init", expr))) + } + Stmt::Expr(stmt) => out.push(Child::new("expr", &stmt.expr)), + Stmt::Block(stmt) => out.extend( + stmt.stmts + .iter() + .enumerate() + .map(|(index, stmt)| Child::at("stmt", index, stmt)), + ), + } + } + + fn push_expr_children(expr: &'a Expr, out: &mut Vec>) { + match expr { + Expr::Literal(_) | Expr::Var(_) => {} + Expr::Tuple(expr) => out.extend( + expr.elements + .iter() + .enumerate() + .map(|(index, expr)| Child::at("element", index, expr)), + ), + Expr::GetIndex(expr) => out.extend([ + Child::new("target", &expr.target), + Child::new("index", &expr.index), + ]), + Expr::Grouping(expr) => out.push(Child::new("expr", &expr.expr)), + Expr::Binary(expr) => out.extend([ + Child::new("left", &expr.left), + Child::new("right", &expr.right), + ]), + Expr::Unary(expr) => out.push(Child::new("operand", &expr.operand)), + Expr::Assign(expr) => out.push(Child::new("value", &expr.value)), + Expr::Call(expr) => { + out.push(Child::new("callee", &expr.callee)); + out.extend( + expr.arguments + .iter() + .enumerate() + .map(|(index, expr)| Child::at("argument", index, expr)), + ); + } + // The parameters are plain names, so only the body holds nodes. + Expr::Function(expr) => out.extend( + expr.body + .stmts + .iter() + .enumerate() + .map(|(index, stmt)| Child::at("body", index, stmt)), + ), + // Normalized away by `Node::from`; reachable only through a + // hand-built `Node::Expr`, which is handled rather than pruned. + Expr::Relational(rel) => out.push(Child::new("rel", &**rel)), + } + } + + fn push_rel_children(rel: &'a RelExpr, out: &mut Vec>) { + match rel { + // A plan leaf: it only *names* an extensional relation. + RelExpr::Source(_) => {} + RelExpr::Output(expr) => out.push(Child::new("relation", &expr.relation)), + RelExpr::Alias(expr) => out.push(Child::new("relation", &expr.relation)), + RelExpr::Distinct(expr) => out.push(Child::new("relation", &expr.relation)), + RelExpr::Union(expr) => out.extend( + expr.relations + .iter() + .enumerate() + .map(|(index, expr)| Child::at("relation", index, expr)), + ), + RelExpr::Difference(expr) => out.extend([ + Child::new("left", &expr.left), + Child::new("right", &expr.right), + ]), + RelExpr::Selection(expr) => out.extend([ + Child::new("relation", &expr.relation), + Child::new("condition", &expr.condition), + ]), + RelExpr::Projection(expr) => { + out.push(Child::new("relation", &expr.relation)); + // The attribute *name* stays a payload of the projection, so the + // index is what ties this expression back to it. + out.extend( + expr.attributes + .iter() + .enumerate() + .map(|(index, (_, expr))| Child::at("select", index, expr)), + ); + } + // A cartesian product is an equi join with an empty `on`, so it has + // no children beyond that join's. + RelExpr::CartesianProduct(expr) => Self::push_equi_join_children(&expr.inner, out), + RelExpr::EquiJoin(expr) => Self::push_equi_join_children(expr, out), + RelExpr::MultiWayEquiJoin(expr) => { + out.extend( + expr.relations + .iter() + .enumerate() + .map(|(index, expr)| Child::at("relation", index, expr)), + ); + // Indexed by equality class rather than by flattened position, + // so `on[0]` names the same class the node's own rendering + // reports the relation indices for. This is why `on_exprs` is + // not used here: it drops the class boundaries. + out.extend(expr.on.iter().enumerate().flat_map(|(index, variable)| { + variable + .occurrences + .iter() + .map(move |(_, expr)| Child::at("on", index, expr)) + })); + out.extend( + expr.attributes + .iter() + .flatten() + .enumerate() + .map(|(index, (_, expr))| Child::at("select", index, expr)), + ); + } + RelExpr::AntiJoin(expr) => { + out.extend([ + Child::new("left", &expr.left), + Child::new("right", &expr.right), + ]); + out.extend( + expr.on + .iter() + .enumerate() + .flat_map(|(index, (left, right))| { + [ + Child::part("on", index, "left", left), + Child::part("on", index, "right", right), + ] + }), + ); + } + RelExpr::FixedPointIter(expr) => { + out.push(Child::new("init", &expr.accumulator.1)); + out.extend( + expr.step + .stmts + .iter() + .enumerate() + .map(|(index, stmt)| Child::at("step", index, stmt)), + ); + } + } + } + + fn push_equi_join_children(expr: &'a EquiJoinExpr, out: &mut Vec>) { + out.extend([ + Child::new("left", &expr.left), + Child::new("right", &expr.right), + ]); + // Each pair is one equality to match on, and its two sides are evaluated + // against *different* relations, so which side a child is cannot be left + // to the reader's arithmetic. + out.extend( + expr.on + .iter() + .enumerate() + .flat_map(|(index, (left, right))| { + [ + Child::part("on", index, "left", left), + Child::part("on", index, "right", right), + ] + }), + ); + out.extend( + expr.attributes + .iter() + .flatten() + .enumerate() + .map(|(index, (_, expr))| Child::at("select", index, expr)), + ); + } + + /// Every event of the subtree rooted at this node. + pub fn walk(self) -> Walk<'a> { + Walk::new([Child::root(self)]) + } + + /// This node and its descendants, parents before children. + pub fn pre_order(self) -> impl Iterator> { + self.walk().filter_map(|event| event.entered()) + } + + /// This node and its descendants, children before parents. + pub fn post_order(self) -> impl Iterator> { + self.walk().filter_map(|event| event.left()) + } + + pub fn as_stmt(self) -> Option<&'a Stmt> { + match self { + Node::Stmt(stmt) => Some(stmt), + _ => None, + } + } + + pub fn as_expr(self) -> Option<&'a Expr> { + match self { + Node::Expr(expr) => Some(expr), + _ => None, + } + } + + pub fn as_rel(self) -> Option<&'a RelExpr> { + match self { + Node::Rel(rel) => Some(rel), + _ => None, + } + } + + /// The [`SourceExpr`] leaf this node is, if any. What a plan-wide source + /// discovery filters a walk on. + pub fn as_source(self) -> Option<&'a SourceExpr> { + match self { + Node::Rel(RelExpr::Source(source)) => Some(source), + _ => None, + } + } +} + +/// One step of a [`Walk`]. Every node is reported twice, so a consumer can pick +/// its order ([`pre_order`], [`post_order`]) or track depth, without the walk +/// having to offer one iterator per traversal. +#[derive(Clone, Copy, Debug)] +pub enum Event<'a> { + Enter(Child<'a>), + Leave(Child<'a>), +} + +impl<'a> Event<'a> { + /// The child, whichever half of its visit this is. + pub fn child(self) -> Child<'a> { + match self { + Event::Enter(child) | Event::Leave(child) => child, + } + } + + pub fn entered(self) -> Option> { + match self { + Event::Enter(child) => Some(child.node), + Event::Leave(_) => None, + } + } + + pub fn left(self) -> Option> { + match self { + Event::Leave(child) => Some(child.node), + Event::Enter(_) => None, + } + } +} + +/// A depth-first walk over one or more subtrees, as a stream of [`Event`]s. +/// Iterative rather than recursive, so a deeply nested plan cannot exhaust the +/// stack. +pub struct Walk<'a> { + /// Pending work, innermost last. A node's `Leave` is pushed underneath its + /// children when the node is entered. + pending: Vec>, + /// Scratch space for [`Node::push_children`], reused across nodes so the + /// walk allocates amortized nothing per node. + children: Vec>, +} + +impl<'a> Walk<'a> { + fn new(roots: impl IntoIterator>) -> Self { + let mut pending: Vec> = roots.into_iter().map(Event::Enter).collect(); + // The stack is popped from the back, so the first root has to end up + // last. + pending.reverse(); + Self { + pending, + children: Vec::new(), + } + } +} + +impl<'a> Iterator for Walk<'a> { + type Item = Event<'a>; + + fn next(&mut self) -> Option { + let event = self.pending.pop()?; + if let Event::Enter(child) = event { + // The node's own leave sits below its children, so it is reported + // once the whole subtree is done. + self.pending.push(Event::Leave(child)); + child.node.push_children(&mut self.children); + self.pending + .extend(self.children.drain(..).rev().map(Event::Enter)); + } + Some(event) + } +} + +/// Every event of `code`, depth-first, statements in order. +pub fn walk(code: &[Stmt]) -> Walk<'_> { + Walk::new(code.iter().map(Child::root)) +} + +/// Every node of `code`, parents before children. +pub fn pre_order(code: &[Stmt]) -> impl Iterator> { + walk(code).filter_map(|event| event.entered()) +} + +/// Every node of `code`, children before parents — the order in which an +/// interpreter reduces them. +pub fn post_order(code: &[Stmt]) -> impl Iterator> { + walk(code).filter_map(|event| event.left()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::host::{ + expr::{BinaryExpr, LiteralExpr, VarExpr}, + operator::Operator, + stmt::{ExprStmt, VarStmt}, + }; + use crate::relational::{ + RelationSchema, + expr::{ + DistinctExpr, EquiJoinExpr, JoinVariable, MultiWayEquiJoinExpr, SelectionExpr, + UnionExpr, + }, + }; + + fn schema(name: &str) -> RelationSchema { + RelationSchema::new(name, ["a", "b"], ["a"]).expect("Correct schema definition") + } + + /// `1 + 2` as a statement. + fn arithmetic() -> Vec { + vec![Stmt::from(ExprStmt { + expr: Expr::from(BinaryExpr { + operator: Operator::Addition, + left: Expr::from(LiteralExpr::from(1_u64)), + right: Expr::from(LiteralExpr::from(2_u64)), + }), + })] + } + + /// A label per node kind, enough to pin an order down in a test. + fn label(node: Node<'_>) -> String { + match node { + Node::Stmt(stmt) => match stmt { + Stmt::Var(stmt) => format!("var {}", stmt.name), + Stmt::Expr(_) => "stmt".to_string(), + Stmt::Block(_) => "block".to_string(), + }, + Node::Expr(expr) => match expr { + Expr::Literal(expr) => expr.value.to_string(), + Expr::Binary(expr) => expr.operator.to_string(), + Expr::Var(expr) => expr.name.clone(), + other => format!("{other:?}"), + }, + Node::Rel(rel) => match rel { + RelExpr::Source(source) => format!("source {}", source.as_id()), + RelExpr::Distinct(_) => "distinct".to_string(), + RelExpr::Union(_) => "union".to_string(), + RelExpr::Selection(_) => "selection".to_string(), + RelExpr::EquiJoin(_) => "join".to_string(), + other => format!("{other:?}"), + }, + } + } + + fn labels<'a>(nodes: impl Iterator>) -> Vec { + nodes.map(label).collect() + } + + #[test] + fn pre_order_reports_parents_before_children_left_to_right() { + let code = arithmetic(); + assert_eq!(labels(pre_order(&code)), ["stmt", "+", "1", "2"]); + } + + #[test] + fn post_order_is_evaluation_order() { + let code = arithmetic(); + assert_eq!(labels(post_order(&code)), ["1", "2", "+", "stmt"]); + } + + #[test] + fn every_node_is_entered_and_left_exactly_once() { + let code = arithmetic(); + let events: Vec<_> = walk(&code).collect(); + assert_eq!(events.len(), 2 * pre_order(&code).count()); + // A well-formed nesting: the depth returns to zero and never goes below. + let mut depth = 0_isize; + for event in events { + depth += match event { + Event::Enter(_) => 1, + Event::Leave(_) => -1, + }; + assert!(depth >= 0, "left a node that was never entered"); + } + assert_eq!(depth, 0, "entered a node that was never left"); + } + + #[test] + fn the_relational_wrapper_is_not_reported_as_a_node() { + // `Expr::Relational` carries nothing of its own, so `distinct(source)` + // must be two nodes below the statement, not four. + let code = vec![Stmt::from(ExprStmt { + expr: Expr::from(DistinctExpr { + relation: Expr::from(SourceExpr::new(schema("edge"))), + }), + })]; + assert_eq!( + labels(pre_order(&code)), + ["stmt", "distinct", "source edge"] + ); + } + + #[test] + fn walking_crosses_between_statements_and_expressions() { + // A source nested in a var initializer inside a union: the walk has to + // change node kind twice to reach it. + let code = vec![Stmt::from(VarStmt { + name: "both".to_string(), + initializer: Some(Expr::from(UnionExpr { + relations: vec![ + Expr::from(SourceExpr::new(schema("left"))), + Expr::from(VarExpr::new("right")), + ], + })), + })]; + assert_eq!( + labels(pre_order(&code)), + ["var both", "union", "source left", "right"] + ); + } + + #[test] + fn a_scan_filters_the_walk_instead_of_restating_the_tree() { + // The pattern that replaced the hand-written source collection in the + // DBSP backend: no knowledge of the tree's shape at the call site. + let code = vec![Stmt::from(ExprStmt { + expr: Expr::from(SelectionExpr { + relation: Expr::from(UnionExpr { + relations: vec![ + Expr::from(SourceExpr::new(schema("left"))), + Expr::from(SourceExpr::new(schema("right"))), + ], + }), + condition: Expr::from(VarExpr::new("a")), + }), + })]; + let sources: Vec<&str> = pre_order(&code) + .filter_map(Node::as_source) + .map(SourceExpr::as_id) + .collect(); + assert_eq!(sources, ["left", "right"]); + } + + /// Every child of `code`'s first relational node, as `