diff --git a/.claude/board/AGENT_LOG.md b/.claude/board/AGENT_LOG.md index 8777d9931..6a13a366a 100644 --- a/.claude/board/AGENT_LOG.md +++ b/.claude/board/AGENT_LOG.md @@ -1,3 +1,20 @@ +## 2026-09-23 — W0 recon: OGAR DocIR projection seam (read-only, one Opus general-purpose run) + +- D-ids: D-RPT-3, D-RPT-4. No edits, no cargo. +- **Finding (VERIFIED-IN-CODE, file:line cited in the run):** `DocCompose` / + `ObjectSlot` / `ResolutionMode` / `resolve_doc` + `DocObjectSource` are SHIPPED. + `DocRenderer` / `FieldRow` / `FieldView::Table` are SPEC-ONLY, and the + resolved projection had no 2-D shape. +- Consequences: + - the ONLY irreducible gap is a grid outcome, which became OGAR #307; + - `lance-graph-report`'s own HTML terminal was a parallel renderer, and was + removed; + - the adapter must be an EXCLUDED lance-graph crate, because of the dependency + direction. +- Also found: paperless-rs is a dead copy (its CLAUDE.md marks it SUPERSEDED + 2026-08-24); the intake seam lives in tesseract-rs `crates/tesseract-paperless*`. + Tantivy there is a borrowed-token probe plus an ordinary BM25 index. + ## 2026-09-22 — SAP/CATS fold-first dependency stack (PR #1257, unmerged) - Substrate: ndarray #318 `d0376505`; lance-graph #1256 `4d27032e`. diff --git a/.claude/board/ISSUES.md b/.claude/board/ISSUES.md index 6777b9c3e..7e3df4816 100644 --- a/.claude/board/ISSUES.md +++ b/.claude/board/ISSUES.md @@ -1,3 +1,20 @@ +## ISS-REPORT-NO-COMPOSITE-KEY-GROUP-FOLD — a 2-D fold costs one pass per partition member (2026-09-23) + +**Status:** OPEN. **Basis:** VERIFIED-IN-CODE. mask-risc's `GroupKey` is one +`U32` lane or a `Via` hop, so `lance-graph-report` folds the widest coordinate as +the key and evaluates every other coordinate as a PARTITION. A partition is a +member conjunct, run tile-local, one pass per member tuple. + +- **Measured:** a 32×12 pivot over 4M rows takes 25 scans / 177 ms (report_bench). +- **Consequence:** a high-cardinality, densely-observed partition side exceeds + `PlannerPolicy::pass_budget`, and the plan is REFUSED (`ReportError::PassBudget`). + It is never run slowly, and never allocated densely. +- **What closes it:** a multi-lane (composite-key) masked group fold in `ndarray::simd`, + named in mask-risc as a `GroupKey` variant, so that ONE pass folds (a, b) → a + dense or sparse cell. +- **What does NOT close it:** a report-side row loop, or a population-sized composite + key lane. Both are forbidden by the missing-capability STOP rule. + ## ISS-R2IL-PROBE-HAS-NO-CI-LINE-UNTIL-OGAR-305 — RESOLVED (OGAR #305 merged; step restored) **Status:** RESOLVED 2026-09-20. AdaWorldAPI/OGAR #305 merged (OGAR main diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index e9f5fb8fe..223662f1d 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,16 @@ +## 2026-09-23 — ReportPlan + report-as-OGAR-projection (branch `claude/brave-mayer-65y3cy`, unmerged) + +### Current Contract Inventory — UNCHANGED (no contract type added; two crates added) +- NEW workspace member `lance-graph-report`: `ReportPlan`, `Selection`, + `CoordSpec` (`Field` / derived `Bucket`), `Measure` / `FoldState` (mergeable), + `PhysicalKey` (role-free), `PlannerPolicy`, `PhysicalPlan` (explain), + `CellSpace` + `View` = `ReportResult`, `boundary::{MemKv, CamLabels, Catalog}` + (KV over the contract's `ContentStore`/`ContentSink`). Lowers into Quack only. +- NEW excluded crate `lance-graph-report-ogar`: `ReportSource` + (`ogar_doc_ir::resolve::DocObjectSource`), `Orientation`, `emit_typst`. + Needs OGAR #307 (`SlotOutcome::Grid`). +- Companion: z8run#1 (`z8run-lance`). + ## 2026-09-22 — SAP/CATS fold-first dependency stack (PR #1257, unmerged) - Substrate: ndarray #318 `d0376505`; lance-graph #1256 `4d27032e`. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 9f1daee4f..8f64f484f 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -1,3 +1,15 @@ +## D-RPT — ReportPlan / zero-copy pivot / report-as-OGAR-projection (2026-09-23) + +Entry: `entries/2026-09-23-report-plan-zero-copy-pivot-docir-convergence.md`. + +| D-id | scope | status | gate / falsifier | +|---|---|---|---| +| **D-RPT-1** | `crates/lance-graph-report`: ReportPlan → Quack lowering, fold planner, CellSpace + View, KV/CAM boundary, JSON/CSV export | In PR | `tests/{agnostic,zero_copy,string_fence,reference_workload}.rs` | +| **D-RPT-2** | z8run `crates/z8run-lance`: 8 native nodes, handle envelopes, physical-key cache (AdaWorldAPI/z8run#1) | In PR | z8run-lance `tests/nodes.rs` | +| **D-RPT-3** | OGAR `SlotOutcome::Grid` / `grid_of` / typst `emit_grid` (AdaWorldAPI/OGAR#307) | In PR | ogar-doc-ir resolver grid tests | +| **D-RPT-4** | `crates/lance-graph-report-ogar`: ReportSource (DocObjectSource) over any object source; Live/Revision; emit_typst | In PR | `tests/convergence.rs` | +| **D-RPT-5** | composite-key (multi-lane) group fold primitive in ndarray::simd / mask-risc | Queued | `ISS-REPORT-NO-COMPOSITE-KEY-GROUP-FOLD` | + ## D-WFL — CodeRabbit round 3 on `9fd6956d` (2026-09-19): three findings, all valid, all outside the diff | D-id | correction | diff --git a/.claude/board/entries/2026-09-23-report-plan-zero-copy-pivot-docir-convergence.md b/.claude/board/entries/2026-09-23-report-plan-zero-copy-pivot-docir-convergence.md new file mode 100644 index 000000000..478818374 --- /dev/null +++ b/.claude/board/entries/2026-09-23-report-plan-zero-copy-pivot-docir-convergence.md @@ -0,0 +1,66 @@ +# 2026-09-23 — ReportPlan: fold-first reporting, zero-copy pivot, and reports as OGAR projection sources + +**Status:** MEASURED (tests + allocation falsifiers + benchmark) · OPEN (composite-key +fold; DocRenderer trait; Snapshot pinning; sparse high-cardinality partitions) +**D-ids:** D-RPT-1 (report plan crate), D-RPT-2 (z8run nodes), D-RPT-3 (OGAR grid +projection), D-RPT-4 (report → ObjectSlot adapter) + +## What landed +- `crates/lance-graph-report` (workspace member). `ReportPlan` = Selection + coordinate + providers with roles + mergeable fold states. It LOWERS INTO Quack (`lance()` → + `Program`), and mask-risc's evaluator runs it, so there is no second evaluator and no + independent aggregate semantics. + - `PhysicalKey` excludes roles, axis order and top-k. A rotated plan and its twin share + one fold; rotation is `Arc` + a new `View`. + - Planner choices, each with an inertness test: + - fold key = the widest ordinal coordinate within budget; + - partitions = member conjuncts, evaluated tile-local; a derived `Bucket` is two range + compares and is never a lane; + - dense accumulator vs sparse, where discovery counts per domain and the product is + never allocated; + - selection carrier = validity plane, execution extent, tile-local ops, or ONE reused + mask when the programs × passes count warrants it. + - The execution modules are string-free: `FieldId` / `MaskId` / `SourceId` / ordinals. + A source fence test enforces it. + - Text lives only in `boundary` (KV = the contract's `ContentStore`/`ContentSink`; the + CAM codebook maps (FieldId, ordinal) ↔ ContentId and stores no text), `render` + (JSON/CSV data export) and `explain`. +- `crates/lance-graph-report-ogar` (EXCLUDED; path-deps the OGAR sibling). + - `ReportSource` wraps ANY `DocObjectSource` and answers `grid_of` for report views. + - One `DocCompose` then arranges object A · report · object B by reference and + resolves them in one `resolve_doc`. `emit_typst` renders through OGAR's emitters. + - `Live` = newest result; `Revision(n)` = source generation n, held by `Arc`; + `Snapshot` → the slot's fallback. +- OGAR #307: `SlotOutcome::Grid` / `ResolvedGrid` / `DocObjectSource::grid_of` + (default `None`) plus typst `emit_grid`. This is the one irreducible gap: flat + (label, value) rows cannot carry a two-coordinate object, and u8 rail positions + cannot address report coordinates. +- z8run #1: `z8run-lance`. FlowMessages carry a ~100-byte handle. The emitted plan == + the native plan, and so do the lowered programs. + +## Measured +- 23 report tests + 4 convergence + 3 OGAR resolver + 6 z8run node tests. + Disable-verified: rotation no-op, dropped partitions, grid_of bypass, orientation + ignored, html refusal. +- Execution allocates the SAME bytes at N and 64·N (F8). The reused-mask twin grows by + exactly the mask's words. Rotation allocates < 64 B (view) / < 256 B (plan + reinterpret), with the payload address unchanged. +- Label resolution for a 100k-row 8×12 pivot = 20 CAM resolutions (presented members). + The fold did 0 CAM and 0 KV work. +- `report_bench` (4M rows, avx2 build): + - range→count 91 µs, 0 mask ops; + - 2D pivot 32×12 177 ms (25 scans: 1 reused mask + 12 partitions × 2 states); + - rotation 49 ns / 16 B via the result view, 124 ns / 140 B via plan reinterpret. + +## OPEN +- **Composite-key fold is missing.** A partition costs one pass per member tuple, so a + high-cardinality, densely-observed partition side is refused with `PassBudget`. + Hand-rolling it here would be wrong: the primitive is a multi-lane group fold in + `ndarray::simd` / mask-risc. → `ISS-REPORT-NO-COMPOSITE-KEY-GROUP-FOLD`. +- **OGAR `DocRenderer` is spec-only.** Every consumer, this adapter included, writes a + thin block walker. +- **Snapshot pinning:** there is no content-addressed report store; it resolves to the + fallback. `FieldView::Table` (askama side) is unbuilt. +- `lance-graph-sap` (#1257) already lowers `GroupSumI32` through Quack for CATS. The + two are not reconciled; this crate is the domain-agnostic plan layer. UNKNOWN whether + CATS should consume it. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index d41cf0bf1..acc5300b4 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -151 entries, 2026-08-06 .. 2026-09-23. +152 entries, 2026-08-06 .. 2026-09-23. | date | entry id | finding | file | |---|---|---|---| | 2026-09-23 | `terminal-elects-materialization-range-plane` | | [2026-09-23-terminal-elects-materialization-range-plane.md](2026-09-23-terminal-elects-materialization-range-plane.md) | +| 2026-09-23 | `report-plan-zero-copy-pivot-docir-convergence` | ReportPlan lowers into Quack; pivot = view over shared Arc; reports become OGAR ObjectSlot sources via grid_of (OGAR #307) | [2026-09-23-report-plan-zero-copy-pivot-docir-convergence.md](2026-09-23-report-plan-zero-copy-pivot-docir-convergence.md) | | 2026-09-23 | `quack-having-sym-sum-presence-mask` | | [2026-09-23-quack-having-sym-sum-presence-mask.md](2026-09-23-quack-having-sym-sum-presence-mask.md) | | 2026-09-23 | `cubecl-llvm-boundary-and-audit-regrade` | | [2026-09-23-cubecl-llvm-boundary-and-audit-regrade.md](2026-09-23-cubecl-llvm-boundary-and-audit-regrade.md) | | 2026-09-23 | `absolute-execution-extent` | | [2026-09-23-absolute-execution-extent.md](2026-09-23-absolute-execution-extent.md) | diff --git a/Cargo.toml b/Cargo.toml index f27efe77e..e2fec658c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,9 @@ members = [ "crates/lance-graph-contract", "crates/lance-graph-mask-risc", "crates/lance-graph-quack", + # Z8RUN capstone: the ReportPlan / zero-copy pivot layer Quack and z8run + # both lower into (no second evaluator; lowers INTO lance-graph-quack). + "crates/lance-graph-report", "crates/neural-debug", "crates/lance-graph-callcenter", "crates/lance-graph-archetype", @@ -102,6 +105,11 @@ exclude = [ # half of the OGIT(ontology)/OGAR(ogar) clean separation. Verify via # `cargo test --manifest-path crates/lance-graph-ogar/Cargo.toml`. "crates/lance-graph-ogar", + # A lance-graph ReportResult as an addressed OGAR ObjectSlot source + # (implements ogar-doc-ir's DocObjectSource::grid_of). EXCLUDED for the same + # reason as lance-graph-ogar: path-deps the OGAR sibling. Verify via + # `cargo test --manifest-path crates/lance-graph-report-ogar/Cargo.toml`. + "crates/lance-graph-report-ogar", # Cognitive Compilation loop (claude/cognitive-compilation-lance-graph-h8sgym). # The NEW piece is the Elixir-shaped template — thinking styles, JITson, and # i4-32D thinking-style vectors already exist; the template was the gap. Four diff --git a/crates/lance-graph-report-ogar/Cargo.toml b/crates/lance-graph-report-ogar/Cargo.toml new file mode 100644 index 000000000..86f022e9e --- /dev/null +++ b/crates/lance-graph-report-ogar/Cargo.toml @@ -0,0 +1,42 @@ +# lance-graph-report-ogar — a report result as an ADDRESSED OGAR projection +# source. EXCLUDED from the lance-graph workspace (own [workspace] root below) +# so OGAR never enters the default build — the same shape as lance-graph-ogar. +# +# Direction: this crate depends on BOTH sides; neither side depends on it. +# lance-graph-report never learns OGAR exists, and OGAR never depends on the +# lance-graph engine (only on the zero-dep contract). The seam is OGAR's own +# `DocObjectSource` trait — this crate implements it, nothing more. +# +# OGAR sibling via NO-PIN path deps (`../../../OGAR`), the fleet convention the +# rust-test workflow's OGAR checkout serves (see r2il-mask-abi-probe / symbiont). +# +# ONE contract source: ogar-doc-ir's `classview` feature git-deps the contract; +# the [patch] below folds it onto the in-repo PATH copy, which is what +# lance-graph-report compiles against. Without it `ClassView` would be two traits. +# +# Build/verify: cargo test --manifest-path crates/lance-graph-report-ogar/Cargo.toml + +[package] +name = "lance-graph-report-ogar" +version = "0.1.0" +edition = "2021" +publish = false +description = "A lance-graph ReportResult as an addressed OGAR ObjectSlot source: implements ogar-doc-ir's DocObjectSource (grid_of) over any existing object source, so reports compose, rotate, pin (Live / Revision) and render through OGAR's existing composition + renderer path — no report-specific rendering ontology." +license = "Apache-2.0" + +[workspace] + +[dependencies] +lance-graph-report = { path = "../lance-graph-report" } +lance-graph-contract = { path = "../lance-graph-contract" } +ogar-doc-ir = { path = "../../../OGAR/crates/ogar-doc-ir", features = ["classview"] } +ogar-render-typst = { path = "../../../OGAR/crates/ogar-render-typst" } +# Only for its `FieldView` row type, which the Typst emitter consumes for +# ordinary (non-grid) object slots. +ogar-render-askama = { path = "../../../OGAR/crates/ogar-render-askama" } + +[dev-dependencies] +serde_json = "1.0" + +[patch."https://github.com/AdaWorldAPI/lance-graph"] +lance-graph-contract = { path = "../lance-graph-contract" } diff --git a/crates/lance-graph-report-ogar/src/lib.rs b/crates/lance-graph-report-ogar/src/lib.rs new file mode 100644 index 000000000..570397751 --- /dev/null +++ b/crates/lance-graph-report-ogar/src/lib.rs @@ -0,0 +1,354 @@ +//! # lance-graph-report-ogar — analytical results as addressed projection sources +//! +//! **A report may become a paper. The reporting engine must not become a +//! paper engine.** This crate adds no rendering ontology, no report +//! document, no report template language. It makes a `ReportResult` one +//! more *object* OGAR's existing composition machinery can address: +//! +//! ```text +//! DocCompose ─ ObjectSlot("ogar://lance-graph/report/7@revision:3", view "r.by_c") +//! │ │ +//! │ ogar_doc_ir::resolve_slot ── lookup(mode) ── view_by_name ── class gate +//! │ │ +//! │ DocObjectSource::grid_of ← THIS crate: a view over Arc +//! ▼ ▼ +//! ResolvedDoc{ Slot{ SlotOutcome::Grid } } ──► ogar-render-typst::emit_grid +//! ``` +//! +//! [`ReportSource`] wraps ANY existing [`DocObjectSource`] (the caller's +//! object world) and adds reports beside it, so one `DocCompose` can arrange +//! `projection(A) · projection(report) · projection(B)` and one `resolve_doc` +//! resolves all three. Composition stores addresses and projection choices; +//! nothing about the result is copied into it. +//! +//! **Pivot rotation is a projection change, not document regeneration.** Two +//! slots on the same report through two grid views resolve to the SAME +//! `Arc` re-viewed ([`lance_graph_report::ReportResult::with_roles`]); +//! this crate never holds a batch, so it cannot scan, fold or recompute. +//! +//! **Resolution modes are OGAR's, unchanged.** `Live` is the newest published +//! result; `Revision(n)` is the result computed at source generation `n` +//! (kept by `Arc`, not by copy); `Snapshot(_)` has no content-addressed +//! store here and resolves to the slot's own fallback (ActionText's +//! missing-object path) — no bespoke report-freezing system. +//! +//! **Labels resolve at the boundary.** `grid_of` is called by the resolver on +//! the way to a renderer; it resolves CAM labels for the presented members +//! only, and keeps every axis member's ordinal tuple (`GridAxis::keys`) so a +//! rendered cell traces back to its aggregate coordinate. + +use std::cell::Cell; +use std::collections::{BTreeMap, HashMap}; + +use lance_graph_contract::class_view::{ClassId, WideFieldMask}; +use lance_graph_contract::ontology::DisplayTemplate; +use lance_graph_contract::selection::{NamedView, RailGraph, ViewId, ViewRegistry}; +use lance_graph_report::render::Terminal; +use lance_graph_report::{CellValue, CoordSpec, ReportResult}; +use ogar_doc_ir::compose::{ObjectRef, ResolutionMode}; +use ogar_doc_ir::resolve::{ + DocObjectSource, GridAxis, ResolvedBlock, ResolvedDoc, ResolvedGrid, SlotOutcome, +}; + +/// The `ObjectRef::app` a report is addressed under. +pub const REPORT_APP: &str = "lance-graph"; +/// The `ObjectRef::class` a report is addressed under. +pub const REPORT_OBJECT_CLASS: &str = "report"; + +/// A report address after resolution: which report, and which revision +/// (`None` = live). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ReportKey { + /// Report id (the `ObjectRef::id`, parsed once at lookup). + pub id: u64, + /// Pinned source generation, or `None` for live. + pub revision: Option, +} + +/// A graph key: the wrapped source's own object, or a report. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Key { + /// An object of the wrapped source. + Object(K), + /// A report result. + Report(ReportKey), +} + +/// The wrapped graph: the inner graph plus report nodes (leaves, no rails). +pub struct Graph<'a, G> { + inner: &'a G, + report_class: ClassId, +} + +impl RailGraph for Graph<'_, G> { + type Key = Key; + + fn class_of(&self, key: Self::Key) -> ClassId { + match key { + Key::Object(k) => self.inner.class_of(k), + Key::Report(_) => self.report_class, + } + } + + fn present_mask(&self, key: Self::Key) -> WideFieldMask { + match key { + Key::Object(k) => self.inner.present_mask(k), + Key::Report(_) => WideFieldMask::EMPTY, + } + } + + fn rail_target(&self, key: Self::Key, position: u8) -> Option { + match key { + Key::Object(k) => self.inner.rail_target(k, position).map(Key::Object), + Key::Report(_) => None, + } + } +} + +/// A grid view's orientation: which canonical coordinates play rows and +/// columns. Pure presentation — two orientations of one report share cells. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Orientation { + /// Row coordinates, outermost first. + pub rows: Vec, + /// Column coordinates, outermost first. + pub columns: Vec, +} + +/// Counters that make the zero-work claims checkable. +#[derive(Debug, Default)] +pub struct AdapterStats { + /// Grids projected (each is a re-view + label resolution; never a fold). + pub grids_projected: Cell, +} + +/// Reports beside an existing object source, as one [`DocObjectSource`]. +pub struct ReportSource<'a, S: DocObjectSource> { + inner: &'a S, + graph: Graph<'a, S::Graph>, + /// Report id → results by source generation. Values are `ReportResult`s + /// (an `Arc` to the shared aggregate space + a view), never cell copies. + reports: HashMap>, + views: Vec<(String, ViewId, Orientation)>, + terminal: Terminal<'a>, + /// Counters. + pub stats: AdapterStats, +} + +impl<'a, S: DocObjectSource> ReportSource<'a, S> { + /// Wrap `inner`; reports resolve as class `report_class` (the caller's + /// minted classid — this crate mints nothing), and their labels resolve + /// through `terminal`'s CAM / KV / catalog. + pub fn new(inner: &'a S, report_class: ClassId, terminal: Terminal<'a>) -> Self { + Self { + inner, + graph: Graph { + inner: inner.graph(), + report_class, + }, + reports: HashMap::new(), + views: Vec::new(), + terminal, + stats: AdapterStats::default(), + } + } + + /// Publish a result under a report id. It becomes the live result, and + /// stays addressable as `Revision(generation)` of its source afterwards. + /// Stores an `Arc`-sharing handle; the cells are not copied. + pub fn publish(&mut self, id: u64, result: ReportResult) { + let generation = u64::from(result.space().key().source.generation); + self.reports + .entry(id) + .or_default() + .insert(generation, result); + } + + /// Register a named grid view (a projection choice) in the caller's + /// registry. The name is the membrane spelling a `DocCompose` carries; + /// resolution turns it into the returned `ViewId` once. + pub fn register_grid_view( + &mut self, + registry: &mut ViewRegistry, + name: impl Into, + orientation: Orientation, + ) -> ViewId { + let id = registry.register(NamedView::new( + self.graph.report_class, + WideFieldMask::EMPTY, + DisplayTemplate::Detail, + )); + self.views.push((name.into(), id, orientation)); + id + } + + fn result(&self, k: ReportKey) -> Option<&ReportResult> { + let revs = self.reports.get(&k.id)?; + match k.revision { + None => revs.values().next_back(), + Some(r) => revs.get(&r), + } + } + + fn project(&self, r: &ReportResult) -> ResolvedGrid { + let g = self.terminal.grid(r); + let fmt = |v: &CellValue| match v { + CellValue::Int(i) => i.to_string(), + CellValue::Real(x) => format!("{x}"), + CellValue::Null => String::new(), + }; + let (row_keys, row_labels, cells) = g + .pages + .first() + .map(|(_, rows)| { + let keys = rows.iter().map(|row| row.key.clone()).collect(); + let labels = rows.iter().map(|row| row.labels.clone()).collect(); + let cells = rows + .iter() + .flat_map(|row| row.cells.iter().flatten().map(fmt)) + .collect(); + (keys, labels, cells) + }) + .unwrap_or_default(); + ResolvedGrid { + rows: GridAxis { + keys: row_keys, + labels: row_labels, + }, + columns: GridAxis { + keys: g.column_keys, + labels: g.columns, + }, + measures: g.measures, + cells, + } + } +} + +impl<'a, S: DocObjectSource> DocObjectSource for ReportSource<'a, S> { + type Graph = Graph<'a, S::Graph>; + + fn graph(&self) -> &Self::Graph { + &self.graph + } + + fn lookup( + &self, + target: &ObjectRef, + mode: &ResolutionMode, + ) -> Option<::Key> { + if target.app != REPORT_APP || target.class != REPORT_OBJECT_CLASS { + return self.inner.lookup(target, mode).map(Key::Object); + } + let id: u64 = target.id.parse().ok()?; + let key = match mode { + ResolutionMode::Live => ReportKey { id, revision: None }, + ResolutionMode::Revision(n) => ReportKey { + id, + revision: Some(*n), + }, + // No content-addressed report store: the slot's fallback answers. + ResolutionMode::Snapshot(_) => return None, + }; + self.result(key).map(|_| Key::Report(key)) + } + + fn value_of(&self, key: ::Key, position: u8) -> Option { + match key { + Key::Object(k) => self.inner.value_of(k, position), + Key::Report(_) => None, + } + } + + fn view_by_name(&self, name: &str) -> Option { + self.views + .iter() + .find(|(n, _, _)| n == name) + .map(|(_, id, _)| *id) + .or_else(|| self.inner.view_by_name(name)) + } + + fn view_for_class(&self, class: ClassId) -> Option { + self.inner.view_for_class(class) + } + + fn grid_of(&self, key: ::Key, view: ViewId) -> Option { + match key { + Key::Object(k) => self.inner.grid_of(k, view), + Key::Report(rk) => { + let (_, _, o) = self.views.iter().find(|(_, id, _)| *id == view)?; + // A re-view of the SAME aggregate space: metadata only. + let r = self.result(rk)?.with_roles(&o.rows, &o.columns, &[]).ok()?; + self.stats + .grids_projected + .set(self.stats.grids_projected.get() + 1); + Some(self.project(&r)) + } + } + } +} + +/// Emit a resolved document as Typst source through OGAR's emitters — the +/// thin block walk every OGAR consumer writes today (OGAR's `DocRenderer` +/// trait is still spec-only; this walker adds no layout of its own). +pub fn emit_typst(doc: &ResolvedDoc) -> String { + use ogar_render_askama::FieldView; + use ogar_render_typst as t; + let mut out = String::new(); + for b in &doc.blocks { + match b { + ResolvedBlock::Heading(h) => out.push_str(&t::emit_heading(h)), + ResolvedBlock::Text(x) => out.push_str(&t::emit_text(x)), + ResolvedBlock::Slot(rs) => match &rs.outcome { + SlotOutcome::Resolved { fields, .. } => { + let rows: Vec = fields + .iter() + .map(|f| FieldView { + position: f.position, + label: f.label.clone(), + predicate: String::new(), + value: f.value.clone(), + }) + .collect(); + out.push_str(&t::emit_field_view(&rs.class_view, &rs.uri, &rows)); + } + SlotOutcome::Grid { grid, .. } => { + let nm = grid.measures.len(); + let mut header = vec![String::new()]; + for c in &grid.columns.labels { + for m in &grid.measures { + header.push(if nm == 1 { + c.join("/") + } else { + format!("{} · {m}", c.join("/")) + }); + } + } + let rows: Vec> = grid + .rows + .labels + .iter() + .enumerate() + .map(|(r, l)| { + let mut row = vec![l.join("/")]; + for c in 0..grid.columns.keys.len() { + for m in 0..nm { + row.push(grid.cell(r, c, m).unwrap_or("").to_string()); + } + } + row + }) + .collect(); + out.push_str(&t::emit_grid(&rs.class_view, &rs.uri, &header, &rows)); + } + SlotOutcome::Fallback { content_sha256_hex } => { + out.push_str(&t::emit_fallback(&rs.class_view, content_sha256_hex)); + } + SlotOutcome::Unresolvable => { + out.push_str(&t::emit_text(&format!("unresolvable: {}", rs.uri))); + } + }, + } + } + out +} diff --git a/crates/lance-graph-report-ogar/tests/convergence.rs b/crates/lance-graph-report-ogar/tests/convergence.rs new file mode 100644 index 000000000..b3045182a --- /dev/null +++ b/crates/lance-graph-report-ogar/tests/convergence.rs @@ -0,0 +1,557 @@ +//! The convergence proof: reports, objects and documents compose through ONE +//! OGAR machinery. Synthetic nouns only (A, B, M, objects O1/O2). +//! +//! W3 rotation through composition · W4 mixed composition · W5 recombination +//! by reference · W6 Live vs Revision · W7 terminal render (Typst via OGAR). + +use std::cell::Cell; +use std::sync::Arc; + +use lance_graph_contract::class_view::{ClassId, ClassView, WideFieldMask}; +use lance_graph_contract::content_store::{ContentId, ContentSink, ContentStore}; +use lance_graph_contract::ontology::{DisplayTemplate, FieldRef}; +use lance_graph_contract::selection::{NamedView, RailGraph, ViewId, ViewRegistry}; +use lance_graph_report::boundary::{CamLabels, Catalog, MemKv}; +use lance_graph_report::render::Terminal; +use lance_graph_report::{ + AbiBatch, AxisRole, Column, CoordSpec, FieldId, LaneData, Measure, MeasureKind, PlannerPolicy, + ReportPlan, ReportResult, SourceId, SourceRef, +}; +use lance_graph_report_ogar::{emit_typst, Orientation, ReportSource}; +use ogar_doc_ir::compose::{ + parse_ogar_uri, DocCompose, DocNode, NodeId, ObjectSlot, SnapshotRef, DOC_COMPOSE_VERSION, +}; +use ogar_doc_ir::resolve::{ + resolve_doc, DocObjectSource, ResolvedBlock, ResolvedGrid, SlotOutcome, +}; + +const A: FieldId = FieldId(1); +const B: FieldId = FieldId(2); +const M: FieldId = FieldId(3); +const OBJ: ClassId = 10; +const REPORT: ClassId = 11; + +// ── an existing object world: two objects whose field values live in KV ── + +struct ObjGraph; +impl RailGraph for ObjGraph { + type Key = u32; + fn class_of(&self, _k: u32) -> ClassId { + OBJ + } + fn present_mask(&self, _k: u32) -> WideFieldMask { + WideFieldMask::from(0b11) + } + fn rail_target(&self, _k: u32, _p: u8) -> Option { + None + } +} + +struct Objects<'k> { + graph: ObjGraph, + refs: Vec<[ContentId; 2]>, + kv: &'k MemKv, + view: ViewId, + derefs: Cell, +} +impl DocObjectSource for Objects<'_> { + type Graph = ObjGraph; + fn graph(&self) -> &ObjGraph { + &self.graph + } + fn lookup( + &self, + t: &ogar_doc_ir::compose::ObjectRef, + _m: &ogar_doc_ir::compose::ResolutionMode, + ) -> Option { + let i: u32 = t.id.strip_prefix('o')?.parse().ok()?; + ((i as usize) < self.refs.len() && t.class == "obj").then_some(i) + } + fn value_of(&self, k: u32, p: u8) -> Option { + self.derefs.set(self.derefs.get() + 1); + let id = *self.refs.get(k as usize)?.get(p as usize)?; + self.kv + .resolve(id) + .map(|b| String::from_utf8_lossy(b).into_owned()) + } + fn view_by_name(&self, name: &str) -> Option { + (name == "obj.card").then_some(self.view) + } + fn view_for_class(&self, _c: ClassId) -> Option { + None + } +} + +struct Labels; +impl ClassView for Labels { + fn fields(&self, class: ClassId) -> &[FieldRef] { + use std::sync::OnceLock; + static F: OnceLock> = OnceLock::new(); + if class == OBJ { + F.get_or_init(|| { + vec![ + FieldRef::new("o:title", "title"), + FieldRef::new("o:body", "body"), + ] + }) + } else { + &[] + } + } + fn template(&self, _c: ClassId) -> DisplayTemplate { + DisplayTemplate::Card + } + fn dolce_category_id(&self, _c: ClassId) -> u8 { + 0 + } +} + +// ── an analytical result: fold(M) over coordinates {A, B} ── + +fn report( + cam: &mut CamLabels, + kv: &mut MemKv, + generation: u32, + n: usize, + seed: u64, +) -> ReportResult { + let mut s = seed | 1; + let mut next = move || { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + s + }; + let a_raw: Vec = (0..n).map(|_| format!("a{}", next() % 4)).collect(); + let b_raw: Vec = (0..n).map(|_| format!("b{}", next() % 7)).collect(); + let m: Vec = (0..n).map(|_| (next() % 100) as i32).collect(); + for i in 0..4 { + cam.intern(A, &format!("a{i}"), kv); + } + for i in 0..7 { + cam.intern(B, &format!("b{i}"), kv); + } + let a = cam.canonicalize(A, a_raw.iter().map(String::as_str), kv); + let b = cam.canonicalize(B, b_raw.iter().map(String::as_str), kv); + let batch = AbiBatch::new(SourceId(3), generation, n) + .with_column(Column::coordinate(A, a, 4)) + .unwrap() + .with_column(Column::coordinate(B, b, 7)) + .unwrap() + .with_column(Column::value(M, LaneData::I32(m.into()))) + .unwrap(); + let plan = ReportPlan::over(SourceRef { + id: SourceId(3), + generation, + }) + .axis(CoordSpec::Field(A), AxisRole::Row) + .axis(CoordSpec::Field(B), AxisRole::Column) + .measure(Measure::of(MeasureKind::Sum, M)); + plan.execute(&batch, &PlannerPolicy::default()).unwrap().0 +} + +fn slot(uri: &str, view: &str, fallback: Option) -> DocNode { + let (target, resolution) = parse_ogar_uri(uri).unwrap(); + DocNode::ObjectSlot { + slot: ObjectSlot { + target, + class_view: view.into(), + field_mask: 0, + wide_mask_words: vec![], + resolution, + fallback, + }, + } +} + +/// Document ├ Section "S1" [slot…] … — built from (heading, slot) pairs. +fn compose(sections: Vec<(&str, DocNode)>) -> DocCompose { + let mut nodes = vec![DocNode::Document { children: vec![] }]; + let mut kids = Vec::new(); + for (h, s) in sections { + let hid = nodes.len() as u32; + nodes.push(DocNode::Text { text: h.into() }); + let sid = nodes.len() as u32; + nodes.push(s); + let sec = nodes.len() as u32; + nodes.push(DocNode::Section { + heading: Some(NodeId(hid)), + children: vec![NodeId(sid)], + }); + kids.push(NodeId(sec)); + } + nodes[0] = DocNode::Document { children: kids }; + DocCompose { + version: DOC_COMPOSE_VERSION.into(), + nodes, + root: NodeId(0), + } +} + +fn grids(doc: &ogar_doc_ir::resolve::ResolvedDoc) -> Vec<&ResolvedGrid> { + doc.blocks + .iter() + .filter_map(|b| match b { + ResolvedBlock::Slot(s) => match &s.outcome { + SlotOutcome::Grid { grid, .. } => Some(grid), + _ => None, + }, + _ => None, + }) + .collect() +} + +struct World { + cam: CamLabels, + kv: MemKv, + cat: Catalog, + refs: Vec<[ContentId; 2]>, +} + +fn world() -> World { + let (cam, mut kv) = (CamLabels::default(), MemKv::default()); + // Two existing objects; their bodies live in KV, addressed by ContentId. + let refs = vec![ + [kv.put_str("First object"), kv.put_str("body of O0")], + [kv.put_str("Second object"), kv.put_str("body of O1")], + ]; + World { + cam, + kv, + cat: Catalog::default().with("M", M), + refs, + } +} + +#[test] +fn rotation_through_composition_rescans_nothing_and_shares_the_cells() { + let mut w = world(); + let r = report(&mut w.cam, &mut w.kv, 1, 20_000, 7); + let payload = r.space().payload_addr(); + let mut reg = ViewRegistry::new(); + let obj_view = reg.register(NamedView::new( + OBJ, + WideFieldMask::from(0b11), + DisplayTemplate::Card, + )); + let objects = Objects { + graph: ObjGraph, + refs: w.refs.clone(), + kv: &w.kv, + view: obj_view, + derefs: Cell::new(0), + }; + let t = Terminal { + cam: &w.cam, + kv: &w.kv, + catalog: &w.cat, + }; + let mut src = ReportSource::new(&objects, REPORT, t); + src.publish(7, r.clone()); + src.register_grid_view( + &mut reg, + "r.a_by_b", + Orientation { + rows: vec![CoordSpec::Field(A)], + columns: vec![CoordSpec::Field(B)], + }, + ); + src.register_grid_view( + &mut reg, + "r.b_by_a", + Orientation { + rows: vec![CoordSpec::Field(B)], + columns: vec![CoordSpec::Field(A)], + }, + ); + + let doc = compose(vec![ + ( + "View 1", + slot("ogar://lance-graph/report/7@live", "r.a_by_b", None), + ), + ( + "View 2", + slot("ogar://lance-graph/report/7@live", "r.b_by_a", None), + ), + ]); + let resolved = resolve_doc(&doc, &src, &Labels, ®, 4).unwrap(); + let g = grids(&resolved); + assert_eq!(g.len(), 2); + let (v1, v2) = (g[0], g[1]); + assert_eq!((v1.rows.keys.len(), v1.columns.keys.len()), (4, 7)); + assert_eq!((v2.rows.keys.len(), v2.columns.keys.len()), (7, 4)); + for a in 0..4 { + for b in 0..7 { + assert_eq!( + v1.cell(a, b, 0), + v2.cell(b, a, 0), + "same cell, rotated view" + ); + } + } + // Identity survives: labels are CAM text, keys are the ordinals. + assert_eq!(v1.rows.labels[2], vec!["a2".to_string()]); + assert_eq!(v1.rows.keys[2], vec![2]); + // Zero population work: the source holds no batch, the aggregate payload + // is the one computed above, and projections are counted. + assert_eq!(r.space().payload_addr(), payload); + assert_eq!(src.stats.grids_projected.get(), 2); + assert_eq!( + Arc::strong_count(r.space()), + 2, + "held by `r` and the published entry only" + ); +} + +#[test] +fn mixed_recombination_stores_references_and_resolves_only_at_render() { + let mut w = world(); + let small = report(&mut w.cam, &mut w.kv, 1, 1_000, 8); + let big = report(&mut w.cam, &mut w.kv, 1, 200_000, 9); + let mut reg = ViewRegistry::new(); + let obj_view = reg.register(NamedView::new( + OBJ, + WideFieldMask::from(0b11), + DisplayTemplate::Card, + )); + let objects = Objects { + graph: ObjGraph, + refs: w.refs.clone(), + kv: &w.kv, + view: obj_view, + derefs: Cell::new(0), + }; + + // Document ├ projection(O0) ├ projection(R) └ projection(O1) + let doc_for = |id: u64| { + compose(vec![ + ("Object A", slot("ogar://app/obj/o0@live", "obj.card", None)), + ( + "Analysis", + slot( + &format!("ogar://lance-graph/report/{id}@live"), + "r.a_by_b", + None, + ), + ), + ("Object B", slot("ogar://app/obj/o1@live", "obj.card", None)), + ]) + }; + // The composition is references + projection choices: its bytes do not + // depend on how many rows fed the report, nor on the objects' bodies. + let (d1, d2) = ( + serde_json::to_string(&doc_for(1)).unwrap(), + serde_json::to_string(&doc_for(2)).unwrap(), + ); + assert_eq!(d1.len(), d2.len()); + assert!( + !d1.contains("body of") && !d1.contains("a0"), + "no object or report content copied in" + ); + + let kv_before = + w.kv.counters + .kv_derefs + .load(std::sync::atomic::Ordering::Relaxed); + let t = Terminal { + cam: &w.cam, + kv: &w.kv, + catalog: &w.cat, + }; + let mut src = ReportSource::new(&objects, REPORT, t); + src.publish(1, small); + src.publish(2, big.clone()); + src.register_grid_view( + &mut reg, + "r.a_by_b", + Orientation { + rows: vec![CoordSpec::Field(A)], + columns: vec![CoordSpec::Field(B)], + }, + ); + assert_eq!( + objects.derefs.get(), + 0, + "building the composition dereferenced nothing" + ); + assert_eq!( + w.kv.counters + .kv_derefs + .load(std::sync::atomic::Ordering::Relaxed), + kv_before + ); + + let resolved = resolve_doc(&doc_for(2), &src, &Labels, ®, 4).unwrap(); + let kinds: Vec<&str> = resolved + .blocks + .iter() + .filter_map(|b| match b { + ResolvedBlock::Slot(s) => Some(match s.outcome { + SlotOutcome::Resolved { .. } => "object", + SlotOutcome::Grid { .. } => "grid", + _ => "other", + }), + _ => None, + }) + .collect(); + assert_eq!(kinds, ["object", "grid", "object"]); + // Resolution dereferenced exactly the two objects' two fields, and the + // report's labels: 4 row + 7 column members (+ one KV read each). + assert_eq!(objects.derefs.get(), 4); + let kv_after = + w.kv.counters + .kv_derefs + .load(std::sync::atomic::Ordering::Relaxed); + assert_eq!(kv_after - kv_before, 4 + 11); + + // W7: one OGAR emitter renders all three kinds. + let typ = emit_typst(&resolved); + assert!(typ.contains("First object") && typ.contains("Second object")); + assert!(typ.contains("columns: 8"), "row header + 7 columns"); + assert!(typ.contains("= Analysis")); +} + +#[test] +fn live_and_revision_resolve_through_ogar_resolution_modes() { + let mut w = world(); + let g1 = report(&mut w.cam, &mut w.kv, 1, 5_000, 11); + let g2 = report(&mut w.cam, &mut w.kv, 2, 5_000, 12); // the source moved on + let mut reg = ViewRegistry::new(); + let obj_view = reg.register(NamedView::new( + OBJ, + WideFieldMask::from(0b11), + DisplayTemplate::Card, + )); + let objects = Objects { + graph: ObjGraph, + refs: w.refs.clone(), + kv: &w.kv, + view: obj_view, + derefs: Cell::new(0), + }; + let t = Terminal { + cam: &w.cam, + kv: &w.kv, + catalog: &w.cat, + }; + let mut src = ReportSource::new(&objects, REPORT, t); + src.publish(5, g1.clone()); + src.publish(5, g2.clone()); + src.register_grid_view( + &mut reg, + "r.a_by_b", + Orientation { + rows: vec![CoordSpec::Field(A)], + columns: vec![CoordSpec::Field(B)], + }, + ); + + let doc = compose(vec![ + ( + "Live", + slot("ogar://lance-graph/report/5@live", "r.a_by_b", None), + ), + ( + "Pinned", + slot("ogar://lance-graph/report/5@revision:1", "r.a_by_b", None), + ), + ( + "Gone", + slot("ogar://lance-graph/report/5@revision:9", "r.a_by_b", None), + ), + ( + "Snapshot", + slot( + &format!("ogar://lance-graph/report/5@sha256:{}", "ab".repeat(32)), + "r.a_by_b", + Some(SnapshotRef { + content_sha256: [0xAB; 32], + }), + ), + ), + ]); + let resolved = resolve_doc(&doc, &src, &Labels, ®, 4).unwrap(); + let g = grids(&resolved); + assert_eq!(g.len(), 2); + assert_ne!( + g[0].cells, g[1].cells, + "live and pinned are different results" + ); + let m = &g1.measures()[0]; + let pinned_cell = match g1.value(m, &[], &[0], &[0]) { + lance_graph_report::CellValue::Int(i) => i.to_string(), + _ => String::new(), + }; + assert_eq!( + g[1].cell(0, 0, 0), + Some(pinned_cell.as_str()), + "revision:1 is the generation-1 result" + ); + let outcomes: Vec<&SlotOutcome> = resolved + .blocks + .iter() + .filter_map(|b| match b { + ResolvedBlock::Slot(s) => Some(&s.outcome), + _ => None, + }) + .collect(); + assert_eq!( + *outcomes[2], + SlotOutcome::Unresolvable, + "an unknown revision fails closed" + ); + assert!( + matches!(outcomes[3], SlotOutcome::Fallback { .. }), + "snapshot answers from the slot's fallback" + ); +} + +#[test] +fn a_label_rename_changes_the_render_not_the_aggregate() { + let mut w = world(); + let r = report(&mut w.cam, &mut w.kv, 1, 5_000, 13); + let payload = r.space().payload_addr(); + assert!(w.cam.rename(A, 0, "renamed-a0", &mut w.kv)); + let mut reg = ViewRegistry::new(); + let obj_view = reg.register(NamedView::new( + OBJ, + WideFieldMask::from(0b11), + DisplayTemplate::Card, + )); + let objects = Objects { + graph: ObjGraph, + refs: w.refs.clone(), + kv: &w.kv, + view: obj_view, + derefs: Cell::new(0), + }; + let t = Terminal { + cam: &w.cam, + kv: &w.kv, + catalog: &w.cat, + }; + let mut src = ReportSource::new(&objects, REPORT, t); + src.publish(1, r.clone()); + src.register_grid_view( + &mut reg, + "r.a_by_b", + Orientation { + rows: vec![CoordSpec::Field(A)], + columns: vec![CoordSpec::Field(B)], + }, + ); + let doc = compose(vec![( + "R", + slot("ogar://lance-graph/report/1@live", "r.a_by_b", None), + )]); + let resolved = resolve_doc(&doc, &src, &Labels, ®, 4).unwrap(); + let g = grids(&resolved); + assert_eq!(g[0].rows.labels[0], vec!["renamed-a0".to_string()]); + assert_eq!( + g[0].rows.keys[0], + vec![0], + "identity is the ordinal, unchanged" + ); + assert_eq!(r.space().payload_addr(), payload); +} diff --git a/crates/lance-graph-report/Cargo.toml b/crates/lance-graph-report/Cargo.toml new file mode 100644 index 000000000..f2e4cd95a --- /dev/null +++ b/crates/lance-graph-report/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "lance-graph-report" +version = "0.1.0" +edition = "2021" +publish = false +description = "The reporting plan both Quack and z8run lower into. A ReportPlan is control plane (selection, axes, measures, a coordinate view); it lowers into lance-graph-quack's Query and is executed by lance-graph-mask-risc's one evaluator over resident SoA lanes. Pivot is a change of coordinate system, never a transpose: never transpose data to pivot — transpose meaning." + +# Same two crates Quack itself stands on, and nothing else: this crate LOWERS +# into Quack (so the two can never grow independent aggregate semantics — F10) +# and runs the resulting `Program`s on scratch it owns. No serde, no ndarray; the zero-dep contract only for its KV traits: +# rendering at the materialization boundary is plain `String` writing, and the +# masking algebra is reached THROUGH mask-risc, never beside it. +[dependencies] +lance-graph-quack = { path = "../lance-graph-quack" } +lance-graph-mask-risc = { path = "../lance-graph-mask-risc" } +# ContentId / ContentStore / ContentSink: the KV ownership surface (zero-dep). +lance-graph-contract = { path = "../lance-graph-contract" } + +[dev-dependencies] diff --git a/crates/lance-graph-report/examples/report_bench.rs b/crates/lance-graph-report/examples/report_bench.rs new file mode 100644 index 000000000..1cda77a1c --- /dev/null +++ b/crates/lance-graph-report/examples/report_bench.rs @@ -0,0 +1,142 @@ +//! Focused reporting benchmarks (not a dashboard). Run: +//! +//! ```sh +//! cargo run --release -p lance-graph-report --example report_bench +//! ``` +//! +//! Prints the SIMD tier the binary was built for, because a timing without +//! its target is an anecdote. Every row reports bytes allocated during the +//! measured call (a counting allocator), which is the number the zero-copy +//! claims rest on; the timings are context. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Instant; + +use lance_graph_report::boundary::{CamLabels, Catalog, MemKv}; +use lance_graph_report::render::Terminal; +use lance_graph_report::*; + +struct Counting; +static BYTES: AtomicU64 = AtomicU64::new(0); +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + BYTES.fetch_add(l.size() as u64, Ordering::Relaxed); + // SAFETY: forwarded verbatim to the system allocator. + unsafe { System.alloc(l) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + // SAFETY: forwarded verbatim to the system allocator. + unsafe { System.dealloc(p, l) } + } +} +#[global_allocator] +static A: Counting = Counting; + +fn bench(name: &str, reps: u32, mut f: impl FnMut() -> T) -> T { + let mut out = f(); + let b0 = BYTES.load(Ordering::Relaxed); + let t = Instant::now(); + for _ in 0..reps { + out = f(); + } + let ns = t.elapsed().as_nanos() as f64 / f64::from(reps); + let bytes = (BYTES.load(Ordering::Relaxed) - b0) / u64::from(reps); + println!("{name:<44} {:>12.0} ns {:>10} B alloc/call", ns, bytes); + out +} + +fn main() { + let n: usize = 4 << 20; + let (fa, fb, fv) = (FieldId(0), FieldId(1), FieldId(2)); + let mut s = 0x9E37_79B9_7F4A_7C15u64; + let mut next = || { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + s + }; + let a: Arc<[u32]> = (0..n).map(|_| (next() % 32) as u32).collect(); + let b: Arc<[u32]> = (0..n).map(|_| (next() % 12) as u32).collect(); + let v: Arc<[i32]> = (0..n).map(|_| (next() % 1000) as i32).collect(); + let focus: Arc<[u64]> = (0..n.div_ceil(64)).map(|_| next() & next()).collect(); + let batch = AbiBatch::new(SourceId(1), 1, n) + .with_column(Column::coordinate(fa, a, 32)) + .unwrap() + .with_column(Column::coordinate(fb, b, 12)) + .unwrap() + .with_column(Column::value(fv, LaneData::I32(v))) + .unwrap() + .with_mask(MaskId(1), focus) + .unwrap(); + println!( + "rows {n} avx512f={} avx2={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx2") + ); + let pol = PlannerPolicy::default(); + let src = SourceRef { + id: SourceId(1), + generation: 1, + }; + let range = Selection::Range(RowRange { + lo: 1000, + hi: (n - 1000) as u32, + }); + let base = ReportPlan::over(src); + let sum = Measure::of(MeasureKind::Sum, fv); + + let p1 = base.clone().filter(range.clone()).measure(Measure::count()); + bench("1 range → count", 20, || { + p1.execute(&batch, &pol).unwrap() + }); + let p2 = base.clone().filter(range.clone()).measure(sum.clone()); + bench("2 range → sum", 20, || p2.execute(&batch, &pol).unwrap()); + let p3 = base + .clone() + .filter(Selection::Mask(MaskId(1))) + .measure(sum.clone()); + bench("3 resident mask → sum", 20, || { + p3.execute(&batch, &pol).unwrap() + }); + let p4 = base + .clone() + .filter(Selection::Mask(MaskId(1))) + .filter(range) + .measure(sum.clone()); + bench("4 mask ∧ range → sum", 20, || { + p4.execute(&batch, &pol).unwrap() + }); + let p5 = base + .clone() + .axis(CoordSpec::Field(fa), AxisRole::Row) + .measure(sum.clone()); + bench("5 grouped sum (32 groups)", 20, || { + p5.execute(&batch, &pol).unwrap() + }); + let p6 = base + .clone() + .pivot(&[CoordSpec::Field(fa)], &[CoordSpec::Field(fb)]) + .measure(sum.clone()) + .filter(Selection::cmp(fv, CmpOp::Ge, Scalar::Int(100))); + let (res, st) = bench("6 2D pivot fold 32×12, filtered", 5, || { + p6.execute(&batch, &pol).unwrap() + }); + println!( + " passes/scans {} mask materializations {} accumulator {} B", + st.population_scans, st.mask_materializations, st.accumulator_bytes + ); + let rp = p6.clone().rotate(); + bench("7a pivot rotation (result view)", 100_000, || res.rotate()); + bench("7b pivot rotation (plan reinterpret)", 100_000, || { + res.reinterpret(&rp).unwrap() + }); + let (cam, kv, cat) = (CamLabels::default(), MemKv::default(), Catalog::default()); + let t = Terminal { + cam: &cam, + kv: &kv, + catalog: &cat, + }; + bench("8 JSON materialization (384 cells)", 1000, || t.json(&res)); +} diff --git a/crates/lance-graph-report/src/batch.rs b/crates/lance-graph-report/src/batch.rs new file mode 100644 index 000000000..1670c2741 --- /dev/null +++ b/crates/lance-graph-report/src/batch.rs @@ -0,0 +1,241 @@ +//! The resident source a report runs over: fixed-width SoA lanes the host +//! already owns. +//! +//! An [`AbiBatch`] is built ONCE, downstream of canonicalization (raw values +//! land in the KV first; see [`crate::boundary`]), and is then only ever +//! BORROWED: every report hands `&[u32]` / `&[i32]` / `&[u64]` views to the +//! evaluator. Nothing here copies a lane, gathers rows, or builds a per-row +//! object; cloning an `Arc` clones a pointer. +//! +//! **String-free by construction.** A field is a [`FieldId`], a coordinate is a +//! `u32` ordinal in a declared domain, a raw variable-size value is a `u64` +//! [`ContentId`](lance_graph_contract::content_store::ContentId) KV reference. +//! Names and labels are not stored here at all — they live in the catalog and +//! the CAM label store at the boundary. + +use std::sync::Arc; + +use lance_graph_mask_risc::{words_for, LaneRef}; + +use crate::ids::{FieldId, MaskId, SourceId}; + +/// One resident fixed-width lane. +#[derive(Debug, Clone)] +pub enum LaneData { + /// Signed 32-bit values (measures, ordered compares, canonical integer time). + I32(Arc<[i32]>), + /// Unsigned 32-bit values (coordinate ordinals / label ids, codes, keys). + U32(Arc<[u32]>), + /// 64-bit references (KV content addresses, GUIDs). Never folded; carried + /// so selected rows can be dereferenced at the terminal boundary. + U64(Arc<[u64]>), +} + +impl LaneData { + fn len(&self) -> usize { + match self { + LaneData::I32(v) => v.len(), + LaneData::U32(v) => v.len(), + LaneData::U64(v) => v.len(), + } + } + + fn as_ref(&self) -> LaneRef<'_> { + match self { + LaneData::I32(v) => LaneRef::I32(v), + LaneData::U32(v) => LaneRef::U32(v), + LaneData::U64(v) => LaneRef::U64(v), + } + } + + fn addr(&self) -> usize { + match self { + LaneData::I32(v) => v.as_ptr() as usize, + LaneData::U32(v) => v.as_ptr() as usize, + LaneData::U64(v) => v.as_ptr() as usize, + } + } +} + +/// A resident field. It knows its lane and, if it is a coordinate, how many +/// ordinals its domain has. It never knows what it means or what it is called. +#[derive(Debug, Clone)] +pub struct Column { + /// Canonical field identity. + pub field: FieldId, + /// The resident lane. + pub lane: LaneData, + /// Domain size for a `U32` coordinate field (ordinals `0..domain`). An + /// ordinal at or past it lies outside the domain and belongs to no cell. + pub domain: Option, +} + +impl Column { + /// A plain value lane (measure source, filter field, KV reference). + pub fn value(field: FieldId, lane: LaneData) -> Self { + Self { + field, + lane, + domain: None, + } + } + + /// A `U32` coordinate field over `domain` ordinals. + pub fn coordinate(field: FieldId, ordinals: Arc<[u32]>, domain: u32) -> Self { + Self { + field, + lane: LaneData::U32(ordinals), + domain: Some(domain), + } + } +} + +/// Errors building a batch. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BatchError { + /// A lane's length differs from the batch's row count. + LenMismatch { + /// The offending field (`None` for a mask). + field: Option, + /// The batch row count (or word count for a mask). + expected: usize, + /// What the lane holds. + found: usize, + }, + /// A domain was declared on a non-`U32` lane. + DomainOnNonOrdinalLane(FieldId), + /// Two columns share a field id. + DuplicateField(FieldId), + /// Two masks share an id, or a mask claimed [`MaskId::ALPHA`]. + DuplicateMask(MaskId), +} + +/// The resident population a report is evaluated over. +/// +/// Plane 0 is always the validity plane ([`MaskId::ALPHA`]): the table IS its +/// validity plane, so "every row" lowers to zero ops. +#[derive(Debug, Clone)] +pub struct AbiBatch { + source: SourceId, + generation: u32, + n_rows: usize, + columns: Vec, + masks: Vec<(MaskId, Arc<[u64]>)>, +} + +impl AbiBatch { + /// A batch over `n_rows` rows whose validity plane is all-set. + pub fn new(source: SourceId, generation: u32, n_rows: usize) -> Self { + let mut alpha = vec![u64::MAX; words_for(n_rows)]; + let tail = n_rows % 64; + if tail != 0 { + if let Some(last) = alpha.last_mut() { + *last = (1u64 << tail) - 1; + } + } + Self { + source, + generation, + n_rows, + columns: Vec::new(), + masks: vec![(MaskId::ALPHA, alpha.into())], + } + } + + /// Attach a column. The lane is moved in (an `Arc`), never copied. + pub fn with_column(mut self, column: Column) -> Result { + if column.lane.len() != self.n_rows { + return Err(BatchError::LenMismatch { + field: Some(column.field), + expected: self.n_rows, + found: column.lane.len(), + }); + } + if column.domain.is_some() && !matches!(column.lane, LaneData::U32(_)) { + return Err(BatchError::DomainOnNonOrdinalLane(column.field)); + } + if self.columns.iter().any(|c| c.field == column.field) { + return Err(BatchError::DuplicateField(column.field)); + } + self.columns.push(column); + Ok(self) + } + + /// Attach a RESIDENT mask (a cached facet mask, a focus, a class mask) + /// supplied by the owner. A report never produces one of these for + /// another report to consume. + pub fn with_mask(mut self, id: MaskId, words: Arc<[u64]>) -> Result { + if words.len() != words_for(self.n_rows) { + return Err(BatchError::LenMismatch { + field: None, + expected: words_for(self.n_rows), + found: words.len(), + }); + } + if self.masks.iter().any(|(m, _)| *m == id) { + return Err(BatchError::DuplicateMask(id)); + } + self.masks.push((id, words)); + Ok(self) + } + + /// The source this batch was published as. + pub fn source(&self) -> SourceId { + self.source + } + + /// Publication generation (a republished source bumps it; a plan minted + /// against an older generation fails closed). + pub fn generation(&self) -> u32 { + self.generation + } + + /// Row count. + pub fn n_rows(&self) -> usize { + self.n_rows + } + + /// The columns, in lane order (column `i` is lane `i`). + pub fn columns(&self) -> &[Column] { + &self.columns + } + + /// Lane index + column of a field. + pub fn column(&self, field: FieldId) -> Option<(u16, &Column)> { + self.columns + .iter() + .enumerate() + .find(|(_, c)| c.field == field) + .map(|(i, c)| (i as u16, c)) + } + + /// Plane index of a resident mask. + pub fn plane_of(&self, id: MaskId) -> Option { + self.masks + .iter() + .position(|(m, _)| *m == id) + .map(|i| i as u16) + } + + /// Borrowed plane views for the evaluator: O(columns + masks) pointers. + pub(crate) fn views(&self) -> (Vec<&[u64]>, Vec>) { + ( + self.masks.iter().map(|(_, w)| &w[..]).collect(), + self.columns.iter().map(|c| c.lane.as_ref()).collect(), + ) + } + + /// Address of a field's first element — the identity the zero-copy + /// falsifiers compare before and after a report runs. + pub fn lane_addr(&self, field: FieldId) -> Option { + self.column(field).map(|(_, c)| c.lane.addr()) + } + + /// The `U64` reference lane of a field, for terminal dereference. + pub fn refs(&self, field: FieldId) -> Option<&[u64]> { + match self.column(field).map(|(_, c)| &c.lane) { + Some(LaneData::U64(v)) => Some(v), + _ => None, + } + } +} diff --git a/crates/lance-graph-report/src/boundary.rs b/crates/lance-graph-report/src/boundary.rs new file mode 100644 index 000000000..e00074cf6 --- /dev/null +++ b/crates/lance-graph-report/src/boundary.rs @@ -0,0 +1,215 @@ +//! The boundary — the ONLY module of this crate that accepts text. +//! +//! ```text +//! external / raw input +//! ↓ parse / normalize / canonicalize (here) +//! KV (ContentId → bytes) CAM ((FieldId, ordinal) ↔ ContentId) +//! ↓ ↓ +//! fixed-width refs / ordinals / numeric lanes → AbiBatch → plan → fold +//! ↓ +//! aggregate coordinates → terminal → CAM → KV → human-readable text +//! ``` +//! +//! **Human-readable identity belongs in CAM, not in the fold substrate.** +//! +//! * [`MemKv`] owns raw facts: it implements the contract's content-addressed +//! [`ContentStore`] / [`ContentSink`] (the workspace's existing KV +//! ownership surface — not a new trait). Raw variable-size values never +//! enter a lane; a `U64` lane carries their [`ContentId`]. +//! * [`CamLabels`] is a codebook per field: ordinal ↔ `ContentId` of the +//! label text. It stores NO text and hashes no text at scan time — lookups +//! key on the content address. Renaming an ordinal re-points it at new +//! bytes; the ordinal, and so every coordinate, mask, fold and cached +//! aggregate that uses it, is untouched (S5, S12). +//! * [`Catalog`] resolves a field NAME to a [`FieldId`] once, when an adapter +//! (z8run config, a SQL front-end) builds a plan. A plan never holds the +//! name (S4). +//! +//! Every lookup, insertion, persist and dereference is counted, so a test can +//! prove the fold did none of them and the terminal did only result-sized work. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use lance_graph_contract::content_store::{ContentId, ContentSink, ContentStore}; + +use crate::ids::FieldId; + +/// Boundary counters (relaxed atomics: diagnostics, not synchronization). +#[derive(Debug, Default)] +pub struct BoundaryCounters { + /// Raw values persisted to KV. + pub kv_puts: AtomicU64, + /// KV dereferences (all happen at a terminal adapter; the fold has no KV). + pub kv_derefs: AtomicU64, + /// CAM ordinal lookups (label → ordinal). + pub cam_lookups: AtomicU64, + /// CAM insertions (a new label minted an ordinal). + pub cam_insertions: AtomicU64, + /// CAM resolutions (ordinal → label), terminal only. + pub cam_resolutions: AtomicU64, +} + +fn bump(c: &AtomicU64) { + c.fetch_add(1, Ordering::Relaxed); +} + +impl BoundaryCounters { + /// Read one counter. + pub fn get(c: &AtomicU64) -> u64 { + c.load(Ordering::Relaxed) + } +} + +/// An in-memory KV over the contract's content-addressed store traits. +#[derive(Debug, Default)] +pub struct MemKv { + map: HashMap>, + /// Counters. + pub counters: BoundaryCounters, +} + +impl ContentStore for MemKv { + fn resolve(&self, id: ContentId) -> Option<&[u8]> { + bump(&self.counters.kv_derefs); + self.map.get(&id).map(|b| &b[..]) + } +} + +impl ContentSink for MemKv { + fn put(&mut self, bytes: &[u8]) -> ContentId { + bump(&self.counters.kv_puts); + let id = ContentId::of(bytes); + self.map.entry(id).or_insert_with(|| bytes.into()); + id + } +} + +impl MemKv { + /// Persist raw values; the lane that enters the batch is their addresses. + pub fn persist<'a>(&mut self, raw: impl IntoIterator) -> Arc<[u64]> { + raw.into_iter().map(|b| self.put(b).0).collect() + } + + /// Dereference one reference as text (terminal use). + pub fn text(&self, id: ContentId) -> Option<&str> { + self.resolve(id).and_then(|b| std::str::from_utf8(b).ok()) + } +} + +/// The CAM label codebook: per field, ordinal ↔ content address of its label. +#[derive(Debug, Default)] +pub struct CamLabels { + fields: HashMap, + /// Counters. + pub counters: BoundaryCounters, +} + +#[derive(Debug, Default)] +struct Codebook { + by_ordinal: Vec, + by_content: HashMap, +} + +impl CamLabels { + /// Canonicalize one textual value of `field` to its ordinal, minting one + /// (and persisting the label text in `kv`) if it is new. Ingest only. + pub fn intern(&mut self, field: FieldId, label: &str, kv: &mut MemKv) -> u32 { + bump(&self.counters.cam_lookups); + let id = ContentId::of_str(label); + let book = self.fields.entry(field).or_default(); + if let Some(&o) = book.by_content.get(&id) { + return o; + } + bump(&self.counters.cam_insertions); + kv.put_str(label); + let o = book.by_ordinal.len() as u32; + book.by_ordinal.push(id); + book.by_content.insert(id, o); + o + } + + /// Canonicalize a whole categorical column at ingest: text in, an + /// ordinal lane out. The text does not survive past this call. + pub fn canonicalize<'a>( + &mut self, + field: FieldId, + raw: impl IntoIterator, + kv: &mut MemKv, + ) -> Arc<[u32]> { + raw.into_iter().map(|s| self.intern(field, s, kv)).collect() + } + + /// Look up an existing label's ordinal (adapter use: resolving a filter + /// literal before it enters a plan). Never mints. + pub fn ordinal(&self, field: FieldId, label: &str) -> Option { + bump(&self.counters.cam_lookups); + self.fields + .get(&field)? + .by_content + .get(&ContentId::of_str(label)) + .copied() + } + + /// The domain size of a field's codebook (ordinals `0..domain`). + pub fn domain(&self, field: FieldId) -> u32 { + self.fields + .get(&field) + .map_or(0, |b| b.by_ordinal.len() as u32) + } + + /// Resolve an ordinal to its label text — the terminal boundary only. + pub fn resolve<'k>(&self, field: FieldId, ordinal: u32, kv: &'k MemKv) -> Option<&'k str> { + bump(&self.counters.cam_resolutions); + let id = *self.fields.get(&field)?.by_ordinal.get(ordinal as usize)?; + kv.text(id) + } + + /// Re-label an ordinal. The ordinal — the identity every coordinate, + /// mask, fold and cache uses — does not change. + pub fn rename(&mut self, field: FieldId, ordinal: u32, label: &str, kv: &mut MemKv) -> bool { + let Some(book) = self.fields.get_mut(&field) else { + return false; + }; + let Some(slot) = book.by_ordinal.get_mut(ordinal as usize) else { + return false; + }; + let new = kv.put_str(label); + book.by_content.remove(slot); + *slot = new; + book.by_content.insert(new, ordinal); + true + } +} + +/// Field-name resolution for adapters: a name resolves to a [`FieldId`] once, +/// before a plan is built; the plan never carries it. +#[derive(Debug, Default, Clone)] +pub struct Catalog { + names: Vec<(String, FieldId)>, +} + +impl Catalog { + /// Register a field name. + pub fn with(mut self, name: impl Into, id: FieldId) -> Self { + self.names.push((name.into(), id)); + self + } + + /// Resolve a name. + pub fn field(&self, name: &str) -> Option { + self.names + .iter() + .find(|(n, _)| n == name) + .map(|(_, id)| *id) + } + + /// The display name of a field (terminal headers, diagnostics). + pub fn name(&self, id: FieldId) -> Option<&str> { + self.names + .iter() + .find(|(_, i)| *i == id) + .map(|(n, _)| n.as_str()) + } +} diff --git a/crates/lance-graph-report/src/exec.rs b/crates/lance-graph-report/src/exec.rs new file mode 100644 index 000000000..7a385f28c --- /dev/null +++ b/crates/lance-graph-report/src/exec.rs @@ -0,0 +1,807 @@ +//! Physical planning and the fold. +//! +//! ```text +//! ReportPlan (control plane, ids only) +//! │ plan_physical — structural; reads no row +//! ▼ +//! PhysicalPlan (selection carrier · coordinate providers · fold key · +//! │ accumulator strategy · pass count · lowered programs) +//! │ run — the only population work +//! ▼ +//! CellSpace (Arc) ── View ──▶ ReportResult ──▶ terminal adapter +//! ``` +//! +//! **Pivot defines coordinates. Fold populates coordinates. Materialization +//! renders coordinates.** Nothing in this file knows what a field means; it +//! knows lanes, ordinals, domains, selections and fold states. +//! +//! Every program executed here is a Quack lowering ([`lance_graph_quack::lower`]) +//! run by mask-risc's one evaluator — this crate has no aggregate semantics +//! of its own (F10) and no row loop (F2): its loops run over PASSES and over +//! result cells, never over rows. +//! +//! # The physical choices, and what drives each +//! +//! * **Fold key** — one ordinal coordinate the substrate scatters into +//! directly (`GroupSumI32` / `GroupReduce`). Chosen as the widest +//! field-backed coordinate whose domain fits the domain-buffer budget. +//! * **Partitions** — every other coordinate. A partition member is a +//! selection conjunct (`field = m`, or a derived bucket's two range +//! compares) evaluated tile by tile during the pass; nothing is +//! materialized for it (A6). One pass per partition-member tuple. +//! * **Accumulator** — dense when the coordinate product fits the dense-cell +//! budget; otherwise sparse, with partition members DISCOVERED first by +//! one domain-sized count pass each, so the product is never allocated (A7). +//! * **Selection carrier** — the validity plane (no ops), an execution +//! extent (scalar folds over a top-level range: zero mask words, F4), +//! tile-local mask ops (the default), or ONE reused population mask when +//! the pass × state count makes re-evaluating the predicate the costlier +//! side. The reused mask is the one population-sized allocation this file +//! can make, it is counted, and it is named in the explain output. +//! +//! The honest limit: a partition costs one pass per member tuple, so a plan +//! whose partition side is high-cardinality AND densely observed exceeds the +//! pass budget and is REFUSED with [`ReportError::PassBudget`] rather than +//! run slowly or allocated densely. The primitive that would lift it — a +//! composite-key (multi-lane) group fold in `ndarray::simd` / mask-risc — is +//! a named substrate gap, not something to hand-roll here. + +use std::collections::HashMap; +use std::sync::Arc; + +use lance_graph_mask_risc::{ + execute_extent, scratch_words_for, tile_words_for, words_for, Foreign, Out, Planes, Program, + Scratch, Value, +}; +use lance_graph_quack::{lower, Agg, Cmp, Col, Filter, GroupAddr, GroupAgg, Mask, Query}; + +use crate::batch::{AbiBatch, LaneData}; +use crate::plan::{CoordSpec, FoldState, PhysicalKey, ReportPlan, SourceRef}; +use crate::result::{CellSpace, DimMeta, Layout, ReportResult}; +use crate::selection::RowRange; +use crate::ReportError; + +/// The knobs the physical planner reads. Every one changes a decision (each +/// has an inertness test in `tests/`), none changes a result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PlannerPolicy { + /// Largest coordinate product stored densely. + pub dense_cell_budget: u64, + /// Largest single-coordinate domain a fold-key / discovery buffer may span. + pub domain_buffer_budget: u32, + /// Most passes over the population one report may make. + pub pass_budget: u64, + /// Reuse ONE materialized selection mask once the plan would otherwise + /// evaluate the selection predicate in at least this many programs. + pub reuse_mask_min_programs: u64, +} + +impl Default for PlannerPolicy { + fn default() -> Self { + Self { + dense_cell_budget: 1 << 20, + domain_buffer_budget: 1 << 22, + pass_budget: 1 << 16, + reuse_mask_min_programs: 3, + } + } +} + +/// How a coordinate places a row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Provider { + /// A resident ordinal lane. + OrdinalLane { + /// Lane index in the batch. + lane: u16, + }, + /// A derived bucketing of a signed lane; each member is a pair of range + /// compares, never a materialized lane. + DerivedBucket { + /// Lane index of the bucketed field. + lane: u16, + }, +} + +/// One canonical dimension as planned. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DimPlan { + /// The coordinate. + pub coord: CoordSpec, + /// Domain size. + pub domain: u32, + /// How rows are placed on it. + pub provider: Provider, +} + +/// The carrier the planner chose for the selection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SelectionCarrier { + /// Every valid row: the validity plane, read in place, zero ops. + ValidityPlane, + /// Mask ops evaluated tile by tile (at most `TILE_WORDS` words per slot). + TileLocal, + /// One population mask, materialized once and read as a resident plane + /// by every later program. + ReusedMask, +} + +/// The accumulator strategy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Accumulator { + /// One cell. + Scalar, + /// Every coordinate of the product has a slot. + Dense { + /// Slots allocated. + cells: u64, + }, + /// Only observed coordinates get a slot; the product is never allocated. + Sparse { + /// The product of domains (for the explain output only). + product: u128, + }, +} + +/// The explainable physical plan. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PhysicalPlan { + /// The source. + pub source: SourceRef, + /// Population size. + pub n_rows: usize, + /// Selection carrier. + pub selection: SelectionCarrier, + /// Execution extent (scalar folds over a top-level range only). + pub extent: Option, + /// Canonical dimensions. + pub dims: Vec, + /// Canonical index of the fold-key dimension, if any. + pub fold_key: Option, + /// Canonical indices of the partition dimensions, in pass order. + pub partitions: Vec, + /// Fold states (index 0 is COUNT). + pub states: Vec, + /// Accumulator strategy. + pub accumulator: Accumulator, + /// Passes over the population (dense: exact; sparse: an upper bound + /// before discovery). + pub passes: u64, + /// The programs of the FIRST pass, as lowered — the physical identity + /// two front-ends must agree on (plan-equivalence tests compare these). + pub first_pass: Vec, +} + +/// What a run actually did — the counters that catch hidden materialization. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ExecStats { + /// Programs executed over the population (each is one scan). + pub population_scans: u64, + /// Of those, discovery scans (sparse accumulators only). + pub discovery_scans: u64, + /// Mask ops executed (each writes tile-local scratch, never a population). + pub tile_mask_ops: u64, + /// Population-sized masks materialized (0 unless [`SelectionCarrier::ReusedMask`]). + pub mask_materializations: u64, + /// Bytes of those masks. + pub mask_bytes: u64, + /// Largest tile scratch any one program needed. + pub scratch_bytes_peak: u64, + /// Fold-state bytes allocated for the result. + pub accumulator_bytes: u64, + /// Transient domain-sized buffer bytes (fold key / discovery), peak. + pub domain_buffer_bytes: u64, + /// Cells stored in the result. + pub result_cells: u64, +} + +struct Resolved { + dims: Vec, + fold_key: Option, + partitions: Vec, + states: Vec, + state_lanes: Vec>, +} + +fn member_filter(d: &DimPlan, m: u32) -> Filter { + match (&d.provider, &d.coord) { + (Provider::OrdinalLane { lane }, _) => Filter::Cmp(Col(*lane), Cmp::EqU32(m)), + (Provider::DerivedBucket { lane }, CoordSpec::Bucket { origin, width, .. }) => { + let lo = origin + width * i64::from(m); + let hi = lo + width; + let mut parts = Vec::with_capacity(2); + if lo > i64::from(i32::MIN) { + parts.push(Filter::Cmp( + Col(*lane), + Cmp::GeI32(lo.min(i64::from(i32::MAX)) as i32), + )); + } + if hi <= i64::from(i32::MAX) { + parts.push(Filter::Cmp( + Col(*lane), + Cmp::LtI32(hi.max(i64::from(i32::MIN)) as i32), + )); + } + if lo > i64::from(i32::MAX) { + // Wholly above the lane's range: the member is empty. + parts = vec![Filter::Cmp(Col(*lane), Cmp::GtI32(i32::MAX))]; + } + match parts.len() { + 0 => Filter::Plane(Mask(0)), + 1 => parts.pop().expect("one part"), + _ => Filter::And(parts), + } + } + (Provider::DerivedBucket { .. }, CoordSpec::Field(_)) => { + unreachable!("bucket provider on a field") + } + } +} + +fn resolve( + key: &PhysicalKey, + batch: &AbiBatch, + policy: &PlannerPolicy, +) -> Result { + let mut dims = Vec::with_capacity(key.coords.len()); + for c in &key.coords { + let f = c.field(); + let (lane, col) = batch.column(f).ok_or(ReportError::UnknownField(f))?; + let dim = match c { + CoordSpec::Field(_) => DimPlan { + coord: c.clone(), + domain: col.domain.ok_or(ReportError::NotACoordinate(f))?, + provider: Provider::OrdinalLane { lane }, + }, + CoordSpec::Bucket { width, count, .. } => { + if !matches!(col.lane, LaneData::I32(_)) || *width <= 0 { + return Err(ReportError::BadBucket(f)); + } + DimPlan { + coord: c.clone(), + domain: *count, + provider: Provider::DerivedBucket { lane }, + } + } + }; + dims.push(dim); + } + let fold_key = dims + .iter() + .enumerate() + .filter(|(_, d)| { + matches!(d.provider, Provider::OrdinalLane { .. }) + && d.domain <= policy.domain_buffer_budget + }) + .max_by_key(|(i, d)| (d.domain, std::cmp::Reverse(*i))) + .map(|(i, _)| i); + let partitions = (0..dims.len()).filter(|&i| Some(i) != fold_key).collect(); + + let mut states = vec![FoldState::Count]; + for m in &key.measures { + for s in m.folds() { + if !states.contains(&s) { + states.push(s); + } + } + } + let state_lanes = states + .iter() + .map(|s| match s { + FoldState::Count => Ok(None), + FoldState::Sum(f) | FoldState::Min(f) | FoldState::Max(f) => { + let (lane, col) = batch.column(*f).ok_or(ReportError::UnknownField(*f))?; + if !matches!(col.lane, LaneData::I32(_)) { + return Err(ReportError::NotAMeasure(*f)); + } + Ok(Some(lane)) + } + }) + .collect::, _>>()?; + Ok(Resolved { + dims, + fold_key, + partitions, + states, + state_lanes, + }) +} + +/// The aggregate a fold state lowers to, keyed or scalar. +fn agg_for(state: &FoldState, lane: Option, key: Option) -> Agg { + let v = lane.map(Col); + match (key.map(Col), state) { + (Some(k), FoldState::Count) => Agg::GroupReduce { + key: GroupAddr::Local(k), + agg: GroupAgg::Count, + }, + (Some(k), FoldState::Sum(_)) => Agg::GroupSumI32 { + key: k, + val: v.expect("lane"), + }, + (Some(k), FoldState::Min(_)) => Agg::GroupReduce { + key: GroupAddr::Local(k), + agg: GroupAgg::MinI32(v.expect("lane")), + }, + (Some(k), FoldState::Max(_)) => Agg::GroupReduce { + key: GroupAddr::Local(k), + agg: GroupAgg::MaxI32(v.expect("lane")), + }, + (None, FoldState::Count) => Agg::Count, + (None, FoldState::Sum(_)) => Agg::SumI32(v.expect("lane")), + (None, FoldState::Min(_)) => Agg::MinI32(v.expect("lane")), + (None, FoldState::Max(_)) => Agg::MaxI32(v.expect("lane")), + } +} + +fn scalar_of(state: &FoldState, v: Value) -> i64 { + match v { + Value::Count(n) => n as i64, + Value::SumI64(s) => s, + Value::OptI32(Some(x)) => i64::from(x), + _ => state.identity(), + } +} + +/// `base AND member₁ AND …` — the base (selection or plane) leads, so it +/// gates every member compare under the survivor skip. +fn conj(base: &Filter, members: &[Filter]) -> Filter { + if members.is_empty() { + return base.clone(); + } + Filter::And( + std::iter::once(base.clone()) + .chain(members.iter().cloned()) + .collect(), + ) +} + +/// Mixed-radix iteration over member tuples. +fn for_each_tuple( + radices: &[Vec], + mut f: impl FnMut(&[u32]) -> Result<(), ReportError>, +) -> Result<(), ReportError> { + if radices.iter().any(Vec::is_empty) { + return Ok(()); + } + let mut ctr = vec![0usize; radices.len()]; + let mut tuple: Vec = radices.iter().map(|r| r[0]).collect(); + loop { + f(&tuple)?; + let mut i = radices.len(); + loop { + if i == 0 { + return Ok(()); + } + i -= 1; + ctr[i] += 1; + if ctr[i] < radices[i].len() { + tuple[i] = radices[i][ctr[i]]; + break; + } + ctr[i] = 0; + tuple[i] = radices[i][0]; + } + } +} + +impl ReportPlan { + fn check_source(&self, batch: &AbiBatch) -> Result<(), ReportError> { + if self.source.id != batch.source() || self.source.generation != batch.generation() { + return Err(ReportError::StaleSource); + } + Ok(()) + } + + /// Plan physically, without touching a row. + pub fn explain( + &self, + batch: &AbiBatch, + policy: &PlannerPolicy, + ) -> Result { + self.check_source(batch)?; + let key = self.physical_key(); + let r = resolve(&key, batch, policy)?; + let product: u128 = r.dims.iter().map(|d| u128::from(d.domain)).product(); + let accumulator = if r.dims.is_empty() { + Accumulator::Scalar + } else if product <= u128::from(policy.dense_cell_budget) { + Accumulator::Dense { + cells: product as u64, + } + } else { + Accumulator::Sparse { product } + }; + let passes: u128 = r + .partitions + .iter() + .map(|&i| u128::from(r.dims[i].domain)) + .product(); + let passes = passes.min(u128::from(u64::MAX)) as u64; + + // Scalar folds take a top-level range as an execution extent. + let (extent, base_sel) = if r.dims.is_empty() { + self.selection.split_range() + } else { + (None, Some(self.selection.clone())) + }; + if let Some(e) = extent { + if e.hi as usize > batch.n_rows() || e.lo > e.hi { + return Err(ReportError::RangeOutOfBounds { + lo: e.lo, + hi: e.hi, + n_rows: batch.n_rows(), + }); + } + } + let base = match &base_sel { + None => Filter::Plane(Mask(0)), + Some(s) => s.lower(batch)?, + }; + let sel_ops = lower(&Query { + filter: base.clone(), + agg: Agg::Count, + })? + .ops + .len(); + let programs = passes.saturating_mul(r.states.len() as u64); + let selection = if sel_ops == 0 { + SelectionCarrier::ValidityPlane + } else if programs >= policy.reuse_mask_min_programs { + SelectionCarrier::ReusedMask + } else { + SelectionCarrier::TileLocal + }; + let first_members: Vec = r + .partitions + .iter() + .map(|&i| member_filter(&r.dims[i], 0)) + .collect(); + let fbase = if selection == SelectionCarrier::ReusedMask { + Filter::Plane(Mask(u16::MAX)) + } else { + base + }; + let key_lane = r.fold_key.map(|i| match r.dims[i].provider { + Provider::OrdinalLane { lane } => lane, + Provider::DerivedBucket { lane } => lane, + }); + let first_pass = r + .states + .iter() + .zip(&r.state_lanes) + .map(|(s, &l)| { + lower(&Query { + filter: conj(&fbase, &first_members), + agg: agg_for(s, l, key_lane), + }) + }) + .collect::, _>>()?; + Ok(PhysicalPlan { + source: self.source, + n_rows: batch.n_rows(), + selection, + extent, + dims: r.dims, + fold_key: r.fold_key, + partitions: r.partitions, + states: r.states, + accumulator, + passes, + first_pass, + }) + } + + /// Plan and fold. The only population work in the crate. + pub fn execute( + &self, + batch: &AbiBatch, + policy: &PlannerPolicy, + ) -> Result<(ReportResult, ExecStats), ReportError> { + let pp = self.explain(batch, policy)?; + let key = self.physical_key(); + let r = resolve(&key, batch, policy)?; + let mut stats = ExecStats::default(); + let n = batch.n_rows(); + let extent = pp.extent.map_or(0..n, |e| e.lo as usize..e.hi as usize); + + let (mut masks, lanes) = batch.views(); + let base_sel = if pp.dims.is_empty() { + self.selection.split_range().1 + } else { + Some(self.selection.clone()) + }; + let mut base = match &base_sel { + None => Filter::Plane(Mask(0)), + Some(s) => s.lower(batch)?, + }; + + // ── selection carrier ──────────────────────────────────────────── + let reused: Vec; + if pp.selection == SelectionCarrier::ReusedMask { + let mut buf = vec![0u64; words_for(n)]; + let prog = lower(&Query { + filter: base.clone(), + agg: Agg::Rows, + })?; + { + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &lanes, + }; + let mut scratch = Scratch::for_program(&prog, n)?; + execute_extent( + &prog, + &planes, + &Foreign::NONE, + &mut scratch, + Out::Mask(&mut buf), + extent.clone(), + )?; + } + stats.population_scans += 1; + stats.tile_mask_ops += prog.ops.len() as u64; + stats.mask_materializations += 1; + stats.mask_bytes += (buf.len() * 8) as u64; + reused = buf; + let idx = u16::try_from(masks.len()).map_err(|_| ReportError::TooManyPlanes)?; + masks.push(&reused); + base = Filter::Plane(Mask(idx)); + } + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &lanes, + }; + + let key_lane = r.fold_key.map(|i| match r.dims[i].provider { + Provider::OrdinalLane { lane } | Provider::DerivedBucket { lane } => lane, + }); + let key_domain = r.fold_key.map_or(1, |i| r.dims[i].domain as usize); + + // One program run: lowers, sizes branch-private scratch, executes. + let run = |filter: Filter, + s: usize, + out: Option<&mut [i64]>, + stats: &mut ExecStats| + -> Result { + let prog = lower(&Query { + filter, + agg: agg_for(&r.states[s], r.state_lanes[s], key_lane), + })?; + let mut scratch = Scratch::for_program(&prog, n)?; + if prog.requires_scratch() { + let b = scratch_words_for(tile_words_for(n), prog.scratch_slots as usize) + .unwrap_or(0) as u64 + * 8; + stats.scratch_bytes_peak = stats.scratch_bytes_peak.max(b); + } + stats.population_scans += 1; + stats.tile_mask_ops += prog.ops.len() as u64; + let out = match out { + Some(o) => Out::I64(o), + None => Out::None, + }; + Ok(execute_extent( + &prog, + &planes, + &Foreign::NONE, + &mut scratch, + out, + extent.clone(), + )?) + }; + + let dims_meta: Vec = pp + .dims + .iter() + .map(|d| DimMeta { + coord: d.coord.clone(), + domain: d.domain, + }) + .collect(); + let nstates = r.states.len(); + + let space = match pp.accumulator { + Accumulator::Scalar => { + let mut values: Vec> = Vec::with_capacity(nstates); + for s in 0..nstates { + let v = run(base.clone(), s, None, &mut stats)?; + values.push(vec![scalar_of(&r.states[s], v)].into_boxed_slice()); + } + CellSpace { + key, + dims: dims_meta, + layout: Layout::Dense { strides: vec![] }, + states: r.states.clone(), + values, + } + } + Accumulator::Dense { cells } => { + if pp.passes > policy.pass_budget { + return Err(ReportError::PassBudget { + passes: pp.passes, + budget: policy.pass_budget, + }); + } + // Storage order: partitions (pass order), then the fold key + // innermost so each pass writes one contiguous run. + let mut strides = vec![0usize; r.dims.len()]; + let mut stride = 1usize; + if let Some(k) = r.fold_key { + strides[k] = 1; + stride = key_domain; + } + for &p in r.partitions.iter().rev() { + strides[p] = stride; + stride *= r.dims[p].domain as usize; + } + let mut values: Vec> = r + .states + .iter() + .map(|s| vec![s.identity(); cells as usize].into_boxed_slice()) + .collect(); + let radices: Vec> = r + .partitions + .iter() + .map(|&p| (0..r.dims[p].domain).collect()) + .collect(); + for_each_tuple(&radices, |tuple| { + let members: Vec = r + .partitions + .iter() + .zip(tuple) + .map(|(&p, &m)| member_filter(&r.dims[p], m)) + .collect(); + let at: usize = r + .partitions + .iter() + .zip(tuple) + .map(|(&p, &m)| m as usize * strides[p]) + .sum(); + for (s, col) in values.iter_mut().enumerate() { + let f = conj(&base, &members); + if r.fold_key.is_some() { + run(f, s, Some(&mut col[at..at + key_domain]), &mut stats)?; + } else { + let v = run(f, s, None, &mut stats)?; + col[at] = scalar_of(&r.states[s], v); + } + } + Ok(()) + })?; + CellSpace { + key, + dims: dims_meta, + layout: Layout::Dense { strides }, + states: r.states.clone(), + values, + } + } + Accumulator::Sparse { .. } => { + // Discovery: which members of each ordinal partition occur + // under the selection — one domain-sized count pass each. + let mut radices: Vec> = Vec::with_capacity(r.partitions.len()); + for &p in &r.partitions { + let d = &r.dims[p]; + match d.provider { + Provider::OrdinalLane { lane } => { + if d.domain > policy.domain_buffer_budget { + return Err(ReportError::DomainBudget { + domain: d.domain, + budget: policy.domain_buffer_budget, + }); + } + let mut buf = vec![0i64; d.domain as usize]; + stats.domain_buffer_bytes = + stats.domain_buffer_bytes.max((buf.len() * 8) as u64); + let prog = lower(&Query { + filter: base.clone(), + agg: Agg::GroupReduce { + key: GroupAddr::Local(Col(lane)), + agg: GroupAgg::Count, + }, + })?; + let mut scratch = Scratch::for_program(&prog, n)?; + execute_extent( + &prog, + &planes, + &Foreign::NONE, + &mut scratch, + Out::I64(&mut buf), + extent.clone(), + )?; + stats.population_scans += 1; + stats.discovery_scans += 1; + stats.tile_mask_ops += prog.ops.len() as u64; + radices.push((0..d.domain).filter(|&m| buf[m as usize] > 0).collect()); + } + Provider::DerivedBucket { .. } => radices.push((0..d.domain).collect()), + } + } + let passes: u128 = radices.iter().map(|r| r.len() as u128).product(); + if passes > u128::from(policy.pass_budget) { + return Err(ReportError::PassBudget { + passes: passes.min(u128::from(u64::MAX)) as u64, + budget: policy.pass_budget, + }); + } + let ndims = r.dims.len(); + let mut coords: Vec> = vec![Vec::new(); ndims]; + let mut vals: Vec> = vec![Vec::new(); nstates]; + let mut inner: Vec> = if r.fold_key.is_some() { + stats.domain_buffer_bytes = stats + .domain_buffer_bytes + .max((key_domain * 8 * nstates) as u64); + r.states + .iter() + .map(|s| vec![s.identity(); key_domain]) + .collect() + } else { + Vec::new() + }; + for_each_tuple(&radices, |tuple| { + let members: Vec = r + .partitions + .iter() + .zip(tuple) + .map(|(&p, &m)| member_filter(&r.dims[p], m)) + .collect(); + let f = conj(&base, &members); + let mut emit = |k: Option, st: &dyn Fn(usize) -> i64| { + for (&p, &m) in r.partitions.iter().zip(tuple) { + coords[p].push(m); + } + if let (Some(kd), Some(k)) = (r.fold_key, k) { + coords[kd].push(k); + } + for (s, v) in vals.iter_mut().enumerate() { + v.push(st(s)); + } + }; + if r.fold_key.is_some() { + for (s, buf) in inner.iter_mut().enumerate() { + buf.fill(r.states[s].identity()); + run(f.clone(), s, Some(buf), &mut stats)?; + } + for (k, &count) in inner[0].iter().enumerate() { + if count > 0 { + emit(Some(k as u32), &|s| inner[s][k]); + } + } + } else { + let got = (0..nstates) + .map(|s| { + run(f.clone(), s, None, &mut stats) + .map(|v| scalar_of(&r.states[s], v)) + }) + .collect::, _>>()?; + if got[0] > 0 { + emit(None, &|s| got[s]); + } + } + Ok(()) + })?; + let coords: Vec> = + coords.into_iter().map(Vec::into_boxed_slice).collect(); + let cells = vals[0].len(); + let mut index = HashMap::with_capacity(cells); + for slot in 0..cells { + let c: Box<[u32]> = coords.iter().map(|col| col[slot]).collect(); + index.insert(c, slot as u32); + } + CellSpace { + key, + dims: dims_meta, + layout: Layout::Sparse { coords, index }, + states: r.states.clone(), + values: vals.into_iter().map(Vec::into_boxed_slice).collect(), + } + } + }; + stats.accumulator_bytes = space.accumulator_bytes() as u64; + stats.result_cells = space.stored_cells() as u64; + Ok((ReportResult::new(Arc::new(space), self)?, stats)) + } +} diff --git a/crates/lance-graph-report/src/explain.rs b/crates/lance-graph-report/src/explain.rs new file mode 100644 index 000000000..ac0cbbf9e --- /dev/null +++ b/crates/lance-graph-report/src/explain.rs @@ -0,0 +1,87 @@ +//! The human-readable physical plan — diagnostics, and the surface that +//! catches accidental hidden materialization before it runs. +//! +//! ```text +//! Source: S1@g3 (1000000 rows) +//! Selection: tile-local mask ops (no population mask) +//! Coordinates: +//! F2 ordinal lane · domain 32 · fold key +//! F7 derived bucket · domain 12 · partition +//! Folds: COUNT, SUM(F4) +//! Physical: 12 pass(es) · dense accumulator 384 cells +//! Materialization: terminal only +//! ``` + +use core::fmt; + +use crate::exec::{Accumulator, PhysicalPlan, Provider, SelectionCarrier}; +use crate::plan::{CoordSpec, FoldState}; + +impl fmt::Display for PhysicalPlan { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!( + f, + "Source: {}@g{} ({} rows)", + self.source.id, self.source.generation, self.n_rows + )?; + let sel = match self.selection { + SelectionCarrier::ValidityPlane => "validity plane (zero ops)", + SelectionCarrier::TileLocal => "tile-local mask ops (no population mask)", + SelectionCarrier::ReusedMask => { + "ONE reused population mask (materialized once, counted)" + } + }; + write!(f, "Selection: {sel}")?; + if let Some(e) = self.extent { + write!( + f, + " · execution extent [{}, {}) (no mask words)", + e.lo, e.hi + )?; + } + writeln!(f)?; + writeln!(f, "Coordinates:")?; + for (i, d) in self.dims.iter().enumerate() { + let prov = match (&d.provider, &d.coord) { + (Provider::OrdinalLane { .. }, _) => "ordinal lane".to_string(), + (Provider::DerivedBucket { .. }, CoordSpec::Bucket { origin, width, .. }) => { + format!("derived bucket (origin {origin}, width {width}; not materialized)") + } + (Provider::DerivedBucket { .. }, _) => "derived".to_string(), + }; + let role = if self.fold_key == Some(i) { + "fold key" + } else { + "partition" + }; + writeln!( + f, + " {} {prov} · domain {} · {role}", + d.coord.field(), + d.domain + )?; + } + let folds: Vec = self + .states + .iter() + .map(|s| match s { + FoldState::Count => "COUNT".to_string(), + FoldState::Sum(x) => format!("SUM({x})"), + FoldState::Min(x) => format!("MIN({x})"), + FoldState::Max(x) => format!("MAX({x})"), + }) + .collect(); + writeln!(f, "Folds: {}", folds.join(", "))?; + let acc = match self.accumulator { + Accumulator::Scalar => "scalar".to_string(), + Accumulator::Dense { cells } => format!("dense accumulator {cells} cells"), + Accumulator::Sparse { product } => { + format!( + "sparse accumulator (observed cells only; product {product} never allocated)" + ) + } + }; + writeln!(f, "Physical: {} pass(es) · {acc}", self.passes)?; + write!(f, "Materialization: terminal only") + } +} diff --git a/crates/lance-graph-report/src/ids.rs b/crates/lance-graph-report/src/ids.rs new file mode 100644 index 000000000..e9cbdbd5c --- /dev/null +++ b/crates/lance-graph-report/src/ids.rs @@ -0,0 +1,42 @@ +//! Canonical, fixed-width identities — the only names the execution core uses. +//! +//! **Strings are presentation metadata, not execution coordinates.** A textual +//! field name resolves to a [`FieldId`] at the boundary (the catalog); a +//! textual categorical value resolves to an ordinal in that field's domain +//! (the CAM label store). After that, nothing variable-length survives into a +//! plan, a selection, a coordinate, a mask, a fold or a cell. + +/// A field (lane) identity. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct FieldId(pub u32); + +/// A resident mask identity. [`MaskId::ALPHA`] is the validity plane. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct MaskId(pub u32); + +impl MaskId { + /// The validity plane every batch carries. + pub const ALPHA: MaskId = MaskId(0); +} + +/// A published source identity. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct SourceId(pub u32); + +impl core::fmt::Display for FieldId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "F{}", self.0) + } +} + +impl core::fmt::Display for MaskId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "M{}", self.0) + } +} + +impl core::fmt::Display for SourceId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "S{}", self.0) + } +} diff --git a/crates/lance-graph-report/src/lib.rs b/crates/lance-graph-report/src/lib.rs new file mode 100644 index 000000000..5566aa6bd --- /dev/null +++ b/crates/lance-graph-report/src/lib.rs @@ -0,0 +1,153 @@ +//! # lance-graph-report — a coordinate-native fold algebra that reporting falls out of +//! +//! ```text +//! population + coordinate interpretation + selection + fold algebra + terminal view +//! ``` +//! +//! **Never transpose data to pivot. Transpose meaning.** +//! +//! **Pivot defines coordinates. Fold populates coordinates. Materialization +//! renders coordinates.** +//! +//! **Coordinate change is not data change.** **Strings are presentation +//! metadata, not execution coordinates.** **Filter IDs first. Hydrate raw +//! values last.** +//! +//! This crate is NOT a second engine. A [`ReportPlan`] lowers into +//! `lance-graph-quack`'s `Query` and every program it runs is executed by +//! `lance-graph-mask-risc`'s one evaluator over resident SoA lanes, so Quack, +//! z8run and native Rust callers share one semantic representation (F10, A13). +//! +//! ## Three layers, never collapsed +//! +//! | layer | owns | here | +//! |---|---|---| +//! | **KV** | raw facts, variable-sized values | [`boundary::MemKv`] over the contract's `ContentStore`/`ContentSink` | +//! | **CAM / codebook** | human-readable labels ↔ canonical ordinals | [`boundary::CamLabels`] (a codebook of `ContentId`s — no text stored) | +//! | **ABI / SoA** | fixed-width ids, ordinals, measures, masks, folds | [`AbiBatch`], [`ReportPlan`], [`exec`], [`result`] | +//! +//! The execution modules (`ids`, `batch`, `selection`, `plan`, `exec`, +//! `result`) contain no `String`; a source fence (`tests/string_fence.rs`) +//! fails the build if one appears. Text lives only in [`boundary`] (ingest, +//! catalog, CAM, KV), [`explain`] (diagnostics) and [`render`] (terminal). +//! +//! ## Module map +//! +//! * [`ids`] — `FieldId`, `MaskId`, `SourceId`. +//! * [`batch`] — resident fixed-width lanes, borrowed never copied. +//! * [`selection`] — lazy selection; a predicate is never a bitmap by construction. +//! * [`plan`] — the control-plane plan: coordinate providers with roles, +//! mergeable fold states, the role-free [`plan::PhysicalKey`]. +//! * [`exec`] — physical planning (fold key / partitions / dense-vs-sparse / +//! selection carrier) and the fold. +//! * [`result`] — the canonical-coordinate aggregate space and zero-copy views. +//! * [`boundary`] — the only string-accepting surface: KV, CAM, catalog, ingest. +//! * [`render`] — terminal adapters (JSON / CSV data export): the materialization boundary. Paged / screen output goes through OGAR (`lance-graph-report-ogar`). +//! * [`explain`] — the human-readable physical plan. + +#![forbid(unsafe_code)] + +pub mod batch; +pub mod boundary; +pub mod exec; +pub mod explain; +pub mod ids; +pub mod plan; +pub mod render; +pub mod result; +pub mod selection; + +pub use batch::{AbiBatch, BatchError, Column, LaneData}; +pub use exec::{Accumulator, ExecStats, PhysicalPlan, PlannerPolicy, Provider, SelectionCarrier}; +pub use ids::{FieldId, MaskId, SourceId}; +pub use plan::{ + AxisRole, AxisSpec, CellValue, CoordSpec, FoldState, Measure, MeasureKind, PhysicalKey, + ReportPlan, SourceRef, TopK, +}; +pub use result::{CellSpace, CellState, ReportResult, ReportShape, View}; +pub use selection::{CmpOp, PredicatePlan, RowRange, Scalar, Selection}; + +use lance_graph_mask_risc::ExecError; +use lance_graph_quack::LowerError; + +/// Everything that can refuse a report. Every refusal is structural and +/// happens before or instead of population work — never a partial answer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReportError { + /// A row range past the population, or inverted. + RangeOutOfBounds { + /// First row. + lo: u32, + /// One past the last. + hi: u32, + /// Population size. + n_rows: usize, + }, + /// A selection names a mask the batch does not carry. + UnknownMask(MaskId), + /// A plan names a field the batch does not carry. + UnknownField(FieldId), + /// `IN ()`. + EmptyIn(FieldId), + /// A literal that does not fit the field's lane kind. + BadLiteral { + /// The field. + field: FieldId, + /// The literal. + value: Scalar, + }, + /// An ordered compare on an ordinal field (ordinals are categorical). + OrderedCompareOnOrdinal(FieldId), + /// A field used as a coordinate has no declared domain. + NotACoordinate(FieldId), + /// A bucket coordinate over a non-`I32` field or with width <= 0. + BadBucket(FieldId), + /// A measure over a non-`I32` field. + NotAMeasure(FieldId), + /// The plan was minted against another source or generation. + StaleSource, + /// `reinterpret` across plans whose folds differ. + PhysicalKeyMismatch, + /// Top-k names a measure the plan does not have. + BadTopK, + /// More resident planes than a `u16` addresses. + TooManyPlanes, + /// The plan needs more population passes than the policy allows. + PassBudget { + /// Passes needed. + passes: u64, + /// Allowed. + budget: u64, + }, + /// A coordinate domain too wide for a discovery / fold-key buffer. + DomainBudget { + /// The domain. + domain: u32, + /// Allowed. + budget: u32, + }, + /// Quack refused the lowering. + Lower(LowerError), + /// The evaluator refused the program. + Exec(ExecError), +} + +impl From for ReportError { + fn from(e: LowerError) -> Self { + ReportError::Lower(e) + } +} + +impl From for ReportError { + fn from(e: ExecError) -> Self { + ReportError::Exec(e) + } +} + +impl core::fmt::Display for ReportError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{self:?}") + } +} + +impl std::error::Error for ReportError {} diff --git a/crates/lance-graph-report/src/plan.rs b/crates/lance-graph-report/src/plan.rs new file mode 100644 index 000000000..3d0679a2a --- /dev/null +++ b/crates/lance-graph-report/src/plan.rs @@ -0,0 +1,393 @@ +//! The report plan: control plane only. +//! +//! A [`ReportPlan`] names a source, a lazy [`Selection`], a set of axes with +//! ROLES, and measures. Every rewrite on it — filter, add/remove an axis, +//! reassign a role, pivot, rotate — is a metadata edit that touches no row. +//! +//! **Pivot defines coordinates. Fold populates coordinates. Materialization +//! renders coordinates.** +//! +//! The split that makes rotation free is [`ReportPlan::physical_key`]: the +//! fold's work depends on WHICH axes exist, never on which role each plays. +//! Two plans that differ only in roles share a physical key, so a result +//! computed for one is reinterpreted for the other without a rescan +//! (falsifiers F13 / F14). + +use crate::ids::{FieldId, SourceId}; +use crate::selection::Selection; + +/// The role an axis plays in the presented coordinate system. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum AxisRole { + /// Down the side. + Row, + /// Across the top. + Column, + /// One page (facet) per member. + Page, +} + +/// A coordinate provider — WHERE in the aggregate space a row lands. +/// +/// Two providers, and neither knows what it measures: +/// +/// * [`CoordSpec::Field`] — a resident `U32` code lane with a declared domain. +/// It can serve as the fold KEY (the one dimension a single substrate pass +/// scatters into). +/// * [`CoordSpec::Bucket`] — a DERIVED coordinate: `floor((v - origin) / +/// width)` over an `I32` lane, `count` members. It is never materialized as +/// a population lane: each member is a pair of tile-evaluated range +/// predicates (`origin + b·width <= v < origin + (b+1)·width`) applied +/// during the fold. A value outside every bucket lies outside the domain. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum CoordSpec { + /// A resident ordinal field. + Field(FieldId), + /// A derived equal-width bucketing of a signed value field. + Bucket { + /// The `I32` field bucketed. + field: FieldId, + /// Lower edge of bucket 0. + origin: i64, + /// Bucket width (> 0). + width: i64, + /// Number of buckets. + count: u32, + }, +} + +impl CoordSpec { + /// The field this coordinate reads. + pub fn field(&self) -> FieldId { + match self { + CoordSpec::Field(f) | CoordSpec::Bucket { field: f, .. } => *f, + } + } +} + +impl From for CoordSpec { + fn from(f: FieldId) -> Self { + CoordSpec::Field(f) + } +} + +/// An axis: a coordinate provider plus its presented role. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct AxisSpec { + /// The coordinate. + pub coord: CoordSpec, + /// Its presented role. + pub role: AxisRole, +} + +/// What a measure computes per cell. +/// +/// The coordinate/pivot engine never matches on this. It asks a measure for +/// its fold states ([`Measure::folds`]), folds each into the aggregate space, +/// merges them along axes for totals ([`FoldState::merge`]) and hands the +/// merged states back to [`Measure::finalize`]. A new measure is a new arm +/// HERE — `folds` + `finalize` — and nothing in the engine changes (A11). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MeasureKind { + /// `COUNT(*)`. + Count, + /// `SUM(field)` — NULL for an empty cell, as in SQL. + Sum, + /// `MIN(field)`. + Min, + /// `MAX(field)`. + Max, + /// `MEAN(field)` — finalized from the SUM and COUNT states. + Mean, +} + +/// One mergeable accumulator the substrate folds per cell. Every state has a +/// merge law, which is what makes totals, parallel partial folds and cached +/// re-aggregation possible without rescanning. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum FoldState { + /// Number of selected rows in the cell. Always present: it defines the + /// EMPTY cell (count 0) for every other state. + Count, + /// Σ of an `I32` field, widened to `i64`. + Sum(FieldId), + /// Minimum of an `I32` field. + Min(FieldId), + /// Maximum of an `I32` field. + Max(FieldId), +} + +impl FoldState { + /// The identity element of the merge. + pub fn identity(&self) -> i64 { + match self { + FoldState::Count | FoldState::Sum(_) => 0, + FoldState::Min(_) => i64::MAX, + FoldState::Max(_) => i64::MIN, + } + } + + /// The merge law (associative, commutative, [`FoldState::identity`] is + /// its identity). + pub fn merge(&self, a: i64, b: i64) -> i64 { + match self { + FoldState::Count | FoldState::Sum(_) => a.wrapping_add(b), + FoldState::Min(_) => a.min(b), + FoldState::Max(_) => a.max(b), + } + } +} + +/// A finalized cell value. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum CellValue { + /// An integer. + Int(i64), + /// A real. + Real(f64), + /// No value: the cell is empty (no selected row) or absent (the + /// coordinate was never observed). Pinned here, not by a renderer. + Null, +} + +/// A measure. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Measure { + /// What it computes. + pub kind: MeasureKind, + /// The `I32` value field (`None` only for `Count`). + pub field: Option, +} + +impl Measure { + /// `COUNT(*)`. + pub fn count() -> Self { + Self { + kind: MeasureKind::Count, + field: None, + } + } + + /// `kind(field)`. + pub fn of(kind: MeasureKind, field: FieldId) -> Self { + Self { + kind, + field: Some(field), + } + } + + /// The fold's operator name (diagnostics / terminal headers). + pub fn op_name(&self) -> &'static str { + match self.kind { + MeasureKind::Count => "COUNT", + MeasureKind::Sum => "SUM", + MeasureKind::Min => "MIN", + MeasureKind::Max => "MAX", + MeasureKind::Mean => "MEAN", + } + } + + /// The fold states this measure needs (beyond the always-present + /// [`FoldState::Count`]). + pub fn folds(&self) -> Vec { + let f = || self.field.unwrap_or(FieldId(u32::MAX)); + match self.kind { + MeasureKind::Count => vec![], + MeasureKind::Sum => vec![FoldState::Sum(f())], + MeasureKind::Min => vec![FoldState::Min(f())], + MeasureKind::Max => vec![FoldState::Max(f())], + MeasureKind::Mean => vec![FoldState::Sum(f())], + } + } + + /// Finalize from merged states. `state(s)` reads one merged state; + /// `count` is the merged COUNT. Empty (`count == 0`) is NULL for every + /// measure but COUNT itself, whose empty answer is a real 0. + pub fn finalize(&self, count: i64, state: &dyn Fn(&FoldState) -> i64) -> CellValue { + if self.kind == MeasureKind::Count { + return CellValue::Int(count); + } + if count == 0 { + return CellValue::Null; + } + let f = &self.folds()[0]; + match self.kind { + MeasureKind::Mean => CellValue::Real(state(f) as f64 / count as f64), + _ => CellValue::Int(state(f)), + } + } +} + +/// Keep the top `k` row-keys by one measure's row total (a result-sized +/// ordering applied to the aggregate, never to the population). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct TopK { + /// Index into [`ReportPlan::measures`]. + pub measure: usize, + /// How many row keys to keep. + pub k: usize, + /// Largest first. + pub descending: bool, +} + +/// The source a plan reads: a published batch, by identity and generation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct SourceRef { + /// Published identity. + pub id: SourceId, + /// The generation the plan was minted against. + pub generation: u32, +} + +/// A report plan. Immutable by convention: rewrites return a new plan, so a +/// plan shared by a DAG fan-out (behind an `Arc`) is never mutated under a +/// sibling branch. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ReportPlan { + /// What to read. + pub source: SourceRef, + /// Which rows. + pub selection: Selection, + /// The coordinate axes, with roles. Order within a role is presentation + /// order (outermost first). + pub axes: Vec, + /// What each cell holds. + pub measures: Vec, + /// Optional result-side ordering. + pub top_k: Option, +} + +/// The part of a plan the FOLD depends on. Roles, axis order and top-k are +/// absent on purpose: they are presentation, and changing them must never +/// cause population work. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PhysicalKey { + /// The source. + pub source: SourceRef, + /// The selection. + pub selection: Selection, + /// The coordinates, sorted (a SET, not a sequence): the canonical + /// dimension order of the aggregate space. + pub coords: Vec, + /// The measures, in plan order. + pub measures: Vec, +} + +impl ReportPlan { + /// A plan over every valid row of `source`, no axes, no measures. + pub fn over(source: SourceRef) -> Self { + Self { + source, + selection: Selection::All, + axes: Vec::new(), + measures: Vec::new(), + top_k: None, + } + } + + /// AND a selection into the plan. + pub fn filter(mut self, s: Selection) -> Self { + self.selection = std::mem::replace(&mut self.selection, Selection::All).and(s); + self + } + + /// Add (or re-role) an axis. An existing axis on the same coordinate + /// changes role and moves to the end of its new role's order. + pub fn axis(mut self, coord: impl Into, role: AxisRole) -> Self { + let coord = coord.into(); + self.axes.retain(|a| a.coord != coord); + self.axes.push(AxisSpec { coord, role }); + self + } + + /// Remove an axis. This changes the coordinate SET, hence the physical + /// key: the cells it separated must merge. + pub fn without_axis(mut self, coord: &CoordSpec) -> Self { + self.axes.retain(|a| &a.coord != coord); + self + } + + /// Replace the whole role assignment in one step (rows, columns, pages, + /// each outermost first). Every coordinate named must already be an + /// axis; axes not named keep their role at the end. Pure metadata. + pub fn view(mut self, rows: &[CoordSpec], columns: &[CoordSpec], pages: &[CoordSpec]) -> Self { + let mut axes = Vec::with_capacity(self.axes.len()); + for (list, role) in [ + (rows, AxisRole::Row), + (columns, AxisRole::Column), + (pages, AxisRole::Page), + ] { + for c in list { + axes.push(AxisSpec { + coord: c.clone(), + role, + }); + } + } + for a in self.axes.drain(..) { + if !axes.iter().any(|b| b.coord == a.coord) { + axes.push(a); + } + } + self.axes = axes; + self + } + + /// Add a measure. + pub fn measure(mut self, m: Measure) -> Self { + self.measures.push(m); + self + } + + /// Set the result ordering. + pub fn top_k(mut self, t: TopK) -> Self { + self.top_k = Some(t); + self + } + + /// Pivot: `rows` become the row axes and `columns` the column axes, in + /// the given order. Any other axis keeps its role. Pure metadata. + pub fn pivot(mut self, rows: &[CoordSpec], columns: &[CoordSpec]) -> Self { + for r in rows { + self = self.axis(r.clone(), AxisRole::Row); + } + for c in columns { + self = self.axis(c.clone(), AxisRole::Column); + } + self + } + + /// Rotate: every row axis becomes a column axis and vice versa. Pure + /// metadata; the physical key is unchanged. + pub fn rotate(mut self) -> Self { + for a in &mut self.axes { + a.role = match a.role { + AxisRole::Row => AxisRole::Column, + AxisRole::Column => AxisRole::Row, + AxisRole::Page => AxisRole::Page, + }; + } + self + } + + /// Coordinates playing `role`, in presentation order. + pub fn axes_in(&self, role: AxisRole) -> Vec<&CoordSpec> { + self.axes + .iter() + .filter(|a| a.role == role) + .map(|a| &a.coord) + .collect() + } + + /// The fold-relevant part of the plan. + pub fn physical_key(&self) -> PhysicalKey { + let mut coords: Vec = self.axes.iter().map(|a| a.coord.clone()).collect(); + coords.sort(); + PhysicalKey { + source: self.source, + selection: self.selection.clone(), + coords, + measures: self.measures.clone(), + } + } +} diff --git a/crates/lance-graph-report/src/render.rs b/crates/lance-graph-report/src/render.rs new file mode 100644 index 000000000..2f1885fed --- /dev/null +++ b/crates/lance-graph-report/src/render.rs @@ -0,0 +1,364 @@ +//! Terminal adapters — the materialization boundary. +//! +//! **A report may contain strings. The reporting engine does not.** This is +//! the one place aggregate coordinates meet text: each presented member is +//! resolved through CAM ([`CamLabels::resolve`]) exactly once per render, so +//! label work is proportional to the result's axis metadata, never to the +//! population. JSON and CSV walk the SAME [`Grid`]; the formats differ only +//! in how they print it (TEST 8). +//! +//! **Data export, not paper.** JSON and CSV are interchange terminals. Paged +//! and screen layout (HTML, Typst → PDF, A2UI) is NOT this crate's: a report +//! reaches those through OGAR's composition path — an `ObjectSlot` naming +//! the result, resolved by `ogar-doc-ir` (`SlotOutcome::Grid`) and emitted by +//! OGAR's renderers (see `lance-graph-report-ogar`). The reporting engine +//! does not become a paper engine; [`Grid`] is the renderer-neutral hand-off +//! both paths share. +//! +//! Detail exports ([`Terminal::detail_csv`]) obey **filter IDs first, +//! hydrate raw values last**: the selection runs as ids in the substrate, +//! the surviving rows are materialized by the one named materializer +//! (`lance_graph_mask_risc::materialize_rows`) HERE, and only those rows' +//! KV references are dereferenced. + +use lance_graph_contract::content_store::ContentId; +use lance_graph_mask_risc::{ + execute_into, materialize_rows, words_for, Foreign, Out, Planes, Scratch, +}; +use lance_graph_quack::{lower, Agg, Query}; + +use crate::batch::LaneData; +use crate::boundary::{CamLabels, Catalog, MemKv}; +use crate::ids::FieldId; +use crate::plan::{AxisRole, CellValue, CoordSpec, ReportPlan}; +use crate::result::ReportResult; +use crate::{AbiBatch, ReportError}; + +/// A rendered grid: the presented cells, labels resolved. Result-sized. +#[derive(Debug, Clone, PartialEq)] +pub struct Grid { + /// Column keys: the ordinal tuple of each presented column (identity, + /// kept for back-trace to the aggregate coordinate). + pub column_keys: Vec>, + /// Column header rows: one label tuple per column key. + pub columns: Vec>, + /// Measure headers. + pub measures: Vec, + /// Pages: (page labels, rows). + pub pages: Vec<(Vec, Vec)>, + /// Grand total per measure. + pub grand_total: Vec, +} + +/// One presented row. +#[derive(Debug, Clone, PartialEq)] +pub struct GridRow { + /// The row's ordinal tuple (identity, for back-trace). + pub key: Vec, + /// Row labels. + pub labels: Vec, + /// `cells[col][measure]`. + pub cells: Vec>, + /// Row total per measure. + pub total: Vec, +} + +/// Render statistics. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct RenderStats { + /// Cells rendered. + pub cells: u64, + /// Output bytes. + pub bytes: u64, +} + +/// The terminal: the label store, the KV and the catalog, borrowed. +pub struct Terminal<'a> { + /// CAM labels. + pub cam: &'a CamLabels, + /// KV. + pub kv: &'a MemKv, + /// Field names. + pub catalog: &'a Catalog, +} + +fn fmt_value(v: CellValue) -> String { + match v { + CellValue::Int(i) => i.to_string(), + CellValue::Real(r) => format!("{r}"), + CellValue::Null => String::new(), + } +} + +fn json_value(v: CellValue) -> String { + match v { + CellValue::Null => "null".to_string(), + other => fmt_value(other), + } +} + +fn json_str(s: &str) -> String { + let mut o = String::with_capacity(s.len() + 2); + o.push('"'); + for c in s.chars() { + match c { + '"' => o.push_str("\\\""), + '\\' => o.push_str("\\\\"), + '\n' => o.push_str("\\n"), + c if (c as u32) < 0x20 => o.push_str(&format!("\\u{:04x}", c as u32)), + c => o.push(c), + } + } + o.push('"'); + o +} + +fn csv_field(s: &str) -> String { + if s.contains([',', '"', '\n']) { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.to_string() + } +} + +impl Terminal<'_> { + fn field_name(&self, f: FieldId) -> String { + self.catalog + .name(f) + .map_or_else(|| f.to_string(), str::to_string) + } + + fn member_label(&self, c: &CoordSpec, m: u32) -> String { + match c { + CoordSpec::Field(f) => self + .cam + .resolve(*f, m, self.kv) + .map_or_else(|| m.to_string(), str::to_string), + CoordSpec::Bucket { origin, width, .. } => { + let lo = origin + width * i64::from(m); + format!("[{lo},{})", lo + width) + } + } + } + + fn labels(&self, r: &ReportResult, role: AxisRole, key: &[u32]) -> Vec { + let v = r.view(); + let dims = match role { + AxisRole::Row => &v.rows, + AxisRole::Column => &v.columns, + AxisRole::Page => &v.pages, + }; + dims.iter() + .zip(key) + .map(|(&d, &m)| self.member_label(&r.space().dims()[d].coord, m)) + .collect() + } + + /// Resolve the presented grid. Labels are resolved once per presented + /// member (memoized per dimension), values come from the shared cells. + pub fn grid(&self, r: &ReportResult) -> Grid { + let ms = r.measures(); + let cols = r.column_keys(); + let rows = r.row_keys(); + let pages = r.page_keys(); + let row_labels: Vec> = rows + .iter() + .map(|k| self.labels(r, AxisRole::Row, k)) + .collect(); + Grid { + column_keys: cols.clone(), + columns: cols + .iter() + .map(|k| self.labels(r, AxisRole::Column, k)) + .collect(), + measures: ms + .iter() + .map(|m| { + format!( + "{}({})", + m.op_name(), + m.field + .map_or_else(|| "*".to_string(), |f| self.field_name(f)) + ) + }) + .collect(), + pages: pages + .iter() + .map(|p| { + let body = rows + .iter() + .zip(&row_labels) + .map(|(rk, labels)| GridRow { + key: rk.clone(), + labels: labels.clone(), + cells: cols + .iter() + .map(|ck| ms.iter().map(|m| r.value(m, p, rk, ck)).collect()) + .collect(), + total: ms.iter().map(|m| r.value(m, p, rk, &[])).collect(), + }) + .collect(); + (self.labels(r, AxisRole::Page, p), body) + }) + .collect(), + grand_total: ms.iter().map(|m| r.grand_total(m)).collect(), + } + } + + /// JSON: `{"columns":[[..]],"measures":[..],"pages":[{"page":[..],"rows":[{"row":[..],"cells":[[..]],"total":[..]}]}],"grand_total":[..]}`. + pub fn json(&self, r: &ReportResult) -> (String, RenderStats) { + let g = self.grid(r); + let strs = |v: &[String]| { + format!( + "[{}]", + v.iter().map(|s| json_str(s)).collect::>().join(",") + ) + }; + let vals = |v: &[CellValue]| { + format!( + "[{}]", + v.iter() + .map(|x| json_value(*x)) + .collect::>() + .join(",") + ) + }; + let mut cells = 0u64; + let pages = g + .pages + .iter() + .map(|(p, rows)| { + let rows = rows + .iter() + .map(|row| { + cells += (row.cells.len() * g.measures.len()) as u64; + format!( + "{{\"row\":{},\"cells\":[{}],\"total\":{}}}", + strs(&row.labels), + row.cells + .iter() + .map(|c| vals(c)) + .collect::>() + .join(","), + vals(&row.total) + ) + }) + .collect::>() + .join(","); + format!("{{\"page\":{},\"rows\":[{rows}]}}", strs(p)) + }) + .collect::>() + .join(","); + let out = format!( + "{{\"columns\":[{}],\"measures\":{},\"pages\":[{pages}],\"grand_total\":{}}}", + g.columns + .iter() + .map(|c| strs(c)) + .collect::>() + .join(","), + strs(&g.measures), + vals(&g.grand_total) + ); + let bytes = out.len() as u64; + (out, RenderStats { cells, bytes }) + } + + /// CSV, long form per page: `page…,row…,column…,measure,value`. + pub fn csv(&self, r: &ReportResult) -> (String, RenderStats) { + let g = self.grid(r); + let mut out = String::from("page,row,column,measure,value\n"); + let mut cells = 0u64; + for (p, rows) in &g.pages { + for row in rows { + for (ci, col) in g.columns.iter().enumerate() { + for (mi, m) in g.measures.iter().enumerate() { + cells += 1; + out.push_str(&format!( + "{},{},{},{},{}\n", + csv_field(&p.join("/")), + csv_field(&row.labels.join("/")), + csv_field(&col.join("/")), + csv_field(m), + fmt_value(row.cells[ci][mi]) + )); + } + } + } + } + let bytes = out.len() as u64; + (out, RenderStats { cells, bytes }) + } + + /// Detail export: the rows the plan's selection keeps, with `fields` + /// hydrated — `U64` references from KV, ordinals through CAM, integers + /// as-is. The selection runs as ids; rows are materialized HERE by the + /// one named materializer; hydration is proportional to the rows + /// exported, never to the population. + pub fn detail_csv( + &self, + plan: &ReportPlan, + batch: &AbiBatch, + fields: &[FieldId], + ) -> Result<(String, RenderStats), ReportError> { + let n = batch.n_rows(); + let filter = plan.selection.lower(batch)?; + let prog = lower(&Query { + filter, + agg: Agg::Rows, + })?; + let (masks, lanes) = batch.views(); + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &lanes, + }; + let mut kept = vec![0u64; words_for(n)]; + let mut scratch = Scratch::for_program(&prog, n)?; + execute_into( + &prog, + &planes, + &Foreign::NONE, + &mut scratch, + Out::Mask(&mut kept), + )?; + let rows = materialize_rows(&kept, n); + let mut out = fields + .iter() + .map(|f| csv_field(&self.field_name(*f))) + .collect::>() + .join(","); + out.push('\n'); + for &row in &rows { + let line = fields + .iter() + .map(|f| { + let text = match batch.column(*f).map(|(_, c)| &c.lane) { + Some(LaneData::U64(v)) => self + .kv + .text(ContentId(v[row])) + .unwrap_or_default() + .to_string(), + Some(LaneData::U32(v)) => self + .cam + .resolve(*f, v[row], self.kv) + .map_or_else(|| v[row].to_string(), str::to_string), + Some(LaneData::I32(v)) => v[row].to_string(), + None => String::new(), + }; + csv_field(&text) + }) + .collect::>() + .join(","); + out.push_str(&line); + out.push('\n'); + } + let bytes = out.len() as u64; + Ok(( + out, + RenderStats { + cells: (rows.len() * fields.len()) as u64, + bytes, + }, + )) + } +} diff --git a/crates/lance-graph-report/src/result.rs b/crates/lance-graph-report/src/result.rs new file mode 100644 index 000000000..e3ccf84d6 --- /dev/null +++ b/crates/lance-graph-report/src/result.rs @@ -0,0 +1,512 @@ +//! The aggregate coordinate space and its views. +//! +//! **Never transpose data to pivot. Transpose meaning.** +//! +//! A [`CellSpace`] holds fold states against CANONICAL coordinate identities — +//! a tuple over the plan's coordinate SET in its canonical (sorted) order. +//! Where a cell physically lives (a dense offset, or a slot in a sparse +//! column set) is private to the space and is never a presentation position. +//! +//! A [`ReportResult`] is `Arc` + a [`View`]: which canonical +//! dimensions play rows, columns and pages, plus an optional result-sized +//! presentation order. Rotating, re-rolling, re-ordering or sorting builds a +//! new `View` over the SAME `Arc` — the cell payload is never moved, copied +//! or recomputed (falsifiers F6 / F7 / F14, A3 / A4 / A5 / A8). +//! +//! Semantics pinned here, for every adapter alike: +//! +//! * a cell whose COUNT is 0 is **empty**: COUNT reads 0, every other measure +//! reads [`CellValue::Null`]; +//! * a coordinate never observed in a sparse space is **absent**: every +//! measure, COUNT included, reads `Null` — nothing was folded there; +//! * a row whose coordinate lies outside a dimension's domain (a code past +//! the domain, a value outside every bucket) belongs to no cell, so it is +//! in no total either: totals are totals OVER THE COORDINATE SPACE; +//! * a filtered-out row contributes nothing anywhere. + +use std::collections::HashMap; +use std::sync::Arc; + +use crate::plan::{AxisRole, CellValue, CoordSpec, FoldState, Measure, PhysicalKey, ReportPlan}; +use crate::ReportError; + +/// One canonical dimension of the space. +#[derive(Debug, Clone)] +pub struct DimMeta { + /// The coordinate provider. + pub coord: CoordSpec, + /// Domain size (ordinals `0..domain`). Labels are NOT here: they are + /// resolved from the CAM label store at the terminal, for the members a + /// view actually presents. + pub domain: u32, +} + +/// How cells are stored. Private to the space: no API exposes which one. +#[derive(Debug)] +pub(crate) enum Layout { + /// Every coordinate of the (bounded) product has a slot; `offset = Σ + /// coord[d] · strides[d]` in canonical dimension order. + Dense { strides: Vec }, + /// Only observed coordinates have a slot: per-dimension coordinate + /// columns (SoA) plus an index from coordinate tuple to slot. + Sparse { + coords: Vec>, + index: HashMap, u32>, + }, +} + +/// The computed aggregate space. +#[derive(Debug)] +pub struct CellSpace { + pub(crate) key: PhysicalKey, + pub(crate) dims: Vec, + pub(crate) layout: Layout, + /// Fold states; `states[0]` is always [`FoldState::Count`]. + pub(crate) states: Vec, + /// One value column per state, one slot per cell. + pub(crate) values: Vec>, +} + +/// Whether a coordinate holds a folded cell. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CellState { + /// Rows were folded here. + Present, + /// The coordinate exists but no selected row landed on it. + Empty, + /// The coordinate was never observed (sparse spaces only). + Absent, +} + +impl CellSpace { + /// The physical key this space answers. + pub fn key(&self) -> &PhysicalKey { + &self.key + } + + /// The canonical dimensions. + pub fn dims(&self) -> &[DimMeta] { + &self.dims + } + + /// Number of stored cells (dense: the product; sparse: observed). + pub fn stored_cells(&self) -> usize { + self.values[0].len() + } + + /// Whether storage is sparse (diagnostic only — semantics never differ). + pub fn is_sparse(&self) -> bool { + matches!(self.layout, Layout::Sparse { .. }) + } + + /// Address of the COUNT column's first slot — the identity the + /// zero-copy falsifiers compare across views. + pub fn payload_addr(&self) -> usize { + self.values[0].as_ptr() as usize + } + + /// Bytes held by fold-state columns (the aggregate allocation). + pub fn accumulator_bytes(&self) -> usize { + self.values.iter().map(|v| v.len() * 8).sum() + } + + fn slot(&self, coord: &[u32]) -> Option { + match &self.layout { + Layout::Dense { strides } => { + let mut off = 0; + for (d, &c) in coord.iter().enumerate() { + if c >= self.dims[d].domain { + return None; + } + off += c as usize * strides[d]; + } + Some(off) + } + Layout::Sparse { index, .. } => index.get(coord).map(|&s| s as usize), + } + } + + /// The state of the cell at a canonical coordinate. + pub fn state(&self, coord: &[u32]) -> CellState { + match self.slot(coord) { + None => CellState::Absent, + Some(s) if self.values[0][s] == 0 => CellState::Empty, + Some(_) => CellState::Present, + } + } + + fn state_index(&self, s: &FoldState) -> usize { + self.states + .iter() + .position(|x| x == s) + .expect("measure state folded") + } + + /// Merge every stored cell matching `fixed` (a partial coordinate: + /// `Some(member)` pins a dimension) and finalize `measure`. O(stored + /// cells) — result-sized, never population-sized. `None` on no match + /// (every matching slot absent). + pub fn merged(&self, measure: &Measure, fixed: &[Option]) -> CellValue { + let mut acc: Vec = self.states.iter().map(FoldState::identity).collect(); + let mut any = false; + let mut visit = |slot: usize| { + any = true; + for (i, st) in self.states.iter().enumerate() { + acc[i] = st.merge(acc[i], self.values[i][slot]); + } + }; + match &self.layout { + Layout::Dense { strides } => { + // Enumerate the sub-product the fixed coordinates leave free. + let free: Vec = (0..self.dims.len()) + .filter(|&d| fixed[d].is_none()) + .collect(); + let base: usize = (0..self.dims.len()) + .filter_map(|d| fixed[d].map(|m| m as usize * strides[d])) + .sum(); + if fixed + .iter() + .enumerate() + .any(|(d, f)| f.is_some_and(|m| m >= self.dims[d].domain)) + { + return CellValue::Null; + } + let mut ctr = vec![0u32; free.len()]; + 'outer: loop { + let off = base + + free + .iter() + .zip(&ctr) + .map(|(&d, &c)| c as usize * strides[d]) + .sum::(); + visit(off); + for i in (0..free.len()).rev() { + ctr[i] += 1; + if ctr[i] < self.dims[free[i]].domain { + continue 'outer; + } + ctr[i] = 0; + } + break; + } + } + Layout::Sparse { coords, .. } => { + #[allow(clippy::needless_range_loop)] // `slot` indexes N coordinate columns at once + for slot in 0..self.stored_cells() { + if fixed + .iter() + .enumerate() + .all(|(d, f)| f.is_none_or(|m| coords[d][slot] == m)) + { + visit(slot); + } + } + } + } + if !any { + return CellValue::Null; + } + let count = acc[0]; + measure.finalize(count, &|s| acc[self.state_index(s)]) + } +} + +/// Which canonical dimensions play which role, and in what order — pure +/// presentation metadata over a [`CellSpace`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct View { + /// Canonical dimension indices down the side, outermost first. + pub rows: Vec, + /// Across the top. + pub columns: Vec, + /// One page per member tuple. + pub pages: Vec, + /// Presentation order of the row keys (indices into + /// [`ReportResult::row_keys`]); `None` = canonical order. Result-sized. + pub row_order: Option>, +} + +/// A computed report: a shared aggregate space seen through a view. +#[derive(Debug, Clone)] +pub struct ReportResult { + pub(crate) space: Arc, + pub(crate) view: View, +} + +/// The shape a result presents as (for adapters that care). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReportShape { + /// No coordinates. + Scalar, + /// One coordinate. + Series, + /// Two or more coordinates. + Matrix, +} + +impl ReportResult { + pub(crate) fn new(space: Arc, plan: &ReportPlan) -> Result { + let r = ReportResult { + view: View { + rows: vec![], + columns: vec![], + pages: vec![], + row_order: None, + }, + space, + }; + r.reinterpret(plan) + } + + /// The shared aggregate space. + pub fn space(&self) -> &Arc { + &self.space + } + + /// The current view. + pub fn view(&self) -> &View { + &self.view + } + + /// Presented shape. + pub fn shape(&self) -> ReportShape { + match self.space.dims.len() { + 0 => ReportShape::Scalar, + 1 => ReportShape::Series, + _ => ReportShape::Matrix, + } + } + + /// The measures, in plan order. + pub fn measures(&self) -> &[Measure] { + &self.space.key.measures + } + + /// View this space through another plan's roles, order and top-k. The + /// plans must share a physical key — otherwise the fold differs and the + /// caller must execute. No population work, no cell copy. + pub fn reinterpret(&self, plan: &ReportPlan) -> Result { + if plan.physical_key() != self.space.key { + return Err(ReportError::PhysicalKeyMismatch); + } + let idx = |c: &CoordSpec| { + self.space + .dims + .iter() + .position(|d| &d.coord == c) + .expect("coord in space") + }; + let pick = |role| plan.axes_in(role).into_iter().map(idx).collect::>(); + let mut out = ReportResult { + space: Arc::clone(&self.space), + view: View { + rows: pick(AxisRole::Row), + columns: pick(AxisRole::Column), + pages: pick(AxisRole::Page), + row_order: None, + }, + }; + if let Some(t) = plan.top_k { + let m = self + .measures() + .get(t.measure) + .ok_or(ReportError::BadTopK)? + .clone(); + out = out.order_rows_by(&m, t.descending, Some(t.k)); + } + Ok(out) + } + + /// Swap rows and columns. Metadata only. + pub fn rotate(&self) -> Self { + let mut v = self.view.clone(); + std::mem::swap(&mut v.rows, &mut v.columns); + v.row_order = None; + ReportResult { + space: Arc::clone(&self.space), + view: v, + } + } + + /// Re-role in one step, by canonical coordinate. Metadata only. + pub fn with_roles( + &self, + rows: &[CoordSpec], + columns: &[CoordSpec], + pages: &[CoordSpec], + ) -> Result { + let idx = |c: &CoordSpec| { + self.space + .dims + .iter() + .position(|d| &d.coord == c) + .ok_or(ReportError::PhysicalKeyMismatch) + }; + let conv = |l: &[CoordSpec]| l.iter().map(idx).collect::, _>>(); + Ok(ReportResult { + space: Arc::clone(&self.space), + view: View { + rows: conv(rows)?, + columns: conv(columns)?, + pages: conv(pages)?, + row_order: None, + }, + }) + } + + /// Order row keys by the merged value of `measure` over each row (a + /// presentation permutation; identity untouched), optionally keeping `k`. + pub fn order_rows_by(&self, measure: &Measure, descending: bool, k: Option) -> Self { + let rows = self.row_keys(); + let key = |v: CellValue| match v { + CellValue::Int(i) => i as f64, + CellValue::Real(r) => r, + CellValue::Null => f64::NEG_INFINITY, + }; + let mut order: Vec = (0..rows.len() as u32).collect(); + let vals: Vec = rows + .iter() + .map(|r| key(self.row_total(measure, r))) + .collect(); + order.sort_by(|&a, &b| { + let (x, y) = (vals[a as usize], vals[b as usize]); + let o = x.partial_cmp(&y).unwrap_or(std::cmp::Ordering::Equal); + if descending { o.reverse() } else { o }.then(a.cmp(&b)) + }); + if let Some(k) = k { + order.truncate(k); + } + let mut v = self.view.clone(); + v.row_order = Some(order.into()); + ReportResult { + space: Arc::clone(&self.space), + view: v, + } + } + + /// Member tuples along `dims` that the view enumerates: the full product + /// for a dense space, the observed projections for a sparse one. + fn keys(&self, dims: &[usize]) -> Vec> { + if dims.is_empty() { + return vec![vec![]]; + } + match &self.space.layout { + Layout::Dense { .. } => { + let mut out = vec![vec![]]; + for &d in dims { + let n = self.space.dims[d].domain; + out = out + .into_iter() + .flat_map(|p: Vec| (0..n).map(move |m| [p.clone(), vec![m]].concat())) + .collect(); + } + out + } + Layout::Sparse { coords, .. } => { + let mut set: Vec> = (0..self.space.stored_cells()) + .map(|s| dims.iter().map(|&d| coords[d][s]).collect()) + .collect(); + set.sort(); + set.dedup(); + set + } + } + } + + /// Row keys in presentation order. + pub fn row_keys(&self) -> Vec> { + let all = self.keys(&self.view.rows); + match &self.view.row_order { + None => all, + Some(o) => o.iter().map(|&i| all[i as usize].clone()).collect(), + } + } + + /// Column keys. + pub fn column_keys(&self) -> Vec> { + self.keys(&self.view.columns) + } + + /// Page keys. + pub fn page_keys(&self) -> Vec> { + self.keys(&self.view.pages) + } + + fn fixed(&self, page: &[u32], row: &[u32], col: &[u32]) -> Vec> { + let mut f = vec![None; self.space.dims.len()]; + for (dims, key) in [ + (&self.view.pages, page), + (&self.view.rows, row), + (&self.view.columns, col), + ] { + for (&d, &m) in dims.iter().zip(key) { + f[d] = Some(m); + } + } + f + } + + /// One presented cell. Dimensions in no role are merged over (they are + /// in the space but not shown). + pub fn value(&self, measure: &Measure, page: &[u32], row: &[u32], col: &[u32]) -> CellValue { + self.space.merged(measure, &self.fixed(page, row, col)) + } + + /// Total over every column (and hidden dimension) for one row, across + /// all pages. + pub fn row_total(&self, measure: &Measure, row: &[u32]) -> CellValue { + self.space.merged(measure, &self.fixed(&[], row, &[])) + } + + /// Total over every row for one column, across all pages. + pub fn column_total(&self, measure: &Measure, col: &[u32]) -> CellValue { + self.space.merged(measure, &self.fixed(&[], &[], col)) + } + + /// Grand total. + pub fn grand_total(&self, measure: &Measure) -> CellValue { + self.space + .merged(measure, &vec![None; self.space.dims.len()]) + } + + /// `value / grand_total` for a cell (derived from aggregated cells, no + /// rescan). `None` when either side is NULL or the total is 0. + pub fn share_of_total( + &self, + measure: &Measure, + page: &[u32], + row: &[u32], + col: &[u32], + ) -> Option { + ratio( + self.value(measure, page, row, col), + self.grand_total(measure), + ) + } + + /// `value / row_total`. + pub fn share_of_row(&self, measure: &Measure, row: &[u32], col: &[u32]) -> Option { + ratio( + self.value(measure, &[], row, col), + self.row_total(measure, row), + ) + } + + /// `value / column_total`. + pub fn share_of_column(&self, measure: &Measure, row: &[u32], col: &[u32]) -> Option { + ratio( + self.value(measure, &[], row, col), + self.column_total(measure, col), + ) + } +} + +fn ratio(a: CellValue, b: CellValue) -> Option { + let f = |v| match v { + CellValue::Int(i) => Some(i as f64), + CellValue::Real(r) => Some(r), + CellValue::Null => None, + }; + let (a, b) = (f(a)?, f(b)?); + (b != 0.0).then(|| a / b) +} diff --git a/crates/lance-graph-report/src/selection.rs b/crates/lance-graph-report/src/selection.rs new file mode 100644 index 000000000..9e9b82abc --- /dev/null +++ b/crates/lance-graph-report/src/selection.rs @@ -0,0 +1,238 @@ +//! Lazy selection: WHICH rows, as a description — never as a bitmap. +//! +//! **Predicate existence is not bitmap existence.** A [`Selection`] is a +//! tree of field ids and fixed-width literals. It lowers into a Quack [`Filter`], and Quack +//! lowers that into straight-line mask ops the evaluator runs TILE BY TILE +//! (at most `TILE_WORDS` words of scratch per slot, whatever the population). +//! No population-sized mask is ever produced because a filter exists. +//! +//! A top-level row range is special-cased for scalar folds: it becomes the +//! evaluator's EXECUTION EXTENT (an outer restriction, no mask op at all), so +//! `Range → Count` folds the validity plane's words over the span and writes +//! nothing (falsifier F4). + +use lance_graph_quack::{Cmp, Col, Filter, Mask}; + +use crate::batch::{AbiBatch, LaneData}; +use crate::ids::{FieldId, MaskId}; +use crate::ReportError; + +/// A half-open row-ordinal range `[lo, hi)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct RowRange { + /// First row. + pub lo: u32, + /// One past the last row. + pub hi: u32, +} + +/// A fixed-width literal. There is deliberately no text variant: a textual +/// categorical value is resolved to its ordinal at the boundary +/// ([`crate::boundary::CamLabels::ordinal`]) BEFORE it enters a plan (S4). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Scalar { + /// A signed integer literal (for `I32` fields). + Int(i64), + /// A coordinate ordinal / label id (for `U32` fields). + Ordinal(u32), +} + +/// A comparison operator. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CmpOp { + /// `=` + Eq, + /// `<>` + Ne, + /// `<` + Lt, + /// `<=` + Le, + /// `>` + Gt, + /// `>=` + Ge, +} + +/// A predicate over one field. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum PredicatePlan { + /// `column value`. + Cmp { + /// Field. + field: FieldId, + /// Operator. + op: CmpOp, + /// Literal. + value: Scalar, + }, + /// `column IN (values)`. + In { + /// Field. + field: FieldId, + /// Literals. + values: Vec, + }, +} + +/// Which rows participate — lazily. Nothing is evaluated until a terminal +/// fold runs, and then only tile by tile. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Selection { + /// Every valid row (the validity plane). + All, + /// A row-ordinal range. + Range(RowRange), + /// A column predicate. + Predicate(PredicatePlan), + /// A resident mask of the batch (a reused facet mask, a focus). + Mask(MaskId), + /// Both. + And(Box, Box), + /// Either. + Or(Box, Box), + /// The first but not the second. + AndNot(Box, Box), +} + +impl Selection { + /// `self AND other`, with `All` as the identity (so a rewrite chain that + /// starts at `All` never grows a no-op conjunct). + pub fn and(self, other: Selection) -> Selection { + match (self, other) { + (Selection::All, s) | (s, Selection::All) => s, + (a, b) => Selection::And(Box::new(a), Box::new(b)), + } + } + + /// `self OR other`. + pub fn or(self, other: Selection) -> Selection { + Selection::Or(Box::new(self), Box::new(other)) + } + + /// `self AND NOT other`. + pub fn and_not(self, other: Selection) -> Selection { + Selection::AndNot(Box::new(self), Box::new(other)) + } + + /// `field value` shorthand. + pub fn cmp(field: FieldId, op: CmpOp, value: Scalar) -> Selection { + Selection::Predicate(PredicatePlan::Cmp { field, op, value }) + } + + /// `field IN (values)` shorthand. + pub fn is_in(field: FieldId, values: impl IntoIterator) -> Selection { + Selection::Predicate(PredicatePlan::In { + field, + values: values.into_iter().collect(), + }) + } + + /// Split off the top-level row ranges (the conjuncts reachable through + /// `And` alone), intersected, from the remainder. + /// + /// The range half becomes an execution extent; the remainder a filter. + /// `None` for the range means no top-level range; `None` for the + /// remainder means nothing but ranges (read the validity plane). + pub(crate) fn split_range(&self) -> (Option, Option) { + match self { + Selection::Range(r) => (Some(*r), None), + Selection::And(a, b) => { + let (ra, sa) = a.split_range(); + let (rb, sb) = b.split_range(); + let range = match (ra, rb) { + (Some(x), Some(y)) => { + let lo = x.lo.max(y.lo); + Some(RowRange { + lo, + hi: x.hi.min(y.hi).max(lo), + }) + } + (x, y) => x.or(y), + }; + let rest = match (sa, sb) { + (Some(x), Some(y)) => Some(x.and(y)), + (x, y) => x.or(y), + }; + (range, rest) + } + other => (None, Some(other.clone())), + } + } + + /// Lower to Quack's filter vocabulary, resolving field ids to lanes + /// against `batch`. Structural: O(tree size), reads no row. + pub(crate) fn lower(&self, batch: &AbiBatch) -> Result { + Ok(match self { + Selection::All => Filter::Plane(Mask(0)), + // Row-ordinal range: `Pred::Range` reads no lane; the `Col` is + // provenance only. Here the range IS a row range by definition, + // so no ordered-lane witness is involved. + Selection::Range(r) => { + if r.hi as usize > batch.n_rows() || r.lo > r.hi { + return Err(ReportError::RangeOutOfBounds { + lo: r.lo, + hi: r.hi, + n_rows: batch.n_rows(), + }); + } + Filter::Cmp(Col(0), Cmp::Range { lo: r.lo, hi: r.hi }) + } + Selection::Mask(id) => Filter::Plane(Mask( + batch.plane_of(*id).ok_or(ReportError::UnknownMask(*id))?, + )), + Selection::Predicate(p) => lower_predicate(p, batch)?, + Selection::And(a, b) => Filter::and([a.lower(batch)?, b.lower(batch)?]), + Selection::Or(a, b) => Filter::or([a.lower(batch)?, b.lower(batch)?]), + Selection::AndNot(a, b) => { + Filter::and([a.lower(batch)?, Filter::negate(b.lower(batch)?)]) + } + }) + } +} + +fn lower_predicate(p: &PredicatePlan, batch: &AbiBatch) -> Result { + match p { + PredicatePlan::Cmp { field, op, value } => leaf(*field, *op, *value, batch), + PredicatePlan::In { field, values } => { + if values.is_empty() { + return Err(ReportError::EmptyIn(*field)); + } + Ok(Filter::or( + values + .iter() + .map(|v| leaf(*field, CmpOp::Eq, *v, batch)) + .collect::, _>>()?, + )) + } + } +} + +fn leaf(field: FieldId, op: CmpOp, value: Scalar, batch: &AbiBatch) -> Result { + let (lane, col) = batch + .column(field) + .ok_or(ReportError::UnknownField(field))?; + let bad = ReportError::BadLiteral { field, value }; + let cmp = match (&col.lane, value) { + (LaneData::I32(_), Scalar::Int(v)) => { + let v = i32::try_from(v).map_err(|_| bad)?; + match op { + CmpOp::Eq => Cmp::EqI32(v), + CmpOp::Ne => Cmp::NeI32(v), + CmpOp::Lt => Cmp::LtI32(v), + CmpOp::Le => Cmp::LeI32(v), + CmpOp::Gt => Cmp::GtI32(v), + CmpOp::Ge => Cmp::GeI32(v), + } + } + (LaneData::U32(_), Scalar::Ordinal(v)) => match op { + CmpOp::Eq => Cmp::EqU32(v), + CmpOp::Ne => Cmp::NeU32(v), + // Ordinals are categorical: the IR has no ordered compare on an + // unsigned lane. Refused, never approximated. + _ => return Err(ReportError::OrderedCompareOnOrdinal(field)), + }, + _ => return Err(bad), + }; + Ok(Filter::Cmp(Col(lane), cmp)) +} diff --git a/crates/lance-graph-report/tests/agnostic.rs b/crates/lance-graph-report/tests/agnostic.rs new file mode 100644 index 000000000..b5aa30d1c --- /dev/null +++ b/crates/lance-graph-report/tests/agnostic.rs @@ -0,0 +1,600 @@ +//! The AGNOSTIC validation suite: synthetic dimensions, no domain nouns. +//! TEST 1–9 of the capstone, plus the planner-knob inertness twins. + +mod common; + +use common::*; +use lance_graph_mask_risc::Program; +use lance_graph_quack::{lower, Agg, Cmp, Col, Filter, Query}; +use lance_graph_report::*; + +const A: FieldId = FieldId(0); +const B: FieldId = FieldId(1); +const C: FieldId = FieldId(2); +const V: FieldId = FieldId(100); +const W: FieldId = FieldId(101); + +fn fa() -> CoordSpec { + CoordSpec::Field(A) +} +fn fb() -> CoordSpec { + CoordSpec::Field(B) +} +fn fc() -> CoordSpec { + CoordSpec::Field(C) +} + +// ── TEST 1 — AXIS ROTATION ────────────────────────────────────────────── +#[test] +fn t1_rotation_is_metadata_only_and_reads_the_same_cells() { + let fx = synthetic(5_000, &[4, 7], 11); + let batch = fx.batch(); + let plan = with_four(ReportPlan::over(src()).pivot(&[fa()], &[fb()]), V); + let (res, stats) = plan.execute(&batch, &PlannerPolicy::default()).unwrap(); + assert_matches_oracle(&fx, &plan, &res, V); + let scans_before = stats.population_scans; + + // Rotation through the RESULT (no batch in reach) and through the PLAN. + let rot = res.rotate(); + let rot_plan = plan.clone().rotate(); + assert_eq!( + rot_plan.physical_key(), + plan.physical_key(), + "roles are not part of the fold" + ); + let rot2 = res.reinterpret(&rot_plan).unwrap(); + for r in [&rot, &rot2] { + assert!( + std::sync::Arc::ptr_eq(r.space(), res.space()), + "cell payload shared, not copied" + ); + assert_eq!(r.space().payload_addr(), res.space().payload_addr()); + } + // rows=A,cols=B cell (a,b) == rows=B,cols=A cell (b,a), every cell. + let m = &res.measures()[1]; + for a in 0..4 { + for b in 0..7 { + assert_eq!(res.value(m, &[], &[a], &[b]), rot.value(m, &[], &[b], &[a])); + assert_eq!( + res.value(m, &[], &[a], &[b]), + rot2.value(m, &[], &[b], &[a]) + ); + } + } + assert_eq!(rot.row_keys().len(), 7); + assert_eq!(rot.column_keys().len(), 4); + // No re-aggregation: the only scans are the original execution's. + assert_eq!(scans_before, stats.population_scans); +} + +// ── TEST 2 — THREE-DIMENSIONAL ROLE PERMUTATION ───────────────────────── +#[test] +fn t2_three_dim_role_permutation_changes_only_view_metadata() { + let fx = synthetic(8_000, &[3, 4, 5], 12); + let batch = fx.batch(); + let p1 = with_four( + ReportPlan::over(src()) + .axis(fa(), AxisRole::Row) + .axis(fb(), AxisRole::Column) + .axis(fc(), AxisRole::Page), + V, + ); + let (r1, _) = p1.execute(&batch, &PlannerPolicy::default()).unwrap(); + assert_matches_oracle(&fx, &p1, &r1, V); + let p2 = p1.clone().view(&[fc(), fb()], &[fa()], &[]); + assert_eq!(p2.physical_key(), p1.physical_key()); + let r2 = r1.reinterpret(&p2).unwrap(); + assert!(std::sync::Arc::ptr_eq(r1.space(), r2.space())); + assert_eq!(r2.view().rows.len(), 2); + assert!(r2.view().pages.is_empty()); + let m = &r1.measures()[0]; + for a in 0..3 { + for b in 0..4 { + for c in 0..5 { + // view 1: page=c, row=a, col=b view 2: row=(c,b), col=a + assert_eq!( + r1.value(m, &[c], &[a], &[b]), + r2.value(m, &[], &[c, b], &[a]) + ); + } + } + } + // Removing a presented axis from the view merges over it — still no fold. + let r3 = r1.with_roles(&[fa()], &[], &[]).unwrap(); + assert!(std::sync::Arc::ptr_eq(r1.space(), r3.space())); + let or = fx.oracle(&p1, Some(V)); + let want: i64 = or.iter().filter(|(k, _)| k[0] == 2).map(|(_, v)| v.0).sum(); + assert_eq!(r3.value(m, &[], &[2], &[]), CellValue::Int(want)); +} + +// ── TEST 3 — RANGE WITHOUT MASK ───────────────────────────────────────── +#[test] +fn t3_range_count_and_sum_write_no_mask() { + let fx = synthetic(20_000, &[3], 13); + let batch = fx.batch(); + let range = Selection::Range(RowRange { + lo: 1_234, + hi: 17_001, + }); + let plan = with_four(ReportPlan::over(src()).filter(range.clone()), V); + let pp = plan.explain(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!( + pp.extent, + Some(RowRange { + lo: 1_234, + hi: 17_001 + }) + ); + assert_eq!(pp.selection, SelectionCarrier::ValidityPlane); + assert!(pp + .first_pass + .iter() + .all(|p| p.ops.is_empty() && !p.requires_scratch())); + let (res, st) = plan.execute(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(st.tile_mask_ops, 0, "no mask op of any size"); + assert_eq!(st.mask_materializations, 0); + assert_eq!(st.scratch_bytes_peak, 0); + assert_matches_oracle(&fx, &plan, &res, V); + + // Can-fire twin: the SAME range, but not top-level (inside an OR), must + // fall back to a tile-local range write — proving the extent path is a + // decision, not a constant. + let never = Selection::cmp(V, CmpOp::Gt, Scalar::Int(10_000)); + let plan2 = with_four(ReportPlan::over(src()).filter(range.or(never)), V); + let pp2 = plan2.explain(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(pp2.extent, None); + assert_ne!(pp2.selection, SelectionCarrier::ValidityPlane); + let (res2, st2) = plan2.execute(&batch, &PlannerPolicy::default()).unwrap(); + assert!(st2.tile_mask_ops > 0); + assert_eq!( + res2.grand_total(&res2.measures()[1]), + res.grand_total(&res.measures()[1]) + ); +} + +// ── TEST 4 — MASK REUSE ───────────────────────────────────────────────── +fn sparse_selection() -> Selection { + Selection::cmp(A, CmpOp::Eq, Scalar::Ordinal(3)) + .and(Selection::cmp(V, CmpOp::Gt, Scalar::Int(40))) + .and_not(Selection::cmp(W, CmpOp::Lt, Scalar::Int(100))) +} + +#[test] +fn t4_planner_reuses_one_mask_only_when_the_selection_is_reevaluated_enough() { + let fx = synthetic(30_000, &[10, 6], 14); + let batch = fx.batch(); + // Anti-vacuity: the selection keeps a small, non-empty minority. + let kept = (0..fx.n) + .filter(|&i| fx.selected(&sparse_selection(), i)) + .count(); + assert!(kept > 0 && kept * 20 < fx.n, "kept {kept}"); + + // One scalar measure: one program, re-evaluation is cheaper → tile-local. + let one = ReportPlan::over(src()).filter(sparse_selection()); + let pp = one.explain(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(pp.selection, SelectionCarrier::TileLocal); + let (_, st) = one.execute(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(st.mask_materializations, 0); + + // Four measures × 6 partition passes: reuse ONE mask. + let many = with_four( + ReportPlan::over(src()) + .filter(sparse_selection()) + .pivot(&[fb()], &[fa()]), + V, + ); + let pp = many.explain(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(pp.selection, SelectionCarrier::ReusedMask); + let (r_reuse, st) = many.execute(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(st.mask_materializations, 1); + assert_eq!(st.mask_bytes, (fx.n.div_ceil(64) * 8) as u64); + assert_matches_oracle(&fx, &many, &r_reuse, V); + + // Inertness twin: raising the threshold silences reuse; results identical. + let no_reuse = PlannerPolicy { + reuse_mask_min_programs: u64::MAX, + ..PlannerPolicy::default() + }; + let (r_eval, st2) = many.execute(&batch, &no_reuse).unwrap(); + assert_eq!(st2.mask_materializations, 0); + assert!( + st2.tile_mask_ops > st.tile_mask_ops, + "re-evaluation costs more mask ops" + ); + for m in r_reuse.measures() { + assert_eq!(r_reuse.grand_total(m), r_eval.grand_total(m)); + } +} + +// ── TEST 5 — SPARSE COORDINATE PRODUCT ────────────────────────────────── +#[test] +fn t5_high_cardinality_sparse_product_is_never_allocated_densely() { + const D: u32 = 1_000_000; + let n = 20_000; + let mut r = Rng(15); + let a_members: Vec = (0..40).map(|_| r.below(u64::from(D)) as u32).collect(); + let b_members: Vec = (0..30).map(|_| r.below(u64::from(D)) as u32).collect(); + let fx = Fixture { + n, + ords: vec![ + ( + A, + (0..n).map(|_| a_members[r.below(40) as usize]).collect(), + D, + ), + ( + B, + (0..n).map(|_| b_members[r.below(30) as usize]).collect(), + D, + ), + ], + vals: vec![(V, (0..n).map(|_| r.below(100) as i32 - 50).collect())], + masks: vec![], + }; + let batch = fx.batch(); + let plan = with_four(ReportPlan::over(src()).pivot(&[fa()], &[fb()]), V); + let pp = plan.explain(&batch, &PlannerPolicy::default()).unwrap(); + assert!( + matches!(pp.accumulator, Accumulator::Sparse { product } if product == 1_000_000_000_000) + ); + let (res, st) = plan.execute(&batch, &PlannerPolicy::default()).unwrap(); + let observed = fx.oracle(&plan, Some(V)).len(); + assert!(observed <= 1_200 && observed > 100, "observed {observed}"); + assert!(res.space().is_sparse()); + assert_eq!(res.space().stored_cells(), observed); + assert_eq!( + st.accumulator_bytes, + (observed * 8 * 4) as u64, + "result scales with observed cells" + ); + // The only non-result buffers are ONE-dimensional domains, never the product. + assert!(st.domain_buffer_bytes <= u64::from(D) * 8 * 4); + assert_eq!(st.discovery_scans, 1); + assert_matches_oracle(&fx, &plan, &res, V); + // Absent vs empty: an unobserved coordinate is absent. + assert_eq!( + res.space().state(&[ + a_members[0], + (0..D).find(|x| !b_members.contains(x)).unwrap() + ]), + CellState::Absent + ); +} + +// ── TEST 6 — DENSE SMALL PRODUCT (+ the dense budget's inertness twin) ── +#[test] +fn t6_tiny_product_is_dense_and_the_budget_decides_not_the_result() { + let fx = synthetic(4_000, &[3, 5], 16); + let batch = fx.batch(); + let plan = with_four(ReportPlan::over(src()).pivot(&[fa()], &[fb()]), V); + let pp = plan.explain(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(pp.accumulator, Accumulator::Dense { cells: 15 }); + let (dense, st) = plan.execute(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(st.accumulator_bytes, 15 * 8 * 4); + assert_eq!(st.discovery_scans, 0); + assert_matches_oracle(&fx, &plan, &dense, V); + + let tight = PlannerPolicy { + dense_cell_budget: 14, + ..PlannerPolicy::default() + }; + let (sparse, _) = plan.execute(&batch, &tight).unwrap(); + assert!( + sparse.space().is_sparse(), + "lowering the budget flips the strategy" + ); + for m in dense.measures() { + for a in 0..3 { + for b in 0..5 { + let d = dense.value(m, &[], &[a], &[b]); + let s = sparse.value(m, &[], &[a], &[b]); + // Empty (dense) and absent (sparse) both read NULL for every + // measure except COUNT, where empty is a real 0. + if m.kind == MeasureKind::Count && s == CellValue::Null { + assert_eq!(d, CellValue::Int(0)); + } else { + assert_eq!(d, s); + } + } + } + } +} + +// ── TEST 7 — DERIVED COORDINATE ───────────────────────────────────────── +#[test] +fn t7_derived_bucket_folds_without_a_materialized_lane() { + let fx = synthetic(25_000, &[4], 17); + let batch = fx.batch(); + // W ∈ [0,1000); buckets of 150 from 100 → [100,250) … 5 buckets up to 850; + // values below 100 and at/above 850 are OUTSIDE the domain. + let bucket = CoordSpec::Bucket { + field: W, + origin: 100, + width: 150, + count: 5, + }; + let plan = with_four( + ReportPlan::over(src()).pivot(std::slice::from_ref(&bucket), &[fa()]), + V, + ); + let pp = plan.explain(&batch, &PlannerPolicy::default()).unwrap(); + let d = pp.dims.iter().find(|d| d.coord == bucket).unwrap(); + assert!(matches!(d.provider, Provider::DerivedBucket { .. })); + assert!( + pp.fold_key.is_some_and(|k| pp.dims[k].coord == fa()), + "the ordinal lane is the fold key" + ); + let (res, st) = plan.execute(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(st.mask_materializations, 0); + assert_eq!( + st.accumulator_bytes, + 5 * 4 * 8 * 4, + "result-sized: 5×4 cells × 4 states" + ); + assert_matches_oracle(&fx, &plan, &res, V); + // Anti-vacuity: rows outside every bucket exist and are excluded. + let outside = (0..fx.n) + .filter(|&i| fx.coord(std::slice::from_ref(&bucket), i).is_none()) + .count(); + assert!(outside > fx.n / 10); + assert_eq!( + res.grand_total(&res.measures()[0]), + CellValue::Int((fx.n - outside) as i64) + ); +} + +// ── TEST 8 — TERMINAL MATERIALIZATION ─────────────────────────────────── +// JSON and CSV here; HTML / Typst go through OGAR's composition path and are +// exercised by `lance-graph-report-ogar` (the same Grid, another emitter). +#[test] +fn t8_json_and_csv_differ_only_at_the_terminal() { + use lance_graph_report::boundary::{CamLabels, Catalog, MemKv}; + use lance_graph_report::render::Terminal; + let fx = synthetic(3_000, &[3, 4], 18); + let batch = fx.batch(); + let plan = with_four(ReportPlan::over(src()).pivot(&[fa()], &[fb()]), V); + let (res, _) = plan.execute(&batch, &PlannerPolicy::default()).unwrap(); + let (cam, kv, cat) = (CamLabels::default(), MemKv::default(), Catalog::default()); + let t = Terminal { + cam: &cam, + kv: &kv, + catalog: &cat, + }; + let addr = res.space().payload_addr(); + let grid = t.grid(&res); + let (json, js) = t.json(&res); + let (csv, cs) = t.csv(&res); + assert_eq!( + res.space().payload_addr(), + addr, + "rendering never touches the cells" + ); + assert_eq!(js.cells, 3 * 4 * 4); + assert_eq!(cs.cells, js.cells); + // The grid keeps identity beside the labels (back-trace). + assert_eq!(grid.column_keys, res.column_keys()); + assert_eq!(grid.pages[0].1[2].key, vec![2]); + // Every CSV value line equals the grid value it came from, in order. + let lines: Vec<&str> = csv.lines().skip(1).collect(); + assert_eq!(lines.len(), 48); + let mut k = 0; + for row in &grid.pages[0].1 { + for c in &row.cells { + for v in c { + let want = match v { + CellValue::Int(i) => i.to_string(), + CellValue::Real(r) => format!("{r}"), + CellValue::Null => String::new(), + }; + assert!( + lines[k].ends_with(&format!(",{want}")), + "{} vs {want}", + lines[k] + ); + assert!(json.contains(&want) || want.is_empty()); + k += 1; + } + } + } +} + +// ── TEST 9 — PLAN EQUIVALENCE (native Rust ≡ hand-built Quack) ────────── +#[test] +fn t9_native_plan_lowers_to_exactly_the_programs_quack_would_write() { + let fx = synthetic(2_000, &[6, 9], 19); + let batch = fx.batch(); + // "WHERE V >= 0 GROUP BY A, B SUM(V)" — A (6) is a partition, B (9) + // the fold key (wider). Pass 0 is A = 0. + let plan = ReportPlan::over(src()) + .filter(Selection::cmp(V, CmpOp::Ge, Scalar::Int(0))) + .pivot(&[fa()], &[fb()]) + .measure(Measure::of(MeasureKind::Sum, V)); + let pp = plan + .explain( + &batch, + &PlannerPolicy { + reuse_mask_min_programs: u64::MAX, + ..PlannerPolicy::default() + }, + ) + .unwrap(); + // Lanes: F0 → 0, F1 → 1, F100 → 2 (fixture order). + let where_ = Filter::Cmp(Col(2), Cmp::GeI32(0)); + let pass0 = Filter::And(vec![where_, Filter::Cmp(Col(0), Cmp::EqU32(0))]); + let quack: Vec = [ + Agg::GroupReduce { + key: lance_graph_quack::GroupAddr::Local(Col(1)), + agg: lance_graph_quack::GroupAgg::Count, + }, + Agg::GroupSumI32 { + key: Col(1), + val: Col(2), + }, + ] + .into_iter() + .map(|agg| { + lower(&Query { + filter: pass0.clone(), + agg, + }) + .unwrap() + }) + .collect(); + assert_eq!(pp.first_pass, quack); + // The same semantic request built in a different order normalizes to + // the same physical key and the same programs. + let plan_b = ReportPlan::over(src()) + .measure(Measure::of(MeasureKind::Sum, V)) + .axis(fb(), AxisRole::Column) + .axis(fa(), AxisRole::Row) + .filter(Selection::cmp(V, CmpOp::Ge, Scalar::Int(0))); + assert_eq!(plan_b.physical_key(), plan.physical_key()); + assert_eq!( + plan_b + .explain( + &batch, + &PlannerPolicy { + reuse_mask_min_programs: u64::MAX, + ..PlannerPolicy::default() + } + ) + .unwrap() + .first_pass, + quack + ); +} + +// ── knob inertness: pass and domain budgets can fire, and stay silent ─── +#[test] +fn pass_budget_refuses_instead_of_running_slowly_and_is_silent_when_met() { + let fx = synthetic(1_000, &[20, 30], 20); + let batch = fx.batch(); + let plan = ReportPlan::over(src()) + .pivot(&[fa()], &[fb()]) + .measure(Measure::count()); + let tight = PlannerPolicy { + pass_budget: 19, + ..PlannerPolicy::default() + }; + assert!(matches!( + plan.execute(&batch, &tight), + Err(ReportError::PassBudget { + passes: 20, + budget: 19 + }) + )); + let ok = PlannerPolicy { + pass_budget: 20, + ..PlannerPolicy::default() + }; + assert!(plan.execute(&batch, &ok).is_ok()); +} + +#[test] +fn domain_budget_moves_the_fold_key() { + let fx = synthetic(1_000, &[20, 30], 21); + let batch = fx.batch(); + let plan = ReportPlan::over(src()) + .pivot(&[fa()], &[fb()]) + .measure(Measure::count()); + let wide = plan.explain(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(wide.dims[wide.fold_key.unwrap()].coord, fb()); + let narrow = plan + .explain( + &batch, + &PlannerPolicy { + domain_buffer_budget: 25, + ..PlannerPolicy::default() + }, + ) + .unwrap(); + assert_eq!(narrow.dims[narrow.fold_key.unwrap()].coord, fa()); + let (x, _) = plan.execute(&batch, &PlannerPolicy::default()).unwrap(); + let (y, _) = plan + .execute( + &batch, + &PlannerPolicy { + domain_buffer_budget: 25, + ..PlannerPolicy::default() + }, + ) + .unwrap(); + for a in 0..20 { + for b in 0..30 { + let m = &x.measures()[0]; + assert_eq!(x.value(m, &[], &[a], &[b]), y.value(m, &[], &[a], &[b])); + } + } +} + +#[test] +fn stale_source_generation_fails_closed() { + let fx = synthetic(100, &[2], 22); + let batch = fx.batch(); + let plan = ReportPlan::over(SourceRef { + id: SourceId(1), + generation: 0, + }) + .measure(Measure::count()); + assert_eq!( + plan.execute(&batch, &PlannerPolicy::default()).unwrap_err(), + ReportError::StaleSource + ); +} + +#[test] +fn top_k_orders_presentation_without_touching_identity() { + let fx = synthetic(5_000, &[8, 3], 23); + let batch = fx.batch(); + let plan = ReportPlan::over(src()) + .pivot(&[fa()], &[fb()]) + .measure(Measure::of(MeasureKind::Sum, W)); + let (res, _) = plan.execute(&batch, &PlannerPolicy::default()).unwrap(); + let top = res + .reinterpret(&plan.clone().top_k(TopK { + measure: 0, + k: 3, + descending: true, + })) + .unwrap(); + assert!(std::sync::Arc::ptr_eq(res.space(), top.space())); + let keys = top.row_keys(); + assert_eq!(keys.len(), 3); + let m = &res.measures()[0]; + let tot = |k: &Vec| match res.row_total(m, k) { + CellValue::Int(i) => i, + _ => i64::MIN, + }; + let mut all: Vec = res.row_keys().iter().map(tot).collect(); + all.sort_unstable_by(|a, b| b.cmp(a)); + assert_eq!(keys.iter().map(tot).collect::>(), all[..3].to_vec()); + // The same member reads the same value whatever its presented position. + for k in &keys { + assert_eq!(top.row_total(m, k), res.row_total(m, k)); + } +} + +#[test] +fn mean_is_derived_from_sum_and_count_and_merges_correctly_in_totals() { + let fx = synthetic(6_000, &[5], 24); + let batch = fx.batch(); + let plan = ReportPlan::over(src()) + .axis(fa(), AxisRole::Row) + .measure(Measure::of(MeasureKind::Mean, V)) + .measure(Measure::of(MeasureKind::Sum, V)) + .measure(Measure::count()); + let pp = plan.explain(&batch, &PlannerPolicy::default()).unwrap(); + assert_eq!( + pp.states, + vec![FoldState::Count, FoldState::Sum(V)], + "MEAN adds no third fold" + ); + let (res, _) = plan.execute(&batch, &PlannerPolicy::default()).unwrap(); + let (sum, cnt): (i64, i64) = (0..fx.n) + .map(|i| (i64::from(fx.vals[0].1[i]), 1)) + .fold((0, 0), |a, b| (a.0 + b.0, a.1 + b.1)); + assert_eq!( + res.grand_total(&res.measures()[0]), + CellValue::Real(sum as f64 / cnt as f64) + ); +} diff --git a/crates/lance-graph-report/tests/common/mod.rs b/crates/lance-graph-report/tests/common/mod.rs new file mode 100644 index 000000000..fdf5f3df0 --- /dev/null +++ b/crates/lance-graph-report/tests/common/mod.rs @@ -0,0 +1,249 @@ +//! Shared synthetic fixtures + the per-row ORACLE. +//! +//! The oracle is a scalar per-row loop — licensed ONLY here, as a test oracle +//! (the `#[cfg(test)]` role raw intrinsics play under `ndarray::simd`). It +//! re-derives every cell independently of the crate, from the raw vectors. +#![allow(dead_code)] + +use std::collections::HashMap; +use std::sync::Arc; + +use lance_graph_report::*; + +/// Deterministic xorshift. +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + pub fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// Raw fixture: ordinal fields (with domains) and signed value fields. +pub struct Fixture { + pub n: usize, + pub ords: Vec<(FieldId, Vec, u32)>, + pub vals: Vec<(FieldId, Vec)>, + pub masks: Vec<(MaskId, Vec)>, +} + +impl Fixture { + pub fn batch(&self) -> AbiBatch { + let mut b = AbiBatch::new(SourceId(1), 1, self.n); + for (f, v, d) in &self.ords { + b = b + .with_column(Column::coordinate(*f, v.clone().into(), *d)) + .unwrap(); + } + for (f, v) in &self.vals { + b = b + .with_column(Column::value(*f, LaneData::I32(v.clone().into()))) + .unwrap(); + } + for (m, w) in &self.masks { + b = b.with_mask(*m, Arc::from(w.clone())).unwrap(); + } + b + } + + fn ord(&self, f: FieldId) -> Option<&Vec> { + self.ords + .iter() + .find(|(x, _, _)| *x == f) + .map(|(_, v, _)| v) + } + fn domain(&self, f: FieldId) -> u32 { + self.ords.iter().find(|(x, _, _)| *x == f).unwrap().2 + } + fn val(&self, f: FieldId) -> Option<&Vec> { + self.vals.iter().find(|(x, _)| *x == f).map(|(_, v)| v) + } + + /// Per-row selection oracle. + pub fn selected(&self, s: &Selection, i: usize) -> bool { + match s { + Selection::All => true, + Selection::Range(r) => (r.lo as usize) <= i && i < r.hi as usize, + Selection::Mask(m) => { + let w = &self.masks.iter().find(|(x, _)| x == m).unwrap().1; + w[i / 64] >> (i % 64) & 1 == 1 + } + Selection::Predicate(PredicatePlan::Cmp { field, op, value }) => { + self.cmp(*field, *op, *value, i) + } + Selection::Predicate(PredicatePlan::In { field, values }) => { + values.iter().any(|v| self.cmp(*field, CmpOp::Eq, *v, i)) + } + Selection::And(a, b) => self.selected(a, i) && self.selected(b, i), + Selection::Or(a, b) => self.selected(a, i) || self.selected(b, i), + Selection::AndNot(a, b) => self.selected(a, i) && !self.selected(b, i), + } + } + + fn cmp(&self, f: FieldId, op: CmpOp, v: Scalar, i: usize) -> bool { + let (x, y): (i64, i64) = match v { + Scalar::Int(k) => (i64::from(self.val(f).unwrap()[i]), k), + Scalar::Ordinal(k) => (i64::from(self.ord(f).unwrap()[i]), i64::from(k)), + }; + match op { + CmpOp::Eq => x == y, + CmpOp::Ne => x != y, + CmpOp::Lt => x < y, + CmpOp::Le => x <= y, + CmpOp::Gt => x > y, + CmpOp::Ge => x >= y, + } + } + + /// Canonical coordinate of row `i` under `coords` (sorted), or None if + /// outside some domain. + pub fn coord(&self, coords: &[CoordSpec], i: usize) -> Option> { + coords + .iter() + .map(|c| match c { + CoordSpec::Field(f) => { + let o = self.ord(*f).unwrap()[i]; + (o < self.domain(*f)).then_some(o) + } + CoordSpec::Bucket { + field, + origin, + width, + count, + } => { + let v = i64::from(self.val(*field).unwrap()[i]); + let b = (v - origin).div_euclid(*width); + (0..i64::from(*count)).contains(&b).then_some(b as u32) + } + }) + .collect() + } + + /// Oracle cells: canonical coordinate → (count, sum, min, max) of `m`. + pub fn oracle( + &self, + plan: &ReportPlan, + m: Option, + ) -> HashMap, (i64, i64, i64, i64)> { + let coords = plan.physical_key().coords; + let mut out: HashMap, (i64, i64, i64, i64)> = HashMap::new(); + for i in 0..self.n { + if !self.selected(&plan.selection, i) { + continue; + } + let Some(c) = self.coord(&coords, i) else { + continue; + }; + let v = m.map_or(0, |f| i64::from(self.val(f).unwrap()[i])); + let e = out.entry(c).or_insert((0, 0, i64::MAX, i64::MIN)); + e.0 += 1; + e.1 += v; + e.2 = e.2.min(v); + e.3 = e.3.max(v); + } + out + } +} + +/// A synthetic population: `dims` ordinal fields F0.. with the given +/// domains, value field F100 in [-50, 50), second value field F101 in [0, 1000). +pub fn synthetic(n: usize, domains: &[u32], seed: u64) -> Fixture { + let mut r = Rng(seed | 1); + let ords = domains + .iter() + .enumerate() + .map(|(i, &d)| { + ( + FieldId(i as u32), + (0..n).map(|_| r.below(u64::from(d)) as u32).collect(), + d, + ) + }) + .collect(); + let vals = vec![ + ( + FieldId(100), + (0..n).map(|_| r.below(100) as i32 - 50).collect(), + ), + (FieldId(101), (0..n).map(|_| r.below(1000) as i32).collect()), + ]; + Fixture { + n, + ords, + vals, + masks: vec![], + } +} + +pub fn src() -> SourceRef { + SourceRef { + id: SourceId(1), + generation: 1, + } +} + +/// Compare every stored/presentable cell of `res` against the oracle for +/// COUNT / SUM / MIN / MAX of `m` (plan must carry those four measures in +/// that order). +pub fn assert_matches_oracle(fx: &Fixture, plan: &ReportPlan, res: &ReportResult, m: FieldId) { + let or = fx.oracle(plan, Some(m)); + let dims = res.space().dims(); + let ms = res.measures().to_vec(); + // Every oracle cell must read back exactly. + for (c, (cnt, sum, mn, mx)) in &or { + let fixed: Vec> = c.iter().map(|&x| Some(x)).collect(); + assert_eq!( + res.space().merged(&ms[0], &fixed), + CellValue::Int(*cnt), + "count at {c:?}" + ); + assert_eq!( + res.space().merged(&ms[1], &fixed), + CellValue::Int(*sum), + "sum at {c:?}" + ); + assert_eq!( + res.space().merged(&ms[2], &fixed), + CellValue::Int(*mn), + "min at {c:?}" + ); + assert_eq!( + res.space().merged(&ms[3], &fixed), + CellValue::Int(*mx), + "max at {c:?}" + ); + } + // And no cell the oracle lacks may carry rows. + if !res.space().is_sparse() && !dims.is_empty() { + let total: i64 = or.values().map(|v| v.0).sum(); + assert_eq!(res.grand_total(&ms[0]), CellValue::Int(total)); + } else { + assert_eq!( + res.space().stored_cells(), + or.len().max(usize::from(dims.is_empty())) + ); + } +} + +pub fn four(m: FieldId) -> [Measure; 4] { + [ + Measure::count(), + Measure::of(MeasureKind::Sum, m), + Measure::of(MeasureKind::Min, m), + Measure::of(MeasureKind::Max, m), + ] +} + +pub fn with_four(mut p: ReportPlan, m: FieldId) -> ReportPlan { + for x in four(m) { + p = p.measure(x); + } + p +} diff --git a/crates/lance-graph-report/tests/reference_workload.rs b/crates/lance-graph-report/tests/reference_workload.rs new file mode 100644 index 000000000..8176285e0 --- /dev/null +++ b/crates/lance-graph-report/tests/reference_workload.rs @@ -0,0 +1,317 @@ +//! The REFERENCE workload — replaceable acceptance data, never ontology. +//! +//! Business nouns (region / month / revenue / memo) exist ONLY in this file, +//! as catalog names and label text at the boundary. The crate under test +//! sees FieldIds and ordinals. +//! +//! Covers: raw → KV / CAM canonicalization; the fold touching neither CAM nor +//! KV; label resolution proportional to presented members; label rename not +//! changing aggregate identity; late hydration of a detail export; Quack +//! parity (the same answer from hand-built Quack programs) with SQL NULL +//! semantics for an empty cell. + +mod common; + +use common::Rng; +use lance_graph_contract::content_store::ContentSink; +use lance_graph_mask_risc::{execute_into, Foreign, LaneRef, Out, Planes, Scratch}; +use lance_graph_quack::{lower, Agg, Cmp, Col, Filter, Query}; +use lance_graph_report::boundary::{BoundaryCounters, CamLabels, Catalog, MemKv}; +use lance_graph_report::render::Terminal; +use lance_graph_report::*; + +const REGIONS: [&str; 8] = [ + "North", "South", "East", "West", "Central", "Coast", "Hills", "Isles", +]; +const MONTHS: [&str; 12] = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", +]; + +struct World { + batch: AbiBatch, + cam: CamLabels, + kv: MemKv, + cat: Catalog, + raw_region: Vec<&'static str>, + raw_month: Vec<&'static str>, + year: Vec, + revenue: Vec, +} + +/// Ingest: raw text → CAM ordinals / KV references, ONCE. Nothing textual +/// reaches the batch. +fn ingest(n: usize) -> World { + let mut r = Rng(0x5eed); + let mut raw_region = Vec::with_capacity(n); + let mut raw_month = Vec::with_capacity(n); + let mut year = Vec::with_capacity(n); + let mut revenue = Vec::with_capacity(n); + let mut memo = Vec::with_capacity(n); + for i in 0..n { + let reg = REGIONS[r.below(8) as usize]; + let y = if reg == "Isles" { + 2025 + } else { + 2025 + r.below(2) as i32 + }; + raw_region.push(reg); + raw_month.push(MONTHS[r.below(12) as usize]); + year.push(y); + revenue.push(r.below(10_000) as i32); + memo.push(format!("order {i} note {}", r.below(1 << 20))); + } + let (fr, fm, fy, fv, fo) = (FieldId(1), FieldId(2), FieldId(3), FieldId(4), FieldId(5)); + let cat = Catalog::default() + .with("region", fr) + .with("month", fm) + .with("year", fy) + .with("revenue", fv) + .with("memo", fo); + let mut cam = CamLabels::default(); + let mut kv = MemKv::default(); + // Canonical label order = first appearance; seed months in calendar order. + for m in MONTHS { + cam.intern(fm, m, &mut kv); + } + for g in REGIONS { + cam.intern(fr, g, &mut kv); + } + let region = cam.canonicalize(fr, raw_region.iter().copied(), &mut kv); + let month = cam.canonicalize(fm, raw_month.iter().copied(), &mut kv); + let memo_refs = kv.persist(memo.iter().map(|s| s.as_bytes())); + let batch = AbiBatch::new(SourceId(9), 4, n) + .with_column(Column::coordinate(fr, region, cam.domain(fr))) + .unwrap() + .with_column(Column::coordinate(fm, month, cam.domain(fm))) + .unwrap() + .with_column(Column::value(fy, LaneData::I32(year.clone().into()))) + .unwrap() + .with_column(Column::value(fv, LaneData::I32(revenue.clone().into()))) + .unwrap() + .with_column(Column::value(fo, LaneData::U64(memo_refs))) + .unwrap(); + World { + batch, + cam, + kv, + cat, + raw_region, + raw_month, + year, + revenue, + } +} + +/// What a UI / SQL adapter does: resolve names ONCE, then build an id-only plan. +fn sales_plan(w: &World) -> ReportPlan { + let f = |n| w.cat.field(n).unwrap(); + ReportPlan::over(SourceRef { + id: SourceId(9), + generation: 4, + }) + .filter(Selection::cmp(f("year"), CmpOp::Eq, Scalar::Int(2026))) + .pivot( + &[CoordSpec::Field(f("region"))], + &[CoordSpec::Field(f("month"))], + ) + .measure(Measure::of(MeasureKind::Sum, f("revenue"))) + .measure(Measure::count()) +} + +fn snapshot(c: &BoundaryCounters) -> [u64; 5] { + [ + &c.kv_puts, + &c.kv_derefs, + &c.cam_lookups, + &c.cam_insertions, + &c.cam_resolutions, + ] + .map(BoundaryCounters::get) +} + +#[test] +fn fold_touches_neither_cam_nor_kv_and_labels_resolve_per_presented_member() { + let w = ingest(100_000); + let plan = sales_plan(&w); + let (cam0, kv0) = (snapshot(&w.cam.counters), snapshot(&w.kv.counters)); + let (res, _) = plan.execute(&w.batch, &PlannerPolicy::default()).unwrap(); + assert_eq!(snapshot(&w.cam.counters), cam0, "the fold did no CAM work"); + assert_eq!(snapshot(&w.kv.counters), kv0, "the fold did no KV work"); + + let t = Terminal { + cam: &w.cam, + kv: &w.kv, + catalog: &w.cat, + }; + let (json, _) = t.json(&res); + let cam1 = snapshot(&w.cam.counters); + let kv1 = snapshot(&w.kv.counters); + // 8 row members + 12 column members, each resolved once — not 100 000. + assert_eq!(cam1[4] - cam0[4], 20); + assert_eq!(kv1[1] - kv0[1], 20); + assert!(json.contains("\"North\"") && json.contains("\"Dec\"")); + assert!(json.contains("SUM(revenue)")); +} + +#[test] +fn quack_parity_and_null_semantics_for_the_reference_pivot() { + let w = ingest(50_000); + let plan = sales_plan(&w); + let (res, _) = plan.execute(&w.batch, &PlannerPolicy::default()).unwrap(); + let (sum_m, cnt_m) = (&res.measures()[0], &res.measures()[1]); + + // Hand-built Quack: per region r, `year = 2026 AND region = r`, + // GROUP BY month SUM(revenue) — executed straight on mask-risc. + let n = w.batch.n_rows(); + let mut alpha = vec![u64::MAX; n.div_ceil(64)]; + if !n.is_multiple_of(64) { + *alpha.last_mut().unwrap() = (1u64 << (n % 64)) - 1; + } + let mref: Vec<&[u64]> = vec![&alpha]; + let region: Vec = w + .raw_region + .iter() + .map(|s| REGIONS.iter().position(|x| x == s).unwrap() as u32) + .collect(); + let month: Vec = w + .raw_month + .iter() + .map(|s| MONTHS.iter().position(|x| x == s).unwrap() as u32) + .collect(); + let lanes = [ + LaneRef::U32(®ion), + LaneRef::U32(&month), + LaneRef::I32(&w.year), + LaneRef::I32(&w.revenue), + ]; + let planes = Planes { + n_rows: w.batch.n_rows(), + masks: &mref, + lanes: &lanes, + }; + for r in 0..8u32 { + let q = Query { + filter: Filter::And(vec![ + Filter::Cmp(Col(2), Cmp::EqI32(2026)), + Filter::Cmp(Col(0), Cmp::EqU32(r)), + ]), + agg: Agg::GroupSumI32 { + key: Col(1), + val: Col(3), + }, + }; + let prog = lower(&q).unwrap(); + let mut out = vec![0i64; 12]; + let mut s = Scratch::for_program(&prog, w.batch.n_rows()).unwrap(); + execute_into(&prog, &planes, &Foreign::NONE, &mut s, Out::I64(&mut out)).unwrap(); + for m in 0..12u32 { + // Independent per-row oracle, from the RAW strings. + let rows: Vec = (0..w.year.len()) + .filter(|&i| { + w.year[i] == 2026 + && w.raw_region[i] == REGIONS[r as usize] + && w.raw_month[i] == MONTHS[m as usize] + }) + .collect(); + let oracle_sum: i64 = rows.iter().map(|&i| i64::from(w.revenue[i])).sum(); + assert_eq!(out[m as usize], oracle_sum, "quack vs oracle"); + let got = res.value(sum_m, &[], &[r], &[m]); + if rows.is_empty() { + // SQL: SUM over no rows is NULL; COUNT is 0. Quack's + // coalescing terminal reads 0 — the report layer pins NULL + // from the COUNT state, identically for every adapter. + assert_eq!(got, CellValue::Null); + assert_eq!(res.value(cnt_m, &[], &[r], &[m]), CellValue::Int(0)); + } else { + assert_eq!(got, CellValue::Int(out[m as usize]), "report vs quack"); + } + } + } + // Anti-vacuity: the empty region (Isles never occurs in 2026) is empty. + assert_eq!(res.row_total(sum_m, &[7]), CellValue::Null); + assert!(matches!(res.row_total(sum_m, &[0]), CellValue::Int(x) if x > 0)); +} + +#[test] +fn renaming_a_label_changes_no_aggregate_identity() { + let mut w = ingest(20_000); + let plan = sales_plan(&w); + let (res, _) = plan.execute(&w.batch, &PlannerPolicy::default()).unwrap(); + let key = plan.physical_key(); + let addr = res.space().payload_addr(); + let fr = w.cat.field("region").unwrap(); + assert!(w.cam.rename(fr, 0, "Nordland", &mut w.kv)); + assert_eq!( + plan.physical_key(), + key, + "the plan holds ordinals, not labels" + ); + let again = res.reinterpret(&plan).unwrap(); + assert_eq!( + again.space().payload_addr(), + addr, + "cached aggregate still valid" + ); + let t = Terminal { + cam: &w.cam, + kv: &w.kv, + catalog: &w.cat, + }; + let (json, _) = t.json(&again); + assert!(json.contains("\"Nordland\"") && !json.contains("\"North\"")); + assert_eq!(w.cam.ordinal(fr, "Nordland"), Some(0)); +} + +#[test] +fn detail_export_filters_ids_first_and_hydrates_only_kept_rows() { + let w = ingest(40_000); + let f = |n| w.cat.field(n).unwrap(); + // An adapter resolves the label literal to its ordinal at the boundary. + let coast = w.cam.ordinal(f("region"), "Coast").unwrap(); + let plan = ReportPlan::over(SourceRef { + id: SourceId(9), + generation: 4, + }) + .filter(Selection::cmp( + f("region"), + CmpOp::Eq, + Scalar::Ordinal(coast), + )) + .filter(Selection::cmp(f("revenue"), CmpOp::Ge, Scalar::Int(9_950))); + let kept = (0..w.year.len()) + .filter(|&i| w.raw_region[i] == "Coast" && w.revenue[i] >= 9_950) + .count(); + assert!(kept > 0 && kept * 100 < w.year.len(), "kept {kept}"); + let (cam0, kv0) = (snapshot(&w.cam.counters), snapshot(&w.kv.counters)); + let t = Terminal { + cam: &w.cam, + kv: &w.kv, + catalog: &w.cat, + }; + let (csv, st) = t + .detail_csv(&plan, &w.batch, &[f("region"), f("revenue"), f("memo")]) + .unwrap(); + assert_eq!(csv.lines().count(), kept + 1); + assert_eq!(st.cells as usize, kept * 3); + // One CAM resolution per kept row (region), one KV deref per kept row for + // the label text plus one for the memo — proportional to OUTPUT rows. + assert_eq!(snapshot(&w.cam.counters)[4] - cam0[4], kept as u64); + assert_eq!(snapshot(&w.kv.counters)[1] - kv0[1], 2 * kept as u64); + assert!(csv + .lines() + .skip(1) + .all(|l| l.starts_with("Coast,") && l.contains(",order "))); +} + +#[test] +fn raw_values_live_in_kv_once_and_duplicates_dedup() { + let mut kv = MemKv::default(); + let refs = kv.persist(["a", "b", "a"].iter().map(|s| s.as_bytes())); + assert_eq!( + refs[0], refs[2], + "content-addressed: one identity per value (S10)" + ); + assert_ne!(refs[0], refs[1]); + let id = kv.put_str("a"); + assert_eq!(id.0, refs[0]); +} diff --git a/crates/lance-graph-report/tests/string_fence.rs b/crates/lance-graph-report/tests/string_fence.rs new file mode 100644 index 000000000..9da722f27 --- /dev/null +++ b/crates/lance-graph-report/tests/string_fence.rs @@ -0,0 +1,62 @@ +//! S1–S4 / S10 source fence: the execution modules contain no text types. +//! +//! Strings are presentation metadata, not execution coordinates. Text may +//! appear only in `boundary.rs` (ingest / KV / CAM / catalog), `render.rs` +//! (terminal) and `explain.rs` (diagnostics). This walks the execution +//! modules' NON-COMMENT lines and rejects any text-type token. + +const EXECUTION: &[&str] = &[ + "ids.rs", + "batch.rs", + "selection.rs", + "plan.rs", + "exec.rs", + "result.rs", +]; +const FORBIDDEN: &[&str] = &[ + "String", + "&str", + "str::", + "format!", + "to_string", + "HashMap Vec { + src.lines() + .enumerate() + .filter(|(_, l)| !l.trim_start().starts_with("//")) + .filter(|(_, l)| FORBIDDEN.iter().any(|t| l.contains(t))) + .map(|(i, l)| format!("{}: {}", i + 1, l.trim())) + .collect() +} + +#[test] +fn execution_modules_are_string_free() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/"); + let mut bad = Vec::new(); + for f in EXECUTION { + let src = std::fs::read_to_string(format!("{dir}{f}")).unwrap(); + bad.extend(violations(&src).into_iter().map(|v| format!("{f}:{v}"))); + } + assert!( + bad.is_empty(), + "text types in the execution core:\n{}", + bad.join("\n") + ); +} + +#[test] +fn the_fence_can_fire() { + // Disable-verified shape: a line an earlier draft of this crate really + // had (`labels: Option>` on a dimension) is caught, and a + // comment mentioning String is not. + assert_eq!( + violations(" pub labels: Option>,").len(), + 1 + ); + assert_eq!(violations(" Label(String),").len(), 1); + assert!(violations(" /// a String is presentation metadata").is_empty()); + assert!(violations(" pub field: FieldId,").is_empty()); +} diff --git a/crates/lance-graph-report/tests/zero_copy.rs b/crates/lance-graph-report/tests/zero_copy.rs new file mode 100644 index 000000000..e85159df6 --- /dev/null +++ b/crates/lance-graph-report/tests/zero_copy.rs @@ -0,0 +1,105 @@ +//! Allocation falsifiers: what scales with the POPULATION, and what doesn't. +//! +//! A per-thread counting allocator measures bytes allocated inside a closure. +//! Two populations 64× apart must allocate the SAME bytes for execution (F8: +//! memory scales with result cardinality, never with rows) and for rotation +//! (F6/F7/A4/A5: rotation is metadata). Lanes are checked by address before +//! and after (F11: no population copy). + +mod common; + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; + +use common::*; +use lance_graph_report::*; + +struct Counting; +thread_local! { + static BYTES: Cell = const { Cell::new(0) }; +} +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + BYTES.with(|b| b.set(b.get() + l.size() as u64)); + // SAFETY: forwarded verbatim to the system allocator. + unsafe { System.alloc(l) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + // SAFETY: forwarded verbatim to the system allocator. + unsafe { System.dealloc(p, l) } + } +} +#[global_allocator] +static A: Counting = Counting; + +fn allocated(f: impl FnOnce() -> T) -> (T, u64) { + let before = BYTES.with(Cell::get); + let t = f(); + (t, BYTES.with(Cell::get) - before) +} + +const RA: FieldId = FieldId(0); +const RB: FieldId = FieldId(1); +const V: FieldId = FieldId(100); + +fn pivot() -> ReportPlan { + with_four( + ReportPlan::over(src()) + .filter(Selection::cmp(V, CmpOp::Gt, Scalar::Int(-20))) + .pivot(&[CoordSpec::Field(RA)], &[CoordSpec::Field(RB)]), + V, + ) +} + +#[test] +fn execution_memory_is_independent_of_population_size() { + let small = synthetic(4_096, &[6, 9], 31).batch(); + let big = synthetic(4_096 * 64, &[6, 9], 31).batch(); + let pol = PlannerPolicy { + reuse_mask_min_programs: u64::MAX, + ..PlannerPolicy::default() + }; + let addr = big.lane_addr(V).unwrap(); + let ((rs, _), a_small) = allocated(|| pivot().execute(&small, &pol).unwrap()); + let ((rb, sb), a_big) = allocated(|| pivot().execute(&big, &pol).unwrap()); + assert_eq!( + a_small, a_big, + "execution allocates the same bytes at N and 64·N" + ); + assert_eq!(big.lane_addr(V).unwrap(), addr, "source lane never moved"); + assert_eq!(sb.mask_materializations, 0); + assert!( + sb.scratch_bytes_peak <= 8 * 8 * 8, + "scratch is tile-sized: {}", + sb.scratch_bytes_peak + ); + assert_eq!(rs.space().stored_cells(), rb.space().stored_cells()); + // Can-fire twin: the reused-mask carrier IS population-sized, and the + // same probe sees it. + let reuse = PlannerPolicy::default(); + let (_, r_small) = allocated(|| pivot().execute(&small, &reuse).unwrap()); + let (_, r_big) = allocated(|| pivot().execute(&big, &reuse).unwrap()); + assert_eq!( + r_big - r_small, + ((4_096 * 64 - 4_096) / 64 * 8) as u64, + "exactly the mask's growth" + ); +} + +#[test] +fn rotation_and_role_changes_allocate_no_population_or_cell_bytes() { + for n in [4_096usize, 4_096 * 64] { + let batch = synthetic(n, &[6, 9], 32).batch(); + let (res, _) = pivot().execute(&batch, &PlannerPolicy::default()).unwrap(); + let rot_plan = pivot().rotate(); + let (rot, bytes) = allocated(|| res.rotate()); + let (rot2, bytes2) = allocated(|| res.reinterpret(&rot_plan).unwrap()); + assert!( + bytes < 64 && bytes2 < 256, + "view metadata only: {bytes} / {bytes2}" + ); + assert!(bytes < res.space().accumulator_bytes() as u64); + assert_eq!(rot.space().payload_addr(), res.space().payload_addr()); + assert_eq!(rot2.space().payload_addr(), res.space().payload_addr()); + } +}