diff --git a/.claude/board/entries/2026-10-05-text-to-numeric-boundary-inventory.md b/.claude/board/entries/2026-10-05-text-to-numeric-boundary-inventory.md new file mode 100644 index 000000000..a41fd7ff5 --- /dev/null +++ b/.claude/board/entries/2026-10-05-text-to-numeric-boundary-inventory.md @@ -0,0 +1,31 @@ +# 2026-10-05 — text → numeric id → Quack → mask-risc: one pattern, four id semantics + +## MEASURED — who runs the pattern (production callers traced) + +| consumer | text in | binder | numeric id | Quack in prod? | final primitive | text during execution | +|---|---|---|---|---|---|---| +| lance-graph-report | field name, categorical label | `Catalog` → `FieldId`; `CamLabels::ordinal` (CAM over `ContentId`) | `FieldId`, per-field CAM ordinal | yes (`exec.rs` calls `lance_graph_quack::lower`) | `Pred::EqU32` via `Cmp::EqU32` | none (`string_fence.rs`; counters in `reference_workload.rs`) | +| lance-graph-sap | CATS columns | `bind.rs` first-occurrence dict per batch (forward map discarded) | batch-local `u32` code, 0 = NULL; employee via `numc` (parsed, not dict) | yes (`CatsQuery::prepare`) | `EqU32` / `GeI32` / `LeI32` + `GroupSumI32` | none; reverse labels kept for egress only | +| lance-graph-java (lgj-abi) | none (static Java field constants) | compile-time `LaneId` | lane index + i64 operand | **no** — `plan_lower.rs` (quack is a dev-dependency) | same `Pred` vocabulary (`EqU32`, `GtI32`, `MatchU32`, …) | none | +| lance-graph-dir-sim | UPN / SMTP, query literal | `Dicts::intern` / `key_lookup` | store-lifetime `ValueId` / `KeyId` | yes | `Pred::EqU32`, `MatchFacet16Strided`, `Gather` | none (`string_fence.rs`, `where_eq.rs`) | + +Java's `lowering_convergence` compares ANSWERS (row counts / group sums) over 28 combine vectors and 9 opcodes, not Programs. Moving Java onto Quack would be convergence, not invention. + +## FINDING — ids are not interchangeable + +- **`ContentId`** (contract): content identity, fnv1a64 of exact bytes. Process-independent, survives reopen, legal anywhere. It is u64, and it has no normalization. +- **report CAM ordinal**: a per-field category. `rename` re-points the label and keeps the ordinal. That is presentation identity, safe because a coordinate means the category, not the text. +- **report `FieldId`**: schema/catalog identity. +- **SAP code**: one bound batch only (`CatsQuery` borrows the batch). It cannot be persisted, and a text literal cannot be resolved to it after binding. +- **dir-sim `ValueId`**: the exact text, shared across attributes, append-only for the store's lifetime. A different text is a different value, and changing it is a `SetAttribute`. It is persisted in `Change` / `ExecutionPlan` / `Precondition`. +- **dir-sim `KeyId`**: the normalized comparison form, with the same lifetime. It is used only for comparison: uniqueness and rename ordering. +- **OGAR `ValuePool` `StrRef`**: a per-batch byte offset, not deduplicated. It is not an identity. +- **Quack `Col`**: a lane position inside one `Planes`. + +Consequence: replacing `ValueId` with the CAM ordinal would be wrong. A CAM rename keeps the id and changes the text, so a plan's compare-and-set would silently target a different string. + +## OPEN + +- Should `ValueId` converge onto `ContentId` (content identity, reopen-stable)? That would need u64 lanes and a collision policy. Decision pending. +- SAP has no query-time literal → code path, because its forward map is dropped at bind. +- Java production still lowers outside Quack. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index a37e51eb6..049bc5ed4 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,11 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -213 entries, 2026-08-06 .. 2026-10-04. +214 entries, 2026-08-06 .. 2026-10-05. | date | entry id | finding | file | |---|---|---|---| +| 2026-10-05 | `text-to-numeric-boundary-inventory` | | [2026-10-05-text-to-numeric-boundary-inventory.md](2026-10-05-text-to-numeric-boundary-inventory.md) | | 2026-10-04 | `D-LXC-22` | | [2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md](2026-10-04-wordnet-clam-chaoda-scope-and-grammar-read-params.md) | | 2026-10-04 | `D-HPS-1` | | [2026-10-04-spog-slab-hotplug-resolution.md](2026-10-04-spog-slab-hotplug-resolution.md) | | 2026-10-04 | `D-HPS-2` | | [2026-10-04-resolve-once-population-execution.md](2026-10-04-resolve-once-population-execution.md) | diff --git a/crates/lance-graph-dir-sim/src/exec.rs b/crates/lance-graph-dir-sim/src/exec.rs index d7a49f0e9..f4f45098e 100644 --- a/crates/lance-graph-dir-sim/src/exec.rs +++ b/crates/lance-graph-dir-sim/src/exec.rs @@ -58,6 +58,12 @@ impl Kept { materialize_rows(&self.bits, self.n_rows) } + /// Add one row (a delta-sized correction, e.g. an overridden value). + pub(crate) fn set(&mut self, row: usize) { + debug_assert!(row < self.n_rows); + self.bits[row / 64] |= 1 << (row % 64); + } + /// `self`'s rows followed by `tail`'s rows offset by `self`'s width — /// one result over a base relation and its delta rows. pub(crate) fn concat(mut self, tail: &Kept) -> Kept { diff --git a/crates/lance-graph-dir-sim/src/lib.rs b/crates/lance-graph-dir-sim/src/lib.rs index c01508cc3..86dbd1d0a 100644 --- a/crates/lance-graph-dir-sim/src/lib.rs +++ b/crates/lance-graph-dir-sim/src/lib.rs @@ -7,14 +7,17 @@ //! OGAR owns the meaning (`ogar-dir-sim`: `Change`, provenance, `Violation`, //! `ExecutionPlan`); this crate owns execution: //! -//! * [`snapshot`] — one observation as SoA lanes and bit planes, `Guid128` -//! sorted so the ordinal is the index; strings in store dictionaries. +//! * [`snapshot`] — one observation as two populations (users, groups) of +//! SoA lanes and bit planes, each at most 65,536 nodes with its own `u16` +//! ordinal space; sparse `user × group` membership; [`Dn128`] hierarchy; +//! strings only in the store's cold label/value table, as [`ValueId`]s. //! * [`view`] — a version = shared `Arc` + delta-sized overlay. //! * [`rule`] — pure population rules over a borrowed [`View`]. //! * [`validate`] — invariants as Quack programs (`Semijoin` anti-joins, //! `GroupReduce` counts). //! * [`store`] — append-only versions, tags, diff, plan, audit. -//! * [`observe`] — `ogar-ad` records → observation. +//! * [`observe`] — `ogar-ad` records → observation (`OuHhtl` → `Dn128`, +//! failing closed). //! //! No network, process or file I/O. Nothing writes to AD, Entra, Exchange, //! LDAP or PowerShell. `#![forbid(unsafe_code)]`. @@ -30,81 +33,151 @@ pub mod validate; pub mod view; pub use exec::Kept; +pub use ogar_dir_sim::{KeyId, ValueId}; pub use rule::{member_counts, GrantGroup, ImplyGroup, Rule, SetPrimarySmtp}; pub use snapshot::{ - pack_ou, BuildError, Dict, Dicts, NodeKind, Observation, ObservedNode, Snapshot, NONE, + BuildError, Dict, DictCounters, Dicts, GroupOrdinal, NodeKind, Observation, ObservedNode, + Population, Snapshot, UserOrdinal, MAX_GROUPS, MAX_USERS, NONE, }; pub use store::{Rejection, SimError, VersionStore}; pub use view::{ApplyError, View}; -use lance_graph_mask_risc::{Foreign, LaneRef, Planes}; +use lance_graph_mask_risc::{words_for, Foreign, LaneRef, Planes, Program, StridedRef}; use lance_graph_quack::{Cmp, Col, Filter, Mask}; -use ogar_dir_core::OuHhtl; +use ogar_dir_core::{DirectoryScope, Dn128}; +use ogar_dir_sim::Attribute; -/// A subtree prefix deeper than the packed lane (4 levels) can address. +/// The program behind [`users_with_key`]: plane 0 = candidate users, +/// lane 0 = an attribute's key lane, one `EqU32` on the key. It holds only +/// numbers: the literal was resolved before it was built. +pub fn key_eq_program(key: KeyId) -> Program { + exec::program( + Filter::and([ + Filter::plane(Mask(0)), + Filter::cmp(Col(0), Cmp::EqU32(key.0)), + ]), + lance_graph_quack::Agg::Rows, + ) +} + +/// `WHERE = ` over the existing users of a version, as +/// a bitmap over user ordinals. The literal arrives as a [`KeyId`] — +/// resolved once at the boundary ([`Dicts::key_lookup`]) — so execution is +/// one [`key_eq_program`] over the base key lane (overridden and deleted +/// users gated out), the same program over the created users' key lane, +/// and a delta-sized check of the overrides. No string is read. +pub fn users_with_key(v: &View<'_>, a: Attribute, key: KeyId) -> Kept { + let p = key_eq_program(key); + let s = v.snap; + let (pop, ov) = v.pop(NodeKind::User); + let base_key = match a { + Attribute::Upn => &pop.upn_key, + Attribute::PrimarySmtp => &pop.smtp_key, + }; + let mut live = v.base_live(NodeKind::User, &pop.all).into_owned(); + for o in ov.overrides(a).keys() { + snapshot::clear_bit(&mut live, usize::from(*o)); + } + let run = |plane: &[u64], lane: &[u32]| { + let lanes = [LaneRef::U32(lane)]; + let masks: [&[u64]; 1] = [plane]; + exec::keep( + &p, + &Planes { + n_rows: lane.len(), + masks: &masks, + lanes: &lanes, + }, + &Foreign::NONE, + ) + }; + let mut base = run(&live, base_key); + for (o, (_, k)) in ov.overrides(a) { + if *k == key.0 { + base.set(usize::from(*o)); + } + } + debug_assert_eq!(base_key.len(), s.users.len()); + let created = ov.created.key(a); + base.concat(&run(&snapshot::ones(created.len()), created)) +} + +/// A subtree query in a directory the version does not describe. Codes are +/// only meaningful under their scope, so the query is refused, not run. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct SubtreeTooDeep(pub usize); +pub struct ScopeMismatch { + /// The version's scope. + pub version: DirectoryScope, + /// The query's scope. + pub query: DirectoryScope, +} -/// Nodes located in the OU subtree `prefix` (ancestor-or-self), as a node -/// bitmap over the version's ordinals — one ternary match on the packed OU -/// lane (`Cmp::MatchU64`), no DN strings. The same program runs over the -/// base lane (deleted nodes gated out) and over the created nodes' lane -/// (delta-sized); the two kept sets are concatenated, never fed onward. -/// Prefixes up to depth 4 are exact; deeper ones are refused. -pub fn subtree(v: &View<'_>, prefix: &OuHhtl) -> Result { - let d = prefix.depth(); - if d > 4 { - return Err(SubtreeTooDeep(d)); +/// Nodes of `kind` located in the subtree of `prefix` (ancestor-or-self), +/// as a bitmap over that population's ordinals. +/// +/// One program, no strings: `located ∧ depth ≥ d ∧ dn ⊇ prefix`, where the +/// last term is a 16-byte ternary match (`Cmp::MatchFacet16Strided`) read +/// in place over the population's `[[u8; 16]]` lane. The depth gate is what +/// keeps a shallower node whose zero tail happens to equal the prefix out. +/// The same program runs over the base lane (deleted nodes gated out) and +/// over the created nodes' lane (delta-sized); the two kept sets are +/// concatenated, never fed onward. +pub fn subtree( + v: &View<'_>, + scope: DirectoryScope, + kind: NodeKind, + prefix: &Dn128, +) -> Result { + if scope != v.snap.scope { + return Err(ScopeMismatch { + version: v.snap.scope, + query: scope, + }); } - let care = if d == 0 { 0 } else { u64::MAX << (64 - 16 * d) }; + let (pattern, care) = prefix.subtree_mask(); let p = exec::program( Filter::and([ Filter::plane(Mask(0)), - Filter::cmp( - Col(0), - Cmp::MatchU64 { - pattern: pack_ou(prefix), - care, - }, - ), + Filter::cmp(Col(0), Cmp::GeI32(prefix.depth() as i32)), + Filter::cmp(Col(1), Cmp::MatchFacet16Strided { pattern, care }), ]), lance_graph_quack::Agg::Rows, ); - let s = v.snap; - let present = v.base_live(&s.ou_present); - let lanes = [LaneRef::U64(&s.ou_hi)]; - let masks: [&[u64]; 1] = [&present]; - let base = exec::keep( - &p, - &Planes { - n_rows: s.len(), - masks: &masks, - lanes: &lanes, - }, - &Foreign::NONE, - ); - let cr = &v.ov.created; - let packed: Vec = cr - .ou - .iter() - .map(|o| o.as_ref().map_or(0, pack_ou)) - .collect(); - let mut located = vec![0u64; lance_graph_mask_risc::words_for(cr.ou.len())]; - for (i, o) in cr.ou.iter().enumerate() { - if o.is_some() { + let run = |present: &[u64], depth: &[i32], dn: &[[u8; 16]]| { + let lanes = [ + LaneRef::I32(depth), + LaneRef::Strided(StridedRef { + bytes: dn.as_flattened(), + first_offset: 0, + stride: 16, + records: dn.len(), + }), + ]; + let masks: [&[u64]; 1] = [present]; + exec::keep( + &p, + &Planes { + n_rows: dn.len(), + masks: &masks, + lanes: &lanes, + }, + &Foreign::NONE, + ) + }; + let (pop, ov) = v.pop(kind); + let present = v.base_live(kind, &pop.dn_present); + let base = run(&present, &pop.dn_depth, &pop.dn); + + let cr = &ov.created.dn; + let mut located = vec![0u64; words_for(cr.len())]; + let (mut depth, mut dn) = (Vec::with_capacity(cr.len()), Vec::with_capacity(cr.len())); + for (i, d) in cr.iter().enumerate() { + if d.is_some() { located[i / 64] |= 1 << (i % 64); } + let d = d.unwrap_or(Dn128::ROOT); + depth.push(d.depth() as i32); + dn.push(d.bytes()); } - let lanes = [LaneRef::U64(&packed)]; - let masks: [&[u64]; 1] = [&located]; - let created = exec::keep( - &p, - &Planes { - n_rows: cr.ou.len(), - masks: &masks, - lanes: &lanes, - }, - &Foreign::NONE, - ); - Ok(base.concat(&created)) + Ok(base.concat(&run(&located, &depth, &dn))) } diff --git a/crates/lance-graph-dir-sim/src/observe.rs b/crates/lance-graph-dir-sim/src/observe.rs index a9d831a59..529bf209a 100644 --- a/crates/lance-graph-dir-sim/src/observe.rs +++ b/crates/lance-graph-dir-sim/src/observe.rs @@ -3,13 +3,22 @@ //! The ingestion boundary: values are read out of the record's value pool //! once and handed to [`Snapshot::build`](crate::Snapshot::build) for //! interning. "Active" is derived from `userAccountControl` bit `0x2` -//! (ACCOUNTDISABLE); the primary SMTP is the `SMTP:` proxy; the OU-HHTL is -//! taken as-is (never a DN string). Memberships are relations that `ogar-ad` -//! records do not carry; the caller adds observed ones. +//! (ACCOUNTDISABLE); the primary SMTP is the `SMTP:` proxy; the location is +//! the record's `OuHhtl` (the ingress wire format) converted to a [`Dn128`], +//! never a DN string. A parent with more than 256 children cannot be a +//! `Dn128` and the whole observation is refused — never hashed or truncated. +//! Memberships are relations that `ogar-ad` records do not carry; the +//! caller adds observed ones. +//! +//! **Open, recorded, not decided here:** a user record with no +//! `userAccountControl` value is read as **enabled** (`is_none_or`). Whether +//! an absent flag should mean enabled, disabled or "unknown" — and how an AD +//! and an Entra observation of the same person are merged — is an open +//! policy question; this module does not choose for it. use crate::snapshot::{NodeKind, Observation, ObservedNode}; use ogar_ad::{AdKind, SCHEMA_V1}; -use ogar_dir_core::{DirRecord, ValuePool}; +use ogar_dir_core::{DirRecord, DirectoryScope, Dn128, Dn128Error, Guid128, ValuePool}; const UAC_ACCOUNTDISABLE: u32 = 0x2; @@ -21,15 +30,45 @@ fn slot(name: &str) -> usize { .expect("ogar-ad schema v1") } -/// Users and groups among `records` (other kinds are skipped). -pub fn from_ad(records: &[DirRecord], pool: &ValuePool) -> Observation { +/// Why a record could not be observed. Either way the whole observation is +/// refused: nothing is silently dropped or merged. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ObserveError { + /// The record's `OuHhtl` is not a `Dn128` (e.g. a 257th child). + Location { + /// The record. + node: Guid128, + /// Why. + error: Dn128Error, + }, + /// The record belongs to another directory than the observation. A + /// `Dn128` carries no scope (it is external context), so a record from + /// another domain or tenant would be indistinguishable from a local one. + ForeignScope { + /// The record. + node: Guid128, + /// The record's own scope. + scope: DirectoryScope, + }, +} + +/// Users and groups among `records` (other kinds are skipped), located in +/// `scope`. Every user or group record must carry that scope. +pub fn from_ad( + scope: DirectoryScope, + records: &[DirRecord], + pool: &ValuePool, +) -> Result { let text = |r: &DirRecord, name: &str| { r.str_ref(slot(name)) .and_then(|s| pool.get(s)) .and_then(|b| std::str::from_utf8(b).ok()) .map(str::to_string) }; - let mut obs = Observation::default(); + let mut obs = Observation { + scope, + ..Observation::default() + }; for r in records { let kind = match r.object_kind() { k if k == AdKind::User as u16 => NodeKind::User, @@ -44,16 +83,28 @@ pub fn from_ad(records: &[DirRecord], pool: &ValuePool) -> Observation { .filter_map(|v| std::str::from_utf8(v).ok()) .find_map(|v| v.strip_prefix("SMTP:").map(str::to_string)) }); + let node = r.node_guid(); + if r.scope_guid() != scope.0 { + return Err(ObserveError::ForeignScope { + node, + scope: DirectoryScope(r.scope_guid()), + }); + } + let dn = r + .ou_hhtl() + .map(|h| Dn128::from_ou_hhtl(&h)) + .transpose() + .map_err(|error| ObserveError::Location { node, error })?; obs.nodes.push(( - r.node_guid(), + node, ObservedNode { kind, active: r.num(0).is_none_or(|uac| uac & UAC_ACCOUNTDISABLE == 0), upn: text(r, "userPrincipalName"), primary_smtp, - ou: r.ou_hhtl(), + dn, }, )); } - obs + Ok(obs) } diff --git a/crates/lance-graph-dir-sim/src/rule.rs b/crates/lance-graph-dir-sim/src/rule.rs index e09ef99fb..4b739bd84 100644 --- a/crates/lance-graph-dir-sim/src/rule.rs +++ b/crates/lance-graph-dir-sim/src/rule.rs @@ -3,15 +3,17 @@ //! A rule receives a [`View`] — borrowed lanes of one version — and returns //! the [`Change`]s it proposes. It has no I/O handle and no way to mutate the //! view. Rules select **populations**: a population is a bitmap over the -//! version's node ordinals, produced by Quack programs, never a loop that -//! runs a workflow per user. +//! version's user ordinals, produced by Quack programs, never a loop that +//! runs a workflow per user. Values are [`ValueId`]s: a rule never compares +//! or builds a string. use crate::exec::group_count_into; use crate::view::View; use lance_graph_mask_risc::{Foreign, LaneRef, Planes}; use lance_graph_quack::{Cmp, Col, Filter, Mask}; use ogar_dir_core::Guid128; -use ogar_dir_sim::{Attribute, Change, EvidenceRef, RuleId}; +use ogar_dir_sim::NodeKind; +use ogar_dir_sim::{Attribute, Change, EvidenceRef, RuleId, ValueId}; /// A pure graph transformation. pub trait Rule { @@ -21,16 +23,27 @@ pub trait Rule { fn propose(&self, v: &View<'_>, evidence: &[EvidenceRef]) -> Vec; } -/// Per-node membership count in `group` for one version: -/// `GROUP BY member COUNT(*) WHERE group = g` over the live base relation, -/// plus the overlay's added rows (delta-sized). One `K = |nodes|` sink; no -/// joined rows. +/// Per-user membership count in `group` for one version: +/// `GROUP BY user COUNT(*) WHERE group = g` over the live base relation, +/// plus the overlay's added rows (delta-sized). One `K = |users|` sink, +/// indexed by user ordinal; no joined rows. pub fn member_counts(v: &View<'_>, group: &Guid128) -> Vec { let s = v.snap; - let mut counts = vec![0i64; v.len()]; - let Some(g) = v.ordinal(group) else { + let mut counts = vec![0i64; v.users_len()]; + let Some(g) = v.group_ordinal(group) else { return counts; }; + let g = u32::from(g.0); + let added = v.added_rows(); + for (uo, go) in added.users.iter().zip(&added.groups) { + if *go == g { + counts[*uo as usize] += 1; + } + } + // A created group has no base rows. + if g as usize >= s.groups.len() { + return counts; + } let live = v.live_rows(); let lanes = [LaneRef::U32(&s.m_user), LaneRef::U32(&s.m_group)]; let masks: [&[u64]; 1] = [&live]; @@ -46,12 +59,6 @@ pub fn member_counts(v: &View<'_>, group: &Guid128) -> Vec { &Foreign::NONE, &mut counts, ); - let (au, ag, _) = v.added_rows(); - for (uo, go) in au.into_iter().zip(ag) { - if go == g && (uo as usize) < counts.len() { - counts[uo as usize] += 1; - } - } counts } @@ -121,9 +128,9 @@ impl Rule for ImplyGroup { member_counts(v, &self.target), ); let active = v.active_users(); - (0..v.len()) - .filter(|&o| active[o / 64] >> (o % 64) & 1 == 1 && cs[o] > 0 && ct[o] == 0) - .filter_map(|o| v.guid(o as u32)) + (0..v.users_len()) + .filter(|&o| crate::snapshot::bit(&active, o) && cs[o] > 0 && ct[o] == 0) + .filter_map(|o| v.guid_in(NodeKind::User, o)) .map(|user| Change::AddMembership { user, group: self.target, @@ -138,8 +145,9 @@ pub struct SetPrimarySmtp { pub rule: RuleId, /// User. pub user: Guid128, - /// New address (raw). - pub to: String, + /// New address, interned by the caller at ingress + /// ([`VersionStore::intern`](crate::VersionStore::intern)). + pub to: ValueId, } impl Rule for SetPrimarySmtp { @@ -147,18 +155,18 @@ impl Rule for SetPrimarySmtp { self.rule } fn propose(&self, v: &View<'_>, _: &[EvidenceRef]) -> Vec { - let Some(o) = v.ordinal(&self.user) else { + if v.user_ordinal(&self.user).is_none() { return Vec::new(); - }; - let from = v.attr(o, Attribute::PrimarySmtp); - if from == Some(self.to.as_str()) { + } + let from = v.attr(&self.user, Attribute::PrimarySmtp); + if from == Some(self.to) { return Vec::new(); } vec![Change::SetAttribute { node: self.user, attribute: Attribute::PrimarySmtp, - from: from.map(str::to_string), - to: Some(self.to.clone()), + from, + to: Some(self.to), }] } } diff --git a/crates/lance-graph-dir-sim/src/snapshot.rs b/crates/lance-graph-dir-sim/src/snapshot.rs index 44e557fa9..701dc2aeb 100644 --- a/crates/lance-graph-dir-sim/src/snapshot.rs +++ b/crates/lance-graph-dir-sim/src/snapshot.rs @@ -1,36 +1,62 @@ //! One observed directory state as SoA lanes — never as objects. //! -//! | lane / plane | type | meaning | -//! |-------------------------|-------------|---------------------------------------------| -//! | `ids` | `[Guid128]` | sorted; a node's **ordinal** is its index | -//! | `user`, `group` | bit planes | node kind | -//! | `active_user` | bit plane | user ∧ enabled — the recipient population | -//! | `upn_val` / `smtp_val` | `[u32]` | raw value id in [`Dicts::values`] | -//! | `upn_key` / `smtp_key` | `[u32]` | normalized key id in [`Dicts::keys`] | -//! | `ou` | `[OuHhtl]` | exact OU location (node state, compare-and-set) | -//! | `ou_hi`, `ou_present` | `[u64]`, plane | OU-HHTL levels 0..4 packed (subtree = prefix match) | -//! | `m_user`, `m_group` | `[u32]` | membership relation, sorted by (user, group) | +//! Users and groups are two independent populations, each at most +//! [`MAX_USERS`] / [`MAX_GROUPS`] = 65,536 nodes and each with its own dense +//! ordinal space ([`UserOrdinal`], [`GroupOrdinal`], both `u16`; every value +//! of a `u16` is a real ordinal, none is reserved). +//! +//! Per population ([`Population`]): +//! +//! | lane / plane | type | meaning | +//! |---------------------------|---------------|------------------------------------------| +//! | `ids` | `[Guid128]` | sorted; a node's **ordinal** is its index | +//! | `active` | bit plane | enabled (users); every group | +//! | `upn_val` / `smtp_val` | `[u32]` | [`ValueId`] in the store's value table | +//! | `upn_key` / `smtp_key` | `[u32]` | [`KeyId`] (comparison form) | +//! | `dn` | `[[u8; 16]]` | [`Dn128`] codes, read in place (strided) | +//! | `dn_depth`, `dn_present` | `[i32]`, plane | hierarchy depth; whether a location is known | +//! +//! Membership is sparse — `UserOrdinal × GroupOrdinal` rows, never a dense +//! matrix — as two lanes `m_user`, `m_group` sorted by `(user, group)`. The +//! values are `u16` ordinals; the lanes are 32-bit because the substrate has +//! no 16-bit lane. A row whose endpoint does not resolve is kept as +//! identities in `m_unresolved` and never enters the lanes, so no lane value +//! stands for "missing". //! //! **Ordinal ≠ identity.** An ordinal is valid only inside one snapshot and -//! is never stored in provenance, diffs or plans; those carry [`Guid128`]. -//! Ordinals come from sorting by `Guid128`, so they — and every dictionary -//! id assigned while building — are independent of ingestion order. +//! never reaches provenance, diffs or plans; those carry [`Guid128`]. +//! Ordinals come from sorting by `Guid128`, so they — and every id interned +//! while building — are independent of ingestion order. //! //! [`Observation`] is the ingestion boundary: it owns its strings once, and -//! [`Snapshot::build`] interns them into the store's dictionaries. After -//! that, execution works on ids and masks; a string is resolved only to -//! report evidence or to compare a compare-and-set value. +//! [`Snapshot::build`] interns them into the store's [`Dicts`] (the cold +//! label/value store). After that, execution works on ids, ordinals and +//! masks; a string is resolved only to report or to actuate. use lance_graph_mask_risc::words_for; -use ogar_dir_core::{Guid128, OuHhtl}; -use ogar_dir_sim::normalize; +use ogar_dir_core::{DirectoryScope, Dn128, Guid128}; +use ogar_dir_sim::{normalize, KeyId, ValueId}; use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub use ogar_dir_sim::NodeKind; + +/// Most users one directory population may hold. +pub const MAX_USERS: usize = 65_536; +/// Most groups one directory population may hold. +pub const MAX_GROUPS: usize = 65_536; -/// "No value" / "unresolved endpoint" sentinel. Out of range for every -/// lane, so mask-risc's zero-fallback treats it as matching nothing. +/// "No value" in a value or key lane. Value and key ids are store-issued and +/// never reach this value (the store refuses to grow that far). pub const NONE: u32 = u32::MAX; -pub use ogar_dir_sim::NodeKind; +/// Dense position of a user inside one snapshot (or one version). +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct UserOrdinal(pub u16); + +/// Dense position of a group inside one snapshot (or one version). +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct GroupOrdinal(pub u16); /// Append-only string dictionary. Ids are assignment order; nothing is /// iterated in hash order. @@ -46,7 +72,10 @@ impl Dict { if let Some(&i) = self.index.get(s) { return i; } - let i = u32::try_from(self.strings.len()).expect("dictionary fits u32"); + let i = u32::try_from(self.strings.len()) + .ok() + .filter(|&i| i != NONE) + .expect("dictionary below u32::MAX entries"); self.strings.push(s.to_string()); self.index.insert(s.to_string(), i); i @@ -59,7 +88,7 @@ impl Dict { pub fn resolve(&self, id: u32) -> Option<&str> { self.strings.get(id as usize).map(String::as_str) } - /// Number of entries (the group universe of a key `GROUP BY`). + /// Number of entries. pub fn len(&self) -> usize { self.strings.len() } @@ -69,33 +98,107 @@ impl Dict { } } -/// Value storage shared by every snapshot and version of one store. +/// Boundary counters, as in `lance-graph-report`'s boundary: every text +/// operation is counted, so a test can prove execution did none. +/// Relaxed atomics: diagnostics, not synchronization. +#[derive(Debug, Default)] +pub struct DictCounters { + /// Text → id, minting if new (ingress). + pub interns: AtomicU64, + /// Text → id, never minting (resolving a query literal). + pub lookups: AtomicU64, + /// Id → text (egress). + pub resolutions: AtomicU64, +} + +impl DictCounters { + /// `[interns, lookups, resolutions]`. + pub fn snapshot(&self) -> [u64; 3] { + [&self.interns, &self.lookups, &self.resolutions].map(|c| c.load(Ordering::Relaxed)) + } +} + +fn bump(c: &AtomicU64) { + c.fetch_add(1, Ordering::Relaxed); +} + +/// The cold label/value store shared by every snapshot and version of one +/// [`VersionStore`](crate::VersionStore): the only place strings live. +/// Append-only, so a [`ValueId`] / [`KeyId`] keeps its meaning for the +/// store's lifetime — across observations, simulations, the desired +/// version, reconciliation and plans. +/// +/// Identity, compared with the other boundaries in this workspace: a +/// [`ValueId`] **is** its exact text (a different text is a different +/// value — changing it is a `SetAttribute`, not a relabel), shared by every +/// attribute, valid for the store's lifetime. That is not +/// `lance-graph-report`'s CAM ordinal, which is per field and whose label +/// can be renamed under a fixed ordinal; and not a batch-local dictionary +/// code (`lance-graph-sap`), which cannot appear in a plan. #[derive(Debug, Default)] pub struct Dicts { - /// Raw values exactly as observed. - pub values: Dict, - /// Normalized comparison keys. - pub keys: Dict, + /// Raw values exactly as observed or requested. + values: Dict, + /// Comparison forms ([`normalize`]). + keys: Dict, + /// `key_of[value] = key`, computed once at interning. + key_of: Vec, + /// Text operations performed. + pub counters: DictCounters, } impl Dicts { - /// `(value id, key id)` for an optional raw value. - pub fn intern_attr(&mut self, s: Option<&str>) -> (u32, u32) { - match s { - None => (NONE, NONE), - Some(s) => (self.values.intern(s), self.keys.intern(&normalize(s))), + /// Ingress: the id of a raw value, interning it (and its comparison + /// key) if new. + pub fn intern(&mut self, s: &str) -> ValueId { + bump(&self.counters.interns); + let v = self.values.intern(s); + if v as usize == self.key_of.len() { + self.key_of.push(self.keys.intern(&normalize(s))); } + ValueId(v) + } + /// The id of a raw value, if this store ever saw it. Never mints. + pub fn lookup(&self, s: &str) -> Option { + bump(&self.counters.lookups); + self.values.get(s).map(ValueId) + } + /// The comparison key of a text (normalized here), if this store ever + /// saw a value with that key. Never mints. This is how a query literal + /// such as `smtp = 'Alice@X.de'` becomes a number, once, before lowering. + pub fn key_lookup(&self, s: &str) -> Option { + bump(&self.counters.lookups); + self.keys.get(&normalize(s)).map(KeyId) + } + /// Egress: the raw value behind an id. + pub fn value(&self, v: ValueId) -> Option<&str> { + bump(&self.counters.resolutions); + self.values.resolve(v.0) + } + /// The comparison key of a value. + pub fn key_of(&self, v: ValueId) -> Option { + self.key_of.get(v.0 as usize).map(|&k| KeyId(k)) + } + /// Egress: the comparison form behind a key. + pub fn key_label(&self, k: KeyId) -> Option<&str> { + bump(&self.counters.resolutions); + self.keys.resolve(k.0) + } + /// Number of distinct comparison keys (the universe of a key `GROUP BY`). + pub fn key_count(&self) -> usize { + self.keys.len() } - /// `(value id, key id)` without interning; `None` if not yet interned. - pub fn lookup_attr(&self, s: Option<&str>) -> Option<(u32, u32)> { - match s { + /// `(value, key)` lane entries for an optional value; `None` if the id + /// was never issued by this store. + pub(crate) fn lane_ids(&self, v: Option) -> Option<(u32, u32)> { + match v { None => Some((NONE, NONE)), - Some(s) => Some((self.values.get(s)?, self.keys.get(&normalize(s))?)), + Some(v) => Some((v.0, *self.key_of.get(v.0 as usize)?)), } } } -/// One node as read from a source (ingestion staging). +/// One node as read from a source (ingestion staging; owns its strings). #[derive(Clone, Debug, PartialEq, Eq)] pub struct ObservedNode { /// Kind. @@ -107,8 +210,8 @@ pub struct ObservedNode { pub upn: Option, /// Raw primary SMTP. pub primary_smtp: Option, - /// OU location, if known. - pub ou: Option, + /// Hierarchy location in the observation's scope, if known. + pub dn: Option, } impl ObservedNode { @@ -119,7 +222,7 @@ impl ObservedNode { active: true, upn: Some(upn.into()), primary_smtp: Some(smtp.into()), - ou: None, + dn: None, } } /// Group. @@ -129,19 +232,7 @@ impl ObservedNode { active: true, upn: None, primary_smtp: None, - ou: None, - } - } -} - -impl From for ogar_dir_sim::NodeState { - fn from(n: ObservedNode) -> Self { - Self { - kind: n.kind, - active: n.active, - upn: n.upn, - primary_smtp: n.primary_smtp, - ou: n.ou, + dn: None, } } } @@ -149,6 +240,8 @@ impl From for ogar_dir_sim::NodeState { /// What a source reported. Order of either list is irrelevant. #[derive(Clone, Debug, Default)] pub struct Observation { + /// The directory (domain / tenant) every [`Dn128`] here belongs to. + pub scope: DirectoryScope, /// Nodes. pub nodes: Vec<(Guid128, ObservedNode)>, /// `(user, group)` memberships; endpoints need not exist (dangling @@ -162,15 +255,16 @@ pub enum BuildError { /// The same Guid128 was observed twice (which one wins would depend on /// ingestion order, so it is refused rather than guessed). DuplicateNode(Guid128), - /// More than `u32::MAX - 1` nodes or memberships. - TooLarge, + /// More users than [`MAX_USERS`]. + TooManyUsers(usize), + /// More groups than [`MAX_GROUPS`]. + TooManyGroups(usize), + /// More membership rows than a `u32` row id can address. + TooManyMemberships, } -pub(crate) fn bit(words: &[u64], i: u32) -> bool { - i != NONE - && words - .get(i as usize / 64) - .is_some_and(|w| w >> (i % 64) & 1 == 1) +pub(crate) fn bit(words: &[u64], i: usize) -> bool { + words.get(i / 64).is_some_and(|w| w >> (i % 64) & 1 == 1) } pub(crate) fn set_bit(words: &mut [u64], i: usize) { words[i / 64] |= 1 << (i % 64); @@ -188,166 +282,210 @@ pub(crate) fn ones(n: usize) -> Vec { w } -/// OU-HHTL levels 0..4 packed into one `u64` (level 0 in the top 16 bits), -/// so "subtree of a prefix of depth ≤ 4" is one ternary match. -pub fn pack_ou(h: &OuHhtl) -> u64 { - (u64::from(h.0[0]) << 48) - | (u64::from(h.0[1]) << 32) - | (u64::from(h.0[2]) << 16) - | u64::from(h.0[3]) -} - -/// One observed state. Immutable once built; versions share it by `Arc`. -#[derive(Debug, PartialEq, Eq)] -pub struct Snapshot { +/// One node population (all users, or all groups) of a snapshot. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Population { pub(crate) ids: Vec, - pub(crate) user: Vec, - pub(crate) group: Vec, - pub(crate) active_user: Vec, + pub(crate) all: Vec, + pub(crate) active: Vec, pub(crate) upn_val: Vec, pub(crate) upn_key: Vec, pub(crate) smtp_val: Vec, pub(crate) smtp_key: Vec, - pub(crate) ou: Vec, - pub(crate) ou_hi: Vec, - pub(crate) ou_present: Vec, - pub(crate) m_user: Vec, - pub(crate) m_group: Vec, - pub(crate) m_all: Vec, - /// Membership rows with an endpoint that resolved to no node: row → - /// the observed identities (rare; kept so evidence never loses identity). - pub(crate) m_unresolved: BTreeMap, + pub(crate) dn: Vec<[u8; 16]>, + pub(crate) dn_depth: Vec, + pub(crate) dn_present: Vec, } -impl Snapshot { - /// Build from an observation, interning strings into `d`. - pub fn build(mut obs: Observation, d: &mut Dicts) -> Result { - obs.nodes.sort_by_key(|n| n.0); - if let Some(w) = obs.nodes.windows(2).find(|w| w[0].0 == w[1].0) { - return Err(BuildError::DuplicateNode(w[0].0)); - } - let n = obs.nodes.len(); - if n >= NONE as usize || obs.members.len() >= NONE as usize { - return Err(BuildError::TooLarge); - } - let mut s = Self { +impl Population { + fn build(nodes: &[&(Guid128, ObservedNode)], d: &mut Dicts) -> Self { + let n = nodes.len(); + let mut p = Self { ids: Vec::with_capacity(n), - user: vec![0; words_for(n)], - group: vec![0; words_for(n)], - active_user: vec![0; words_for(n)], + all: ones(n), + active: vec![0; words_for(n)], upn_val: Vec::with_capacity(n), upn_key: Vec::with_capacity(n), smtp_val: Vec::with_capacity(n), smtp_key: Vec::with_capacity(n), - ou: Vec::with_capacity(n), - ou_hi: Vec::with_capacity(n), - ou_present: vec![0; words_for(n)], - m_user: Vec::new(), - m_group: Vec::new(), - m_all: Vec::new(), - m_unresolved: BTreeMap::new(), + dn: Vec::with_capacity(n), + dn_depth: Vec::with_capacity(n), + dn_present: vec![0; words_for(n)], }; - for (i, (id, node)) in obs.nodes.iter().enumerate() { - s.ids.push(*id); - match node.kind { - NodeKind::User => { - set_bit(&mut s.user, i); - if node.active { - set_bit(&mut s.active_user, i); - } - } - NodeKind::Group => set_bit(&mut s.group, i), + for (i, (id, node)) in nodes.iter().enumerate() { + p.ids.push(*id); + if node.active || node.kind == NodeKind::Group { + set_bit(&mut p.active, i); } - let (uv, uk) = d.intern_attr(node.upn.as_deref()); - let (sv, sk) = d.intern_attr(node.primary_smtp.as_deref()); - s.upn_val.push(uv); - s.upn_key.push(uk); - s.smtp_val.push(sv); - s.smtp_key.push(sk); - s.ou.push(node.ou.unwrap_or(OuHhtl::ROOT)); - s.ou_hi.push(node.ou.as_ref().map_or(0, pack_ou)); - if node.ou.is_some() { - set_bit(&mut s.ou_present, i); + let ids = |s: &Option, d: &mut Dicts| { + let v = s.as_deref().map(|s| d.intern(s)); + d.lane_ids(v).expect("just interned") + }; + let (uv, uk) = ids(&node.upn, d); + let (sv, sk) = ids(&node.primary_smtp, d); + p.upn_val.push(uv); + p.upn_key.push(uk); + p.smtp_val.push(sv); + p.smtp_key.push(sk); + let dn = node.dn.unwrap_or(Dn128::ROOT); + p.dn.push(dn.bytes()); + p.dn_depth.push(dn.depth() as i32); + if node.dn.is_some() { + set_bit(&mut p.dn_present, i); } } - let mut rows: Vec<(u32, u32, Guid128, Guid128)> = obs - .members - .iter() - .map(|(u, g)| { - ( - s.ordinal(u).unwrap_or(NONE), - s.ordinal(g).unwrap_or(NONE), - *u, - *g, - ) - }) - .collect(); - rows.sort(); - rows.dedup(); - for (r, (uo, go, ug, gg)) in rows.into_iter().enumerate() { - if uo == NONE || go == NONE { - s.m_unresolved.insert(r as u32, (ug, gg)); - } - s.m_user.push(uo); - s.m_group.push(go); - } - s.m_all = ones(s.m_user.len()); - Ok(s) + p } /// Node count. pub fn len(&self) -> usize { self.ids.len() } - /// True if no nodes. + /// True if empty. pub fn is_empty(&self) -> bool { self.ids.is_empty() } - /// Membership row count. + /// Ordinal of an identity (binary search over the sorted id lane). + pub(crate) fn ordinal(&self, g: &Guid128) -> Option { + self.ids.binary_search(g).ok().map(|i| i as u16) + } + pub(crate) fn dn_of(&self, o: usize) -> Option { + if !bit(&self.dn_present, o) { + return None; + } + let depth = self.dn_depth[o] as usize; + Dn128::new(&self.dn[o][..depth]).ok() + } +} + +/// One observed state. Immutable once built; versions share it by `Arc`. +#[derive(Debug, PartialEq, Eq)] +pub struct Snapshot { + pub(crate) scope: DirectoryScope, + pub(crate) users: Population, + pub(crate) groups: Population, + pub(crate) m_user: Vec, + pub(crate) m_group: Vec, + pub(crate) m_all: Vec, + /// Observed memberships with an endpoint that is not a node of the + /// expected kind, by identity, sorted. Never in the lanes. + pub(crate) m_unresolved: Vec<(Guid128, Guid128)>, +} + +impl Snapshot { + /// Build from an observation, interning strings into `d`. + pub fn build(mut obs: Observation, d: &mut Dicts) -> Result { + obs.nodes.sort_by_key(|n| n.0); + if let Some(w) = obs.nodes.windows(2).find(|w| w[0].0 == w[1].0) { + return Err(BuildError::DuplicateNode(w[0].0)); + } + let (users, groups): (Vec<_>, Vec<_>) = obs + .nodes + .iter() + .partition(|(_, n)| n.kind == NodeKind::User); + if users.len() > MAX_USERS { + return Err(BuildError::TooManyUsers(users.len())); + } + if groups.len() > MAX_GROUPS { + return Err(BuildError::TooManyGroups(groups.len())); + } + if obs.members.len() >= u32::MAX as usize { + return Err(BuildError::TooManyMemberships); + } + let users = Population::build(&users, d); + let groups = Population::build(&groups, d); + let mut rows: Vec<(u16, u16)> = Vec::with_capacity(obs.members.len()); + let mut unresolved = Vec::new(); + for (u, g) in &obs.members { + match (users.ordinal(u), groups.ordinal(g)) { + (Some(uo), Some(go)) => rows.push((uo, go)), + _ => unresolved.push((*u, *g)), + } + } + rows.sort_unstable(); + rows.dedup(); + unresolved.sort_unstable(); + unresolved.dedup(); + let (m_user, m_group): (Vec, Vec) = rows + .into_iter() + .map(|(u, g)| (u32::from(u), u32::from(g))) + .unzip(); + let m_all = ones(m_user.len()); + Ok(Self { + scope: obs.scope, + users, + groups, + m_user, + m_group, + m_all, + m_unresolved: unresolved, + }) + } + + /// The directory scope every location in this snapshot belongs to. + pub fn scope(&self) -> DirectoryScope { + self.scope + } + /// The user population. + pub fn users(&self) -> &Population { + &self.users + } + /// The group population. + pub fn groups(&self) -> &Population { + &self.groups + } + /// Resolved membership row count (unresolved rows excluded). pub fn membership_rows(&self) -> usize { self.m_user.len() } - /// Dense execution ordinal of an identity (binary search over the sorted id lane). - pub fn ordinal(&self, g: &Guid128) -> Option { - self.ids.binary_search(g).ok().map(|i| i as u32) + /// Ordinal of a user. + pub fn user_ordinal(&self, g: &Guid128) -> Option { + self.users.ordinal(g).map(UserOrdinal) } - /// Identity of an ordinal. - pub fn guid(&self, o: u32) -> Option { - self.ids.get(o as usize).copied() + /// Ordinal of a group. + pub fn group_ordinal(&self, g: &Guid128) -> Option { + self.groups.ordinal(g).map(GroupOrdinal) } - /// Identities of membership row `r`. - pub(crate) fn member_guids(&self, r: u32) -> (Guid128, Guid128) { - if let Some(p) = self.m_unresolved.get(&r) { - return *p; + /// Identity of a user ordinal. + pub fn user_guid(&self, o: UserOrdinal) -> Option { + self.users.ids.get(usize::from(o.0)).copied() + } + /// Identity of a group ordinal. + pub fn group_guid(&self, o: GroupOrdinal) -> Option { + self.groups.ids.get(usize::from(o.0)).copied() + } + /// The population of a kind. + pub(crate) fn population(&self, kind: NodeKind) -> &Population { + match kind { + NodeKind::User => &self.users, + NodeKind::Group => &self.groups, } + } + /// Identities of resolved membership row `r`. + pub(crate) fn member_guids(&self, r: u32) -> (Guid128, Guid128) { ( - self.ids[self.m_user[r as usize] as usize], - self.ids[self.m_group[r as usize] as usize], + self.users.ids[self.m_user[r as usize] as usize], + self.groups.ids[self.m_group[r as usize] as usize], ) } - /// Row of membership `(user, group)` in the base relation, if observed. + /// Row of resolved membership `(user, group)`, if observed. `O(log m)`: + /// rows are sorted by `(user, group)`. pub(crate) fn member_row(&self, user: &Guid128, group: &Guid128) -> Option { - match (self.ordinal(user), self.ordinal(group)) { - (Some(uo), Some(go)) => { - // Rows are sorted by the full (user, group, …) tuple, so the - // (user, group) prefix is monotonic over EVERY row — including - // half-resolved ones (`NONE` sorts after every ordinal). - let (mut lo, mut hi) = (0usize, self.m_user.len()); - while lo < hi { - let mid = (lo + hi) / 2; - match (self.m_user[mid], self.m_group[mid]).cmp(&(uo, go)) { - std::cmp::Ordering::Less => lo = mid + 1, - std::cmp::Ordering::Greater => hi = mid, - std::cmp::Ordering::Equal => return Some(mid as u32), - } - } - None + let (uo, go) = (self.users.ordinal(user)?, self.groups.ordinal(group)?); + let key = (u32::from(uo), u32::from(go)); + let (mut lo, mut hi) = (0usize, self.m_user.len()); + while lo < hi { + let mid = (lo + hi) / 2; + match (self.m_user[mid], self.m_group[mid]).cmp(&key) { + std::cmp::Ordering::Less => lo = mid + 1, + std::cmp::Ordering::Greater => hi = mid, + std::cmp::Ordering::Equal => return Some(mid as u32), } - _ => self - .m_unresolved - .iter() - .find(|(_, p)| **p == (*user, *group)) - .map(|(r, _)| *r), } + None + } + /// Whether `(user, group)` is an observed but unresolved membership. + pub(crate) fn is_unresolved_member(&self, user: &Guid128, group: &Guid128) -> bool { + self.m_unresolved.binary_search(&(*user, *group)).is_ok() } } diff --git a/crates/lance-graph-dir-sim/src/store.rs b/crates/lance-graph-dir-sim/src/store.rs index 9e87c2339..cc2749d27 100644 --- a/crates/lance-graph-dir-sim/src/store.rs +++ b/crates/lance-graph-dir-sim/src/store.rs @@ -14,8 +14,8 @@ use crate::validate::validate; use crate::view::{ApplyError, Overlay, View}; use ogar_dir_core::Guid128; use ogar_dir_sim::{ - Attribute, Change, EvidenceRef, ExecutionPlan, NodeState, Origin, PlanError, Version, - VersionId, Violation, TAG_DESIRED, TAG_OBSERVED, + Attribute, Change, EvidenceRef, ExecutionPlan, KeyId, NodeKind, NodeState, Origin, PlanError, + ValueId, Version, VersionId, Violation, TAG_DESIRED, TAG_OBSERVED, }; use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; @@ -29,6 +29,9 @@ pub enum SimError { EmptyProposal(ogar_dir_sim::RuleId), /// The proposal does not apply to the parent. Apply(ApplyError), + /// The two versions describe different directories; their hierarchy + /// codes are not comparable. + ScopeMismatch(VersionId, VersionId), } /// Refusal to make a version desired. The version stays as evidence. @@ -56,6 +59,33 @@ impl VersionStore { Self::default() } + /// Ingress: the stable id of a raw value (interned if new). Every + /// snapshot and version of this store resolves the same id to the same + /// value, from observation through plan. + pub fn intern(&mut self, s: &str) -> ValueId { + self.dicts.intern(s) + } + /// The id of a raw value this store has seen. Never mints. + pub fn lookup(&self, s: &str) -> Option { + self.dicts.lookup(s) + } + /// Egress: the raw value behind an id. + pub fn value(&self, v: ValueId) -> Option<&str> { + self.dicts.value(v) + } + /// The comparison key of a value. + pub fn key_of(&self, v: ValueId) -> Option { + self.dicts.key_of(v) + } + /// Egress: the comparison form behind a key. + pub fn key_label(&self, k: KeyId) -> Option<&str> { + self.dicts.key_label(k) + } + /// The cold label/value store. + pub fn dicts(&self) -> &Dicts { + &self.dicts + } + fn next_id(&self) -> VersionId { VersionId(self.versions.len() as u64) } @@ -138,18 +168,6 @@ impl VersionStore { if delta.is_empty() { return Err(SimError::EmptyProposal(rule.id())); } - for c in &delta { - match c { - Change::SetAttribute { to, .. } => { - self.dicts.intern_attr(to.as_deref()); - } - Change::CreateNode { state, .. } => { - self.dicts.intern_attr(state.upn.as_deref()); - self.dicts.intern_attr(state.primary_smtp.as_deref()); - } - _ => {} - } - } let mut view = self.view(parent)?; for c in &delta { view.apply(c).map_err(SimError::Apply)?; @@ -208,6 +226,9 @@ impl VersionStore { /// [`Change::DeleteNode`]. pub fn diff(&self, a: VersionId, b: VersionId) -> Result, SimError> { let (va, vb) = (self.view(a)?, self.view(b)?); + if va.snap.scope != vb.snap.scope { + return Err(SimError::ScopeMismatch(a, b)); + } let mut out = if std::ptr::eq(va.snap, vb.snap) { diff_shared(&va, &vb) } else { @@ -241,13 +262,20 @@ impl VersionStore { // it was recorded), so the only failure here is an unknown id. let view = |v| self.view(v).map_err(|_| PlanError::UnknownVersion(v)); let (vr, vt, vb) = (view(root)?, view(target)?, view(basis)?); + if vb.snap.scope != vt.snap.scope { + return Err(PlanError::ScopeMismatch { basis, target }); + } let intent = diff_shared(&vr, &vt); let ops = outstanding(&vb, intent).map_err(|node| PlanError::Unconvergeable { basis, target, node, })?; - Ok(ExecutionPlan::from_diff(basis, target, ops)) + // Ordering of value transfers compares comparison keys; every value + // in a stored version was issued by this store, so the fallback is + // never taken. + let key = |v: ValueId| self.dicts.key_of(v).unwrap_or(KeyId(v.0)); + Ok(ExecutionPlan::from_diff(basis, target, ops, key)) } /// Why does `v` contain membership `(user, group)`? The lineage from @@ -283,30 +311,25 @@ impl VersionStore { fn set_change( node: Guid128, attribute: Attribute, - from: Option<&str>, - to: Option<&str>, + from: Option, + to: Option, ) -> Option { - (from != to).then(|| Change::SetAttribute { + (from != to).then_some(Change::SetAttribute { node, attribute, - from: from.map(str::to_string), - to: to.map(str::to_string), + from, + to, }) } /// Upn and primary-SMTP changes between two present nodes. fn attr_changes(node: Guid128, a: &NodeState, b: &NodeState, out: &mut Vec) { - out.extend(set_change( - node, - Attribute::Upn, - a.upn.as_deref(), - b.upn.as_deref(), - )); + out.extend(set_change(node, Attribute::Upn, a.upn, b.upn)); out.extend(set_change( node, Attribute::PrimarySmtp, - a.primary_smtp.as_deref(), - b.primary_smtp.as_deref(), + a.primary_smtp, + b.primary_smtp, )); } @@ -328,8 +351,11 @@ fn diff_shared(a: &View<'_>, b: &View<'_>) -> Vec { // Node presence: created or deleted in either overlay. let mut nodes: BTreeSet = BTreeSet::new(); for v in [a, b] { - nodes.extend(v.ov.created.ids.iter().copied()); - nodes.extend(v.ov.deleted.iter().map(|&o| s.ids[o as usize])); + for kind in [NodeKind::User, NodeKind::Group] { + let (p, o) = v.pop(kind); + nodes.extend(o.created.ids.iter().copied()); + nodes.extend(o.deleted.iter().map(|&d| p.ids[usize::from(d)])); + } } for g in &nodes { node_changes(*g, a.node_state(g), b.node_state(g), &mut out); @@ -340,6 +366,7 @@ fn diff_shared(a: &View<'_>, b: &View<'_>) -> Vec { a.ov.added.iter().chain(&b.ov.added).copied().collect(); for v in [a, b] { pairs.extend(v.ov.removed.iter().map(|&r| s.member_guids(r))); + pairs.extend(v.ov.removed_unresolved.iter().copied()); } for (u, g) in pairs { match (a.is_member(&u, &g), b.is_member(&u, &g)) { @@ -351,19 +378,24 @@ fn diff_shared(a: &View<'_>, b: &View<'_>) -> Vec { // Attribute overrides of base nodes present in both versions (a node // created or deleted between them is covered above). - for attr in [Attribute::Upn, Attribute::PrimarySmtp] { - let touched: BTreeSet = - a.ov.overrides(attr) + for kind in [NodeKind::User, NodeKind::Group] { + let p = s.population(kind); + for attr in [Attribute::Upn, Attribute::PrimarySmtp] { + let touched: BTreeSet = a + .pop(kind) + .1 + .overrides(attr) .keys() - .chain(b.ov.overrides(attr).keys()) + .chain(b.pop(kind).1.overrides(attr).keys()) .copied() .collect(); - for o in touched { - let g = s.ids[o as usize]; - if nodes.contains(&g) { - continue; + for o in touched { + let g = p.ids[usize::from(o)]; + if nodes.contains(&g) { + continue; + } + out.extend(set_change(g, attr, a.attr(&g, attr), b.attr(&g, attr))); } - out.extend(set_change(g, attr, a.attr(o, attr), b.attr(o, attr))); } } out @@ -371,9 +403,9 @@ fn diff_shared(a: &View<'_>, b: &View<'_>) -> Vec { /// Existing nodes of a view, ascending by identity. fn live_ids(v: &View<'_>) -> Vec { - let mut ids: Vec = (0..v.snap.len() as u32) - .filter_map(|o| v.guid(o)) - .chain(v.ov.created.ids.iter().copied()) + let mut ids: Vec = [NodeKind::User, NodeKind::Group] + .into_iter() + .flat_map(|k| (0..v.len_in(k)).filter_map(move |i| v.guid_in(k, i))) .collect(); ids.sort(); ids @@ -402,6 +434,13 @@ fn diff_full(a: &View<'_>, b: &View<'_>) -> Vec { lance_graph_mask_risc::materialize_rows(&live, v.snap.membership_rows()) .into_iter() .map(|r| v.snap.member_guids(r as u32)) + .chain( + v.snap + .m_unresolved + .iter() + .filter(|p| !v.ov.removed_unresolved.contains(p)) + .copied(), + ) .chain(v.ov.added.iter().copied()) .collect() }; @@ -422,7 +461,7 @@ fn diff_full(a: &View<'_>, b: &View<'_>) -> Vec { /// identity lookup, so the work is proportional to the intent. /// /// `Err(node)`: a node the intent creates already exists in `basis` with a -/// kind, enabled flag or OU that no change can converge (the algebra sets +/// kind, enabled flag or location that no change can converge (the algebra sets /// only UPN and primary SMTP), so the create is neither done nor doable. fn outstanding(basis: &View<'_>, intent: Vec) -> Result, Guid128> { let mut out = Vec::new(); @@ -445,12 +484,12 @@ fn outstanding(basis: &View<'_>, intent: Vec) -> Result, Gui .. } => { // A node gone from reality has nothing left to set. - if let Some(o) = basis.ordinal(&node) { + if basis.exists(&node) { out.extend(set_change( node, attribute, - basis.attr(o, attribute), - to.as_deref(), + basis.attr(&node, attribute), + to, )); } } @@ -458,8 +497,8 @@ fn outstanding(basis: &View<'_>, intent: Vec) -> Result, Gui None => out.push(Change::CreateNode { node, state }), // Already exists: only its settable attributes may differ. Some(actual) => { - if (actual.kind, actual.active, actual.ou) - != (state.kind, state.active, state.ou) + if (actual.kind, actual.active, actual.dn) + != (state.kind, state.active, state.dn) { return Err(node); } diff --git a/crates/lance-graph-dir-sim/src/validate.rs b/crates/lance-graph-dir-sim/src/validate.rs index eace10a5f..3d77d0bda 100644 --- a/crates/lance-graph-dir-sim/src/validate.rs +++ b/crates/lance-graph-dir-sim/src/validate.rs @@ -2,30 +2,34 @@ //! //! | invariant | relational form | lowering | //! |----------------------|---------------------------------------------------------|----------------------------------------| -//! | edge integrity | `members ANTI JOIN users ∪ members ANTI JOIN groups` | `Rows(¬Semijoin(user,·) ∨ ¬Semijoin(group,·))` — `MaskOp::Gather` over the node kind planes | -//! | unique SMTP / UPN | `GROUP BY key HAVING count > 1` over active users | `GroupReduce Count` keyed on the normalized-key dictionary id | +//! | edge integrity | `members ANTI JOIN users ∪ members ANTI JOIN groups` | `Rows(¬Semijoin(user,·) ∨ ¬Semijoin(group,·))` — `MaskOp::Gather` over the live user and group planes | +//! | unique SMTP / UPN | `GROUP BY key HAVING count > 1` over active users | `GroupReduce Count` keyed on [`KeyId`] | //! -//! No user objects are built. The integrity check reads the two membership -//! lanes and two kind planes — it cannot see a string. Uniqueness reads one -//! key lane and one plane; strings are resolved only for the (few) keys that -//! actually collide, to report them. +//! No user objects are built and no string is read: the integrity check +//! reads the two membership lanes and the two live-node planes, uniqueness +//! one key lane and one plane. A violation carries identities and the +//! [`KeyId`]; resolving a key to text is the reporter's job. +//! +//! Memberships whose endpoint never resolved to an ordinal are not in the +//! lanes (no sentinel ordinal exists); they are dangling by construction and +//! reported from the snapshot's unresolved table and the overlay, both +//! evidence-sized. //! //! Materialisations, all at the evidence boundary and bounded by the number //! of violations: the offending membership rows (`materialize_rows` of the //! kept mask) and each duplicate key's owner rows. use crate::exec::{group_count_into, keep, program}; -use crate::snapshot::bit; +use crate::snapshot::{bit, ones}; use crate::view::View; use lance_graph_mask_risc::{Foreign, ForeignPlane as FPlane, LaneRef, Planes, Program}; use lance_graph_quack::{Agg, Cmp, Col, Filter, ForeignPlane, Mask}; use ogar_dir_core::Guid128; -use ogar_dir_sim::{normalize, Attribute, Endpoint, NodeKind, Violation}; +use ogar_dir_sim::{Attribute, Endpoint, KeyId, NodeKind, Violation}; /// The edge-integrity program over a membership relation: lane 0 = user /// ordinal, lane 1 = group ordinal, plane 0 = live rows; foreign plane 0 = -/// the node table's user plane, 1 = its group plane. An anti-join on each -/// side; an unresolved endpoint (`NONE`) is out of range and gathers 0. +/// the live user plane, 1 = the live group plane. An anti-join on each side. pub fn dangling_program() -> Program { program( Filter::and([ @@ -39,23 +43,24 @@ pub fn dangling_program() -> Program { ) } -/// Dangling memberships of a version (base rows still live + added rows). +/// Dangling memberships of a version. /// -/// The foreign planes are the version's node-kind planes: the snapshot's, -/// minus deleted nodes, plus created ones. They are composed from resident -/// planes and the overlay, never taken from a program's output. +/// Resolved rows (base still live, overlay added): one anti-join program +/// against the version's live user and group planes — the snapshot's, minus +/// deleted nodes, plus created ones, composed from resident planes and the +/// overlay, never taken from a program's output. Unresolved rows: reported +/// directly. pub fn dangling(v: &View<'_>) -> Vec { let s = v.snap; - let (users, groups) = (v.kind_plane(NodeKind::User), v.kind_plane(NodeKind::Group)); - let width = v.len(); + let (users, groups) = (v.existing(NodeKind::User), v.existing(NodeKind::Group)); let fps = [ FPlane { words: &users, - rows: width, + rows: v.users_len(), }, FPlane { words: &groups, - rows: width, + rows: v.groups_len(), }, ]; let foreign = Foreign { @@ -64,7 +69,7 @@ pub fn dangling(v: &View<'_>) -> Vec { }; let p = dangling_program(); let side = |uo: u32| { - if bit(&users, uo) { + if bit(&users, uo as usize) { Endpoint::Group } else { Endpoint::User @@ -89,39 +94,57 @@ pub fn dangling(v: &View<'_>) -> Vec { }); } - // The overlay relation: delta-sized lanes, same program. - let (au, ag, ids) = v.added_rows(); - let all = crate::snapshot::ones(au.len()); - let lanes = [LaneRef::U32(&au), LaneRef::U32(&ag)]; + // Identity-held rows (overlay adds, observed unresolved pairs) that + // resolve in this version: delta-sized lanes, same program. + let added = v.added_rows(); + let all = ones(added.users.len()); + let lanes = [LaneRef::U32(&added.users), LaneRef::U32(&added.groups)]; let masks: [&[u64]; 1] = [&all]; let planes = Planes { - n_rows: au.len(), + n_rows: added.users.len(), masks: &masks, lanes: &lanes, }; for r in keep(&p, &planes, &foreign).rows() { - let (user, group) = ids[r]; + let user = v.guid_in(NodeKind::User, added.users[r] as usize); + let group = v.guid_in(NodeKind::Group, added.groups[r] as usize); + if let (Some(user), Some(group)) = (user, group) { + out.push(Violation::DanglingMembership { + user, + group, + missing: side(added.users[r]), + }); + } + } + + // Identity-held rows that still do not resolve in this version. + for &(user, group) in &added.unresolved { + let missing = if v.user_ordinal(&user).is_some() { + Endpoint::Group + } else { + Endpoint::User + }; out.push(Violation::DanglingMembership { user, group, - missing: side(au[r]), + missing, }); } out.sort(); out } -/// Active users sharing a normalized value of `a`. +/// Active users sharing a comparison key of `a`. pub fn duplicates(v: &View<'_>, a: Attribute) -> Vec { let s = v.snap; - let keys = &v.dicts.keys; - let k = keys.len(); + let u = &s.users; + let k = v.dicts.key_count(); if k == 0 { return Vec::new(); } let base_key = match a { - Attribute::Upn => &s.upn_key, - Attribute::PrimarySmtp => &s.smtp_key, + Attribute::Upn => &u.upn_key, + Attribute::PrimarySmtp => &u.smtp_key, }; let owners_plane = v.live_owners(a); let mut counts = vec![0i64; k]; @@ -130,7 +153,7 @@ pub fn duplicates(v: &View<'_>, a: Attribute) -> Vec { let lanes = [LaneRef::U32(base_key)]; let masks: [&[u64]; 1] = [&owners_plane]; let base = Planes { - n_rows: s.len(), + n_rows: u.len(), masks: &masks, lanes: &lanes, }; @@ -142,27 +165,28 @@ pub fn duplicates(v: &View<'_>, a: Attribute) -> Vec { &mut counts, ); - // Delta rows — overrides of base nodes and created nodes, as one small + // Delta rows — overrides of base users and created users, as one small // relation (ordinal, key) — through the same GROUP BY, admitted by a // semijoin of the ordinal against the version's active-user plane. - let n = s.len() as u32; - let created = v.ov.created.key(a); - let (oo, ok): (Vec, Vec) = - v.ov.overrides(a) - .iter() - .map(|(o, (_, key))| (*o, *key)) - .chain( - created - .iter() - .enumerate() - .map(|(i, key)| (n + i as u32, *key)), - ) - .unzip(); + let n = u.len() as u32; + let ou = &v.ov.users; + let (oo, ok): (Vec, Vec) = ou + .overrides(a) + .iter() + .map(|(o, (_, key))| (u32::from(*o), *key)) + .chain( + ou.created + .key(a) + .iter() + .enumerate() + .map(|(i, key)| (n + i as u32, *key)), + ) + .unzip(); let active = v.active_users(); - let all = crate::snapshot::ones(oo.len()); + let all = ones(oo.len()); let fps = [FPlane { words: &active, - rows: v.len(), + rows: v.users_len(), }]; let foreign = Foreign { planes: &fps, @@ -197,22 +221,19 @@ pub fn duplicates(v: &View<'_>, a: Attribute) -> Vec { let mut owners: Vec = keep(&p, &base, &Foreign::NONE) .rows() .into_iter() - .map(|o| s.ids[o]) + .map(|o| u.ids[o]) .collect(); owners.extend( oo.iter() .zip(&ok) - .filter(|(o, kk)| **kk == key && bit(&active, **o)) - .filter_map(|(o, _)| v.guid(*o)), + .filter(|(o, kk)| **kk == key && bit(&active, **o as usize)) + .filter_map(|(o, _)| v.guid_in(NodeKind::User, *o as usize)), ); owners.sort(); - let value = keys.resolve(key).map(normalize).unwrap_or_default(); + let key = KeyId(key); out.push(match a { - Attribute::Upn => Violation::DuplicateUpn { upn: value, owners }, - Attribute::PrimarySmtp => Violation::DuplicateSmtp { - address: value, - owners, - }, + Attribute::Upn => Violation::DuplicateUpn { key, owners }, + Attribute::PrimarySmtp => Violation::DuplicateSmtp { key, owners }, }); } out diff --git a/crates/lance-graph-dir-sim/src/view.rs b/crates/lance-graph-dir-sim/src/view.rs index 6d5490ecf..302571876 100644 --- a/crates/lance-graph-dir-sim/src/view.rs +++ b/crates/lance-graph-dir-sim/src/view.rs @@ -5,10 +5,10 @@ //! an [`Overlay`] whose size is proportional to its accumulated delta: //! //! * added membership identity pairs, -//! * removed base membership rows (sorted row ids), -//! * per-attribute override maps `ordinal → (value id, key id)`, -//! * created nodes as their own small SoA lanes ([`Created`]), -//! * deleted base nodes (sorted ordinals). +//! * removed membership rows (sorted row ids; unresolved rows by identity), +//! * per population: attribute overrides `ordinal → (value, key)`, created +//! nodes as their own small SoA lanes ([`Created`]), and deleted base +//! ordinals. //! //! Nothing in the overlay is allocated in proportion to the directory: one //! mutation adds one entry. Queries build their "still live" planes when @@ -16,18 +16,21 @@ //! over the overlay's delta rows, and fold the two results. The base is //! never copied. //! -//! **Ordinal space of a view.** Base nodes keep their snapshot ordinals -//! `0..n`; created nodes take `n..n + c` in `Guid128` order. A deleted base -//! node keeps its slot but has no ordinal ([`View::ordinal`] returns `None`) -//! and is cleared from every live plane. Ordinals are valid only inside one -//! view and are never stored: the overlay keeps identities wherever an -//! ordinal could shift (added memberships, created nodes). +//! **Ordinal spaces of a view.** Users and groups are numbered separately. +//! Base nodes keep their snapshot ordinals `0..n`; created nodes take +//! `n..n + c` in `Guid128` order; a population never exceeds 65,536. A +//! deleted base node keeps its slot but has no ordinal and is cleared from +//! every live plane. Ordinals are valid only inside one view; the overlay +//! keeps identities wherever an ordinal could shift. -use crate::snapshot::{bit, clear_bit, set_bit, Dicts, Snapshot, NONE}; +use crate::snapshot::{ + bit, clear_bit, set_bit, Dicts, GroupOrdinal, Population, Snapshot, UserOrdinal, MAX_GROUPS, + MAX_USERS, NONE, +}; use lance_graph_mask_risc::{words_for, Foreign, LaneRef, Planes}; use lance_graph_quack::{Cmp, Col, Filter, Mask}; -use ogar_dir_core::{Guid128, OuHhtl}; -use ogar_dir_sim::{Attribute, Change, NodeKind, NodeState}; +use ogar_dir_core::{Dn128, Guid128}; +use ogar_dir_sim::{Attribute, Change, NodeKind, NodeState, ValueId}; use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; @@ -43,14 +46,14 @@ pub enum ApplyError { /// Attribute. attribute: Attribute, /// What the change expected. - expected: Option, + expected: Option, /// What the version holds. - actual: Option, + actual: Option, }, /// The membership already holds (add) / does not hold (remove). NoOp(Change), - /// A value in the change was never interned (store bug, not input). - Uninterned, + /// A value id the store never issued (caller bug, not input). + Uninterned(ValueId), /// `CreateNode` of an identity that already exists. NodeExists(Guid128), /// `CreateNode` of an observed identity this lineage deleted, with a @@ -72,19 +75,22 @@ pub enum ApplyError { /// `CreateNode` of a group with `active = false`. Groups have no /// enabled flag; their canonical state carries `active = true`. InactiveGroup(Guid128), + /// `CreateNode` into a population already at its bound + /// ([`MAX_USERS`] / [`MAX_GROUPS`]). + PopulationFull(NodeKind), } -/// Nodes a version created, as SoA lanes sorted by identity. Delta-sized. +/// Nodes a version created in one population, as SoA lanes sorted by +/// identity. Delta-sized. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct Created { pub(crate) ids: Vec, - pub(crate) kind: Vec, pub(crate) active: Vec, pub(crate) upn_val: Vec, pub(crate) upn_key: Vec, pub(crate) smtp_val: Vec, pub(crate) smtp_key: Vec, - pub(crate) ou: Vec>, + pub(crate) dn: Vec>, } impl Created { @@ -93,25 +99,23 @@ impl Created { } fn insert(&mut self, at: usize, g: Guid128, s: &NodeState, upn: (u32, u32), smtp: (u32, u32)) { self.ids.insert(at, g); - self.kind.insert(at, s.kind); self.active.insert(at, s.active); self.upn_val.insert(at, upn.0); self.upn_key.insert(at, upn.1); self.smtp_val.insert(at, smtp.0); self.smtp_key.insert(at, smtp.1); - self.ou.insert(at, s.ou); + self.dn.insert(at, s.dn); } fn remove(&mut self, at: usize) { self.ids.remove(at); - self.kind.remove(at); self.active.remove(at); self.upn_val.remove(at); self.upn_key.remove(at); self.smtp_val.remove(at); self.smtp_key.remove(at); - self.ou.remove(at); + self.dn.remove(at); } - fn len(&self) -> usize { + pub(crate) fn len(&self) -> usize { self.ids.len() } fn lanes(&mut self, a: Attribute) -> (&mut Vec, &mut Vec) { @@ -128,34 +132,24 @@ impl Created { } } -/// The delta-sized part of a version. +/// One population's share of a version's delta. #[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct Overlay { - /// Added memberships, by identity (resolved to ordinals per query). - pub(crate) added: BTreeSet<(Guid128, Guid128)>, - /// Removed base membership rows. - pub(crate) removed: BTreeSet, +pub(crate) struct PopOverlay { /// UPN overrides of base nodes: ordinal → (value id, key id), `NONE` = cleared. - pub(crate) upn: BTreeMap, + pub(crate) upn: BTreeMap, /// Primary-SMTP overrides of base nodes. - pub(crate) smtp: BTreeMap, + pub(crate) smtp: BTreeMap, /// Created nodes. pub(crate) created: Created, /// Deleted base nodes (ordinals). - pub(crate) deleted: BTreeSet, + pub(crate) deleted: BTreeSet, } -impl Overlay { - /// Number of delta entries held. - pub(crate) fn delta_len(&self) -> usize { - self.added.len() - + self.removed.len() - + self.upn.len() - + self.smtp.len() - + self.created.len() - + self.deleted.len() +impl PopOverlay { + fn len(&self) -> usize { + self.upn.len() + self.smtp.len() + self.created.len() + self.deleted.len() } - pub(crate) fn overrides(&self, a: Attribute) -> &BTreeMap { + pub(crate) fn overrides(&self, a: Attribute) -> &BTreeMap { match a { Attribute::Upn => &self.upn, Attribute::PrimarySmtp => &self.smtp, @@ -163,6 +157,35 @@ impl Overlay { } } +/// The delta-sized part of a version. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Overlay { + /// Added memberships, by identity (resolved to ordinals per query). + pub(crate) added: BTreeSet<(Guid128, Guid128)>, + /// Removed resolved membership rows. + pub(crate) removed: BTreeSet, + /// Removed unresolved (observed, dangling) memberships. + pub(crate) removed_unresolved: BTreeSet<(Guid128, Guid128)>, + /// User population delta. + pub(crate) users: PopOverlay, + /// Group population delta. + pub(crate) groups: PopOverlay, +} + +impl Overlay { + /// Number of delta entries held. + pub(crate) fn delta_len(&self) -> usize { + self.added.len() + + self.removed.len() + + self.removed_unresolved.len() + + self.users.len() + + self.groups.len() + } +} + +/// A node position inside a view: its population and its index there. +pub(crate) type Slot = (NodeKind, usize); + /// A coherent read of one version: shared snapshot + overlay. #[derive(Debug)] pub struct View<'s> { @@ -180,160 +203,197 @@ impl<'s> View<'s> { pub fn snapshot(&self) -> &'s Snapshot { self.snap } + /// The store's label/value table (egress formatting only). + pub fn dicts(&self) -> &'s Dicts { + self.dicts + } /// Delta entries this version holds over its snapshot. pub fn delta_len(&self) -> usize { self.ov.delta_len() } - /// Dense execution ordinal of an existing node. `O(log n + log c)`. - pub fn ordinal(&self, g: &Guid128) -> Option { - match self.snap.ordinal(g) { - Some(o) if !self.ov.deleted.contains(&o) => Some(o), + + pub(crate) fn pop(&self, kind: NodeKind) -> (&'s Population, &PopOverlay) { + match kind { + NodeKind::User => (&self.snap.users, &self.ov.users), + NodeKind::Group => (&self.snap.groups, &self.ov.groups), + } + } + fn pop_mut(&mut self, kind: NodeKind) -> &mut PopOverlay { + match kind { + NodeKind::User => &mut self.ov.users, + NodeKind::Group => &mut self.ov.groups, + } + } + /// Slot count of a population: base slots plus created nodes. A deleted + /// base node keeps its slot (cleared in every live plane). + pub(crate) fn len_in(&self, kind: NodeKind) -> usize { + let (p, o) = self.pop(kind); + p.len() + o.created.len() + } + /// Index of an existing node within a population. + pub(crate) fn index_in(&self, kind: NodeKind, g: &Guid128) -> Option { + let (p, o) = self.pop(kind); + match p.ordinal(g) { + Some(i) if !o.deleted.contains(&i) => Some(usize::from(i)), Some(_) => None, - None => self - .ov - .created - .find(g) - .ok() - .map(|i| (self.snap.len() + i) as u32), + None => o.created.find(g).ok().map(|i| p.len() + i), } } - /// Identity of an existing node's ordinal. - pub fn guid(&self, o: u32) -> Option { - let n = self.snap.len(); - if (o as usize) < n { - (!self.ov.deleted.contains(&o)).then(|| self.snap.ids[o as usize]) + /// Identity of an existing node's index. + pub(crate) fn guid_in(&self, kind: NodeKind, i: usize) -> Option { + let (p, o) = self.pop(kind); + if i < p.len() { + (!o.deleted.contains(&(i as u16))).then(|| p.ids[i]) } else { - self.ov.created.ids.get(o as usize - n).copied() + o.created.ids.get(i - p.len()).copied() } } - /// Population width: base slots plus created nodes. A deleted base node - /// keeps its slot (cleared in every live plane). - pub fn len(&self) -> usize { - self.snap.len() + self.ov.created.len() + /// Where an existing node lives. + pub(crate) fn locate(&self, g: &Guid128) -> Option { + [NodeKind::User, NodeKind::Group] + .into_iter() + .find_map(|k| self.index_in(k, g).map(|i| (k, i))) + } + + /// User count, including created users (the user ordinal universe). + pub fn users_len(&self) -> usize { + self.len_in(NodeKind::User) + } + /// Group count, including created groups. + pub fn groups_len(&self) -> usize { + self.len_in(NodeKind::Group) } - /// True if the version has no node slots. - pub fn is_empty(&self) -> bool { - self.len() == 0 + /// Ordinal of an existing user. `O(log n + log c)`. + pub fn user_ordinal(&self, g: &Guid128) -> Option { + self.index_in(NodeKind::User, g) + .map(|i| UserOrdinal(i as u16)) + } + /// Ordinal of an existing group. + pub fn group_ordinal(&self, g: &Guid128) -> Option { + self.index_in(NodeKind::Group, g) + .map(|i| GroupOrdinal(i as u16)) + } + /// Identity of a user ordinal. + pub fn user_guid(&self, o: UserOrdinal) -> Option { + self.guid_in(NodeKind::User, usize::from(o.0)) + } + /// Identity of a group ordinal. + pub fn group_guid(&self, o: GroupOrdinal) -> Option { + self.guid_in(NodeKind::Group, usize::from(o.0)) } /// True if the node exists in this version. pub fn exists(&self, g: &Guid128) -> bool { - self.ordinal(g).is_some() + self.locate(g).is_some() } - /// A base plane widened to [`Self::len`]: deleted base nodes cleared, - /// created rows set where `created` says so. Borrowed when the overlay - /// has neither (the common simulation case). - fn live_plane(&self, base: &'s [u64], created: impl Fn(usize) -> bool) -> Cow<'s, [u64]> { - let (n, c) = (self.snap.len(), self.ov.created.len()); - if c == 0 && self.ov.deleted.is_empty() { + /// A base plane widened to the population's slot count: deleted base + /// nodes cleared, created rows set where `created` says so. Borrowed + /// when the overlay touches neither (the common simulation case). + fn live_plane( + &self, + kind: NodeKind, + base: &'s [u64], + created: impl Fn(usize) -> bool, + ) -> Cow<'s, [u64]> { + let (p, o) = self.pop(kind); + let (n, c) = (p.len(), o.created.len()); + if c == 0 && o.deleted.is_empty() { return Cow::Borrowed(base); } - let mut p = base.to_vec(); - p.resize(words_for(n + c), 0); - for &o in &self.ov.deleted { - clear_bit(&mut p, o as usize); + let mut plane = base.to_vec(); + plane.resize(words_for(n + c), 0); + for &d in &o.deleted { + clear_bit(&mut plane, usize::from(d)); } for i in (0..c).filter(|&i| created(i)) { - set_bit(&mut p, n + i); + set_bit(&mut plane, n + i); } - Cow::Owned(p) + Cow::Owned(plane) + } + /// Active users as a bit plane over the user ordinals. + pub fn active_users(&self) -> Cow<'s, [u64]> { + let cr = &self.ov.users.created; + self.live_plane(NodeKind::User, &self.snap.users.active, |i| cr.active[i]) + } + /// Existing nodes of a population as a bit plane over its ordinals. + pub(crate) fn existing(&self, kind: NodeKind) -> Cow<'s, [u64]> { + let (p, _) = self.pop(kind); + self.live_plane(kind, &p.all, |_| true) } /// A base-width plane with deleted base nodes cleared. - pub(crate) fn base_live(&self, base: &'s [u64]) -> Cow<'s, [u64]> { - if self.ov.deleted.is_empty() { + pub(crate) fn base_live(&self, kind: NodeKind, base: &'s [u64]) -> Cow<'s, [u64]> { + let (_, o) = self.pop(kind); + if o.deleted.is_empty() { return Cow::Borrowed(base); } - let mut p = base.to_vec(); - for &o in &self.ov.deleted { - clear_bit(&mut p, o as usize); + let mut plane = base.to_vec(); + for &d in &o.deleted { + clear_bit(&mut plane, usize::from(d)); } - Cow::Owned(p) - } - /// Active users as a bit plane over [`Self::len`]. - pub fn active_users(&self) -> Cow<'s, [u64]> { - let cr = &self.ov.created; - self.live_plane(&self.snap.active_user, |i| { - cr.kind[i] == NodeKind::User && cr.active[i] - }) - } - /// Existing nodes of `kind` as a bit plane over [`Self::len`]. - pub(crate) fn kind_plane(&self, kind: NodeKind) -> Cow<'s, [u64]> { - let base = match kind { - NodeKind::User => &self.snap.user, - NodeKind::Group => &self.snap.group, - }; - let cr = &self.ov.created; - self.live_plane(base, |i| cr.kind[i] == kind) + Cow::Owned(plane) } + /// Effective membership, by identity. `O(log m)` + overlay lookup. pub fn is_member(&self, user: &Guid128, group: &Guid128) -> bool { if self.ov.added.contains(&(*user, *group)) { return true; } - self.snap - .member_row(user, group) - .is_some_and(|r| !self.ov.removed.contains(&r)) + if let Some(r) = self.snap.member_row(user, group) { + return !self.ov.removed.contains(&r); + } + self.snap.is_unresolved_member(user, group) + && !self.ov.removed_unresolved.contains(&(*user, *group)) } - fn attr_ids(&self, o: u32, a: Attribute) -> Option<(u32, u32)> { - let n = self.snap.len(); - if (o as usize) >= n { - let i = o as usize - n; - let cr = &self.ov.created; + fn attr_ids(&self, (kind, i): Slot, a: Attribute) -> Option<(u32, u32)> { + let (p, o) = self.pop(kind); + if i >= p.len() { + let c = &o.created; + let j = i - p.len(); return Some(match a { - Attribute::Upn => (*cr.upn_val.get(i)?, cr.upn_key[i]), - Attribute::PrimarySmtp => (*cr.smtp_val.get(i)?, cr.smtp_key[i]), + Attribute::Upn => (*c.upn_val.get(j)?, c.upn_key[j]), + Attribute::PrimarySmtp => (*c.smtp_val.get(j)?, c.smtp_key[j]), }); } - if let Some(ids) = self.ov.overrides(a).get(&o) { + if let Some(ids) = o.overrides(a).get(&(i as u16)) { return Some(*ids); } - let s = self.snap; Some(match a { - Attribute::Upn => (*s.upn_val.get(o as usize)?, s.upn_key[o as usize]), - Attribute::PrimarySmtp => (*s.smtp_val.get(o as usize)?, s.smtp_key[o as usize]), + Attribute::Upn => (*p.upn_val.get(i)?, p.upn_key[i]), + Attribute::PrimarySmtp => (*p.smtp_val.get(i)?, p.smtp_key[i]), }) } - /// Effective raw value of an attribute — the one place a value is - /// resolved to a string (compare-and-set, evidence, plan). - pub fn attr(&self, node: u32, a: Attribute) -> Option<&'s str> { - let (val, _) = self.attr_ids(node, a)?; - (val != NONE) - .then(|| self.dicts.values.resolve(val)) - .flatten() + /// Effective value of an attribute of an existing node. + pub fn attr(&self, node: &Guid128, a: Attribute) -> Option { + self.slot_attr(self.locate(node)?, a) + } + pub(crate) fn slot_attr(&self, slot: Slot, a: Attribute) -> Option { + let (v, _) = self.attr_ids(slot, a)?; + (v != NONE).then_some(ValueId(v)) } - /// The canonical semantic state of an existing node (evidence / - /// compare-and-set boundary: resolves strings). + /// The canonical semantic state of an existing node — ids only. pub fn node_state(&self, g: &Guid128) -> Option { - let o = self.ordinal(g)?; - let n = self.snap.len(); - let (kind, active, ou) = if (o as usize) < n { - let s = self.snap; - let ou = bit(&s.ou_present, o).then(|| s.ou[o as usize]); - if bit(&s.user, o) { - (NodeKind::User, bit(&s.active_user, o), ou) - } else { - // Groups have no enabled flag (see `ApplyError::InactiveGroup`). - (NodeKind::Group, true, ou) - } + let (kind, i) = self.locate(g)?; + let (p, o) = self.pop(kind); + let (active, dn) = if i < p.len() { + let active = kind == NodeKind::Group || bit(&p.active, i); + (active, p.dn_of(i)) } else { - let cr = &self.ov.created; - let i = o as usize - n; - (cr.kind[i], cr.active[i], cr.ou[i]) + let j = i - p.len(); + (o.created.active[j], o.created.dn[j]) }; - let text = |a| self.attr(o, a).map(str::to_string); Some(NodeState { kind, active, - upn: text(Attribute::Upn), - primary_smtp: text(Attribute::PrimarySmtp), - ou, + upn: self.slot_attr((kind, i), Attribute::Upn), + primary_smtp: self.slot_attr((kind, i), Attribute::PrimarySmtp), + dn, }) } - /// The base membership rows still live in this version. + /// The resolved membership rows still live in this version. pub(crate) fn live_rows(&self) -> Cow<'s, [u64]> { if self.ov.removed.is_empty() { return Cow::Borrowed(&self.snap.m_all); @@ -345,49 +405,75 @@ impl<'s> View<'s> { Cow::Owned(p) } - /// Base active users (base width) minus deleted nodes and the nodes + /// Base active users (base width) minus deleted users and the users /// whose attribute `a` is overridden — the base rows that still own /// their observed value. pub(crate) fn live_owners(&self, a: Attribute) -> Cow<'s, [u64]> { - let ov = self.ov.overrides(a); - if ov.is_empty() && self.ov.deleted.is_empty() { - return Cow::Borrowed(&self.snap.active_user); + let o = &self.ov.users; + let ov = o.overrides(a); + if ov.is_empty() && o.deleted.is_empty() { + return Cow::Borrowed(&self.snap.users.active); } - let mut p = self.snap.active_user.clone(); - for o in ov.keys().chain(&self.ov.deleted) { - clear_bit(&mut p, *o as usize); + let mut p = self.snap.users.active.clone(); + for d in ov.keys().chain(&o.deleted) { + clear_bit(&mut p, usize::from(*d)); } Cow::Owned(p) } - /// Added memberships as delta-sized ordinal lanes (`NONE` = endpoint - /// does not exist in this version) plus their identities. - pub(crate) fn added_rows(&self) -> (Vec, Vec, Vec<(Guid128, Guid128)>) { - let mut u = Vec::with_capacity(self.ov.added.len()); - let mut g = Vec::with_capacity(self.ov.added.len()); - for (a, b) in &self.ov.added { - u.push(self.ordinal(a).unwrap_or(NONE)); - g.push(self.ordinal(b).unwrap_or(NONE)); + /// The live memberships held by identity — added in the overlay, or + /// observed with an endpoint the snapshot lacked — split by resolution + /// against THIS view: delta-sized `(user, group)` ordinal lanes for the + /// pairs whose endpoints both exist now (an observed pair resolves once + /// a version creates its missing endpoint), and the identities of the + /// rest (dangling). Evidence-sized: the snapshot's unresolved table + /// plus the overlay. + pub(crate) fn added_rows(&self) -> AddedRows { + let mut out = AddedRows::default(); + let observed = self + .snap + .m_unresolved + .iter() + .filter(|p| !self.ov.removed_unresolved.contains(p)); + for &(u, g) in self.ov.added.iter().chain(observed) { + match (self.user_ordinal(&u), self.group_ordinal(&g)) { + (Some(uo), Some(go)) => { + out.users.push(u32::from(uo.0)); + out.groups.push(u32::from(go.0)); + } + _ => out.unresolved.push((u, g)), + } } - (u, g, self.ov.added.iter().copied().collect()) + out } /// Whether `node` still takes part in a live membership, on either side. /// - /// Base relation: one `Count` program over the two membership lanes - /// gated by the observed rows (borrowed, no plane built), minus the - /// removed rows touching the node (delta-sized). The relation is sorted - /// by user, not by group, so the group side has no index: the work is a - /// scan of the membership lanes, the allocation is one scratch tile. + /// Resolved base rows: one `Count` program over the membership lane of + /// the node's side, gated by the observed rows (borrowed, no plane + /// built), minus the removed rows touching the node (delta-sized). The + /// relation is sorted by user, not by group, so the group side has no + /// index: the work is a scan of one lane, the allocation one scratch + /// tile. Unresolved and added rows are delta- or evidence-sized. pub(crate) fn has_membership(&self, node: &Guid128) -> bool { if self.ov.added.iter().any(|(u, g)| u == node || g == node) { return true; } + if self + .snap + .m_unresolved + .iter() + .any(|p| (p.0 == *node || p.1 == *node) && !self.ov.removed_unresolved.contains(p)) + { + return true; + } let s = self.snap; - let Some(o) = s.ordinal(node) else { - return false; + let (lane, o) = match (s.users.ordinal(node), s.groups.ordinal(node)) { + (Some(o), _) => (&s.m_user, o), + (None, Some(o)) => (&s.m_group, o), + (None, None) => return false, }; - let lanes = [LaneRef::U32(&s.m_user), LaneRef::U32(&s.m_group)]; + let lanes = [LaneRef::U32(lane)]; let masks: [&[u64]; 1] = [&s.m_all]; let planes = Planes { n_rows: s.membership_rows(), @@ -397,10 +483,7 @@ impl<'s> View<'s> { let observed = crate::exec::count( Filter::and([ Filter::plane(Mask(0)), - Filter::or([ - Filter::cmp(Col(0), Cmp::EqU32(o)), - Filter::cmp(Col(1), Cmp::EqU32(o)), - ]), + Filter::cmp(Col(0), Cmp::EqU32(u32::from(o))), ]), &planes, &Foreign::NONE, @@ -409,11 +492,17 @@ impl<'s> View<'s> { .ov .removed .iter() - .filter(|&&r| s.m_user[r as usize] == o || s.m_group[r as usize] == o) + .filter(|&&r| lane[r as usize] == u32::from(o)) .count(); observed > removed } + fn lane_ids(&self, v: Option) -> Result<(u32, u32), ApplyError> { + self.dicts + .lane_ids(v) + .ok_or_else(|| ApplyError::Uninterned(v.expect("None always resolves"))) + } + /// Apply one change to the overlay. Pure with respect to everything but /// `self.ov`; the snapshot is never touched. pub(crate) fn apply(&mut self, c: &Change) -> Result<(), ApplyError> { @@ -422,24 +511,26 @@ impl<'s> View<'s> { if self.is_member(user, group) { return Err(ApplyError::NoOp(c.clone())); } - match self.snap.member_row(user, group) { - Some(r) => { - self.ov.removed.remove(&r); - } - None => { - self.ov.added.insert((*user, *group)); - } + if let Some(r) = self.snap.member_row(user, group) { + self.ov.removed.remove(&r); + } else if !self.ov.removed_unresolved.remove(&(*user, *group)) { + self.ov.added.insert((*user, *group)); } } Change::RemoveMembership { user, group } => { + if !self.is_member(user, group) { + return Err(ApplyError::NoOp(c.clone())); + } if self.ov.added.remove(&(*user, *group)) { return Ok(()); } match self.snap.member_row(user, group) { - Some(r) if self.is_member(user, group) => { + Some(r) => { self.ov.removed.insert(r); } - _ => return Err(ApplyError::NoOp(c.clone())), + None => { + self.ov.removed_unresolved.insert((*user, *group)); + } } } Change::SetAttribute { @@ -448,71 +539,80 @@ impl<'s> View<'s> { from, to, } => { - let o = self.ordinal(node).ok_or(ApplyError::UnknownNode(*node))?; - let actual = self.attr(o, *attribute); - if actual != from.as_deref() { + let slot = self.locate(node).ok_or(ApplyError::UnknownNode(*node))?; + let actual = self.slot_attr(slot, *attribute); + if actual != *from { return Err(ApplyError::Stale { node: *node, attribute: *attribute, - expected: from.clone(), - actual: actual.map(str::to_string), + expected: *from, + actual, }); } - let ids = self - .dicts - .lookup_attr(to.as_deref()) - .ok_or(ApplyError::Uninterned)?; - let n = self.snap.len(); - if (o as usize) >= n { - let (val, key) = self.ov.created.lanes(*attribute); - val[o as usize - n] = ids.0; - key[o as usize - n] = ids.1; + let ids = self.lane_ids(*to)?; + let (kind, i) = slot; + let base_len = self.pop(kind).0.len(); + let base_val = { + let p = self.pop(kind).0; + match attribute { + Attribute::Upn => p.upn_val.get(i).copied(), + Attribute::PrimarySmtp => p.smtp_val.get(i).copied(), + } + }; + let o = self.pop_mut(kind); + if i >= base_len { + let (val, key) = o.created.lanes(*attribute); + val[i - base_len] = ids.0; + key[i - base_len] = ids.1; return Ok(()); } - let base = match attribute { - Attribute::Upn => self.snap.upn_val[o as usize], - Attribute::PrimarySmtp => self.snap.smtp_val[o as usize], - }; let map = match attribute { - Attribute::Upn => &mut self.ov.upn, - Attribute::PrimarySmtp => &mut self.ov.smtp, + Attribute::Upn => &mut o.upn, + Attribute::PrimarySmtp => &mut o.smtp, }; // Net effect only: setting a value back to the observed one // removes the override instead of recording a no-op change. - if ids.0 == base { - map.remove(&o); + if Some(ids.0) == base_val { + map.remove(&(i as u16)); } else { - map.insert(o, ids); + map.insert(i as u16, ids); } } Change::CreateNode { node, state } => { if state.kind == NodeKind::Group && !state.active { return Err(ApplyError::InactiveGroup(*node)); } - // Identity uniqueness: one node per Guid128 in a version. + // Identity uniqueness: one node per Guid128 in a version, + // across both populations. if self.exists(node) { return Err(ApplyError::NodeExists(*node)); } - if let Some(o) = self.snap.ordinal(node) { + let kind = state.kind; + if let Some(o) = self.snap.population(kind).ordinal(node) { // Deleted in this lineage: recreating the observed node // unchanged is an undo; anything else reuses an identity. - self.ov.deleted.remove(&o); + self.pop_mut(kind).deleted.remove(&o); if self.node_state(node).as_ref() != Some(state) { - self.ov.deleted.insert(o); + self.pop_mut(kind).deleted.insert(o); return Err(ApplyError::IdentityReused(*node)); } return Ok(()); } - let upn = self - .dicts - .lookup_attr(state.upn.as_deref()) - .ok_or(ApplyError::Uninterned)?; - let smtp = self - .dicts - .lookup_attr(state.primary_smtp.as_deref()) - .ok_or(ApplyError::Uninterned)?; - let at = self.ov.created.find(node).unwrap_err(); - self.ov.created.insert(at, *node, state, upn, smtp); + if self.snap.population(other(kind)).ordinal(node).is_some() { + return Err(ApplyError::IdentityReused(*node)); + } + let max = match kind { + NodeKind::User => MAX_USERS, + NodeKind::Group => MAX_GROUPS, + }; + if self.len_in(kind) >= max { + return Err(ApplyError::PopulationFull(kind)); + } + let upn = self.lane_ids(state.upn)?; + let smtp = self.lane_ids(state.primary_smtp)?; + let o = self.pop_mut(kind); + let at = o.created.find(node).unwrap_err(); + o.created.insert(at, *node, state, upn, smtp); } Change::DeleteNode { node, state } => { let actual = self @@ -529,20 +629,36 @@ impl<'s> View<'s> { if self.has_membership(node) { return Err(ApplyError::NodeHasMemberships(*node)); } - match self.ov.created.find(node) { + let kind = actual.kind; + let base = self.snap.population(kind).ordinal(node); + let o = self.pop_mut(kind); + match (o.created.find(node), base) { // Create-then-delete nets out to nothing. - Ok(i) => self.ov.created.remove(i), - Err(_) => { - let Some(o) = self.snap.ordinal(node) else { - return Err(ApplyError::UnknownNode(*node)); - }; - self.ov.upn.remove(&o); - self.ov.smtp.remove(&o); - self.ov.deleted.insert(o); + (Ok(i), _) => o.created.remove(i), + (Err(_), Some(b)) => { + o.upn.remove(&b); + o.smtp.remove(&b); + o.deleted.insert(b); } + (Err(_), None) => return Err(ApplyError::UnknownNode(*node)), } } } Ok(()) } } + +/// [`View::added_rows`]: resolved delta rows as lanes, unresolved as ids. +#[derive(Debug, Default)] +pub(crate) struct AddedRows { + pub(crate) users: Vec, + pub(crate) groups: Vec, + pub(crate) unresolved: Vec<(Guid128, Guid128)>, +} + +fn other(kind: NodeKind) -> NodeKind { + match kind { + NodeKind::User => NodeKind::Group, + NodeKind::Group => NodeKind::User, + } +} diff --git a/crates/lance-graph-dir-sim/tests/alloc.rs b/crates/lance-graph-dir-sim/tests/alloc.rs index 1f6a32017..c0ffc5563 100644 --- a/crates/lance-graph-dir-sim/tests/alloc.rs +++ b/crates/lance-graph-dir-sim/tests/alloc.rs @@ -3,11 +3,16 @@ //! proportional to the delta, not to the directory. Detects an accidentally //! population-copying or row-exploding version path. //! +//! Scales: 1,000, 16,384 and 65,536 users — the last is the population +//! bound ([`MAX_USERS`]). Larger directories are no longer representable. +//! //! Own test binary (the counting allocator is process-global). use lance_graph_dir_sim::*; use ogar_dir_core::Guid128; -use ogar_dir_sim::{Attribute, Change, EvidenceRef, NodeKind, NodeState, RuleId, VersionId}; +use ogar_dir_sim::{ + Attribute, Change, EvidenceRef, NodeKind, NodeState, RuleId, ValueId, VersionId, +}; use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -36,12 +41,12 @@ fn guid(i: u32) -> Guid128 { const LONER: u32 = u32::MAX - 2; -/// A directory of `n` users, each in `employees`, plus a user in no group -/// and an empty `exchange` group. -fn directory(n: u32) -> (VersionStore, VersionId) { +/// A directory of `users` users — all but one in `employees`, plus a user +/// in no group — and an empty `exchange` group. +fn directory(users: usize) -> (VersionStore, VersionId) { let (employees, exchange) = (guid(u32::MAX - 1), guid(u32::MAX)); let mut obs = Observation::default(); - for i in 0..n { + for i in 0..(users - 1) as u32 { obs.nodes.push(( guid(i), ObservedNode::user(&format!("u{i}@example.test"), &format!("u{i}@example.test")), @@ -81,8 +86,16 @@ enum Op { DeleteNode, } -fn change(op: Op, st: &VersionStore, g0: VersionId) -> Change { +/// The change for `op`. Its values are interned here, at ingress, outside +/// the measured window — as a real caller would before simulating. +fn change(op: Op, st: &mut VersionStore, g0: VersionId) -> Change { let (employees, exchange) = (guid(u32::MAX - 1), guid(u32::MAX)); + let mut v = |s: &str| -> Option { Some(st.intern(s)) }; + let (u7, renamed, new) = ( + v("u7@example.test"), + v("renamed@example.test"), + v("new@example.test"), + ); match op { Op::AddMembership => Change::AddMembership { user: guid(7), @@ -95,17 +108,17 @@ fn change(op: Op, st: &VersionStore, g0: VersionId) -> Change { Op::SetAttribute => Change::SetAttribute { node: guid(7), attribute: Attribute::PrimarySmtp, - from: Some("u7@example.test".into()), - to: Some("renamed@example.test".into()), + from: u7, + to: renamed, }, Op::CreateNode => Change::CreateNode { node: guid(u32::MAX - 3), state: NodeState { kind: NodeKind::User, active: true, - upn: Some("new@example.test".into()), - primary_smtp: Some("new@example.test".into()), - ou: None, + upn: new, + primary_smtp: new, + dn: None, }, }, Op::DeleteNode => Change::DeleteNode { @@ -116,11 +129,17 @@ fn change(op: Op, st: &VersionStore, g0: VersionId) -> Change { } /// Bytes allocated by `simulate` and by `diff` for one change of kind `op` -/// in a directory of `n` users. The change itself is built outside the +/// in a directory of `n` users (a create starts one below, so the version +/// it makes holds exactly `n`). The change itself is built outside the /// measured window. -fn measure(op: Op, n: u32) -> (usize, usize) { - let (mut st, g0) = directory(n); - let rule = Propose(vec![change(op, &st, g0)]); +fn measure(op: Op, n: usize) -> (usize, usize) { + let users = if matches!(op, Op::CreateNode) { + n - 1 + } else { + n + }; + let (mut st, g0) = directory(users); + let rule = Propose(vec![change(op, &mut st, g0)]); let ev = vec![EvidenceRef("REQ-1".into())]; let before = BYTES.load(Ordering::Relaxed); @@ -144,24 +163,25 @@ fn one_mutation_of_each_kind_is_delta_sized_not_population_sized() { Op::DeleteNode, ] { let (s1, d1) = measure(op, 1_000); - let (s100, d100) = measure(op, 100_000); - let (s200, d200) = measure(op, 200_000); + let (s16, d16) = measure(op, 16_384); + let (s64, d64) = measure(op, MAX_USERS); eprintln!( - "{op:?}: simulate {s1} / {s100} / {s200} B, diff {d1} / {d100} / {d200} B \ - @ 1k / 100k / 200k users" + "{op:?}: simulate {s1} / {s16} / {s64} B, diff {d1} / {d16} / {d64} B \ + @ 1k / 16,384 / 65,536 users" ); - // A copy of even one u32 lane at 100k users is 400 KB; a node or - // membership bitmap is 12.5 KB. Delta-sized work stays far below both. + // A copy of one u32 lane at 65,536 users is 256 KB, a node bitmap + // 8 KB, the u16 ordinal space itself 128 KB. Delta-sized work stays + // below all of them. assert!( - s200 < 8_192 && d200 < 8_192, - "{op:?}: {s200} / {d200} B @200k" + s64 < 8_192 && d64 < 8_192, + "{op:?}: {s64} / {d64} B @65,536" ); - // Doubling a directory that is already large changes nothing. + // Quadrupling a directory that is already a full tile changes nothing. assert!( - s200 <= s100 + 256 && d200 <= d100 + 256, - "{op:?}: allocation grew with population past 100k" + s64 <= s16 + 256 && d64 <= d16 + 256, + "{op:?}: allocation grew with population past 16,384" ); - // From 1k to 100k only the delete guard may grow, and only by its + // From 1k to 16,384 only the delete guard may grow, and only by its // `Count` program's scratch: one tile per slot, a tile capped at // `TILE_WORDS` (16,384 rows), so constant above that size. let tile_growth = if matches!(op, Op::DeleteNode) { @@ -170,8 +190,8 @@ fn one_mutation_of_each_kind_is_delta_sized_not_population_sized() { 256 }; assert!( - s100 <= s1 + tile_growth && d100 <= d1 + tile_growth, - "{op:?}: allocation grew with population from 1k to 100k" + s16 <= s1 + tile_growth && d16 <= d1 + tile_growth, + "{op:?}: allocation grew with population from 1k to 16,384" ); } } diff --git a/crates/lance-graph-dir-sim/tests/bounds.rs b/crates/lance-graph-dir-sim/tests/bounds.rs new file mode 100644 index 000000000..1cec645e3 --- /dev/null +++ b/crates/lance-graph-dir-sim/tests/bounds.rs @@ -0,0 +1,501 @@ +//! The bounded numeric substrate: two independent 65,536-node ordinal +//! spaces, sparse `u16 × u16` membership, the `Dn128` hierarchy, and the +//! stable `ValueId` boundary between strings and execution. + +use lance_graph_dir_sim::*; +use ogar_dir_core::{DirectoryScope, Dn128, Dn128Error, Guid128}; +use ogar_dir_sim::{ + Attribute, Change, EvidenceRef, NodeState, Operation, Precondition, RuleId, VersionId, +}; + +const SCOPE: DirectoryScope = DirectoryScope(Guid128([0x5C; 16])); + +/// Users are `0x00…`, groups `0x01…`, so each kind's ordinal is `i`. +fn user(i: u32) -> Guid128 { + let mut b = [0u8; 16]; + b[1..5].copy_from_slice(&i.to_be_bytes()); + Guid128(b) +} +fn group(i: u32) -> Guid128 { + let mut b = user(i).0; + b[0] = 1; + Guid128(b) +} +fn bare(kind: NodeKind) -> ObservedNode { + ObservedNode { + kind, + active: true, + upn: None, + primary_smtp: None, + dn: None, + } +} +fn population(users: u32, groups: u32) -> Observation { + let mut obs = Observation { + scope: SCOPE, + ..Observation::default() + }; + obs.nodes + .extend((0..users).map(|i| (user(i), bare(NodeKind::User)))); + obs.nodes + .extend((0..groups).map(|i| (group(i), bare(NodeKind::Group)))); + obs +} + +struct Propose(Vec); +impl Rule for Propose { + fn id(&self) -> RuleId { + RuleId { + name: "Propose", + version: 1, + } + } + fn propose(&self, _: &View<'_>, _: &[EvidenceRef]) -> Vec { + self.0.clone() + } +} +fn apply_err(r: Result) -> ApplyError { + match r { + Err(SimError::Apply(e)) => e, + other => panic!("expected an apply error, got {other:?}"), + } +} + +// ---- 1. two independent 64k ordinal spaces ------------------------------- + +#[test] +fn both_populations_reach_65536_at_once_and_every_u16_is_an_ordinal() { + let n = MAX_USERS as u32; + assert_eq!(MAX_GROUPS, MAX_USERS); + let mut obs = population(n, n); + // The last ordinal on both sides, as one membership row: no value of a + // u16 is reserved, so (65535, 65535) is an ordinary row. + obs.members.push((user(n - 1), group(n - 1))); + obs.members.push((user(0), group(0))); + let mut st = VersionStore::new(); + let v = st.observe("lab", 0, obs).unwrap(); + let view = st.view(v).unwrap(); + assert_eq!( + (view.users_len(), view.groups_len()), + (MAX_USERS, MAX_GROUPS) + ); + // 131,072 nodes in total: one shared space would not hold them. + assert_eq!(view.user_ordinal(&user(n - 1)), Some(UserOrdinal(u16::MAX))); + assert_eq!( + view.group_ordinal(&group(n - 1)), + Some(GroupOrdinal(u16::MAX)) + ); + assert_eq!(view.user_ordinal(&user(0)), Some(UserOrdinal(0))); + assert_eq!(view.group_ordinal(&group(0)), Some(GroupOrdinal(0))); + assert_eq!(view.user_guid(UserOrdinal(u16::MAX)), Some(user(n - 1))); + assert!(view.is_member(&user(n - 1), &group(n - 1))); + assert!(!view.is_member(&user(n - 1), &group(0))); + assert_eq!(st.snapshot(v).unwrap().membership_rows(), 2); + assert!(st.validate(v).unwrap().is_empty()); +} + +#[test] +fn a_population_past_65536_is_refused_per_kind() { + let n = MAX_USERS as u32; + let mut st = VersionStore::new(); + assert_eq!( + st.observe("lab", 0, population(n + 1, 0)), + Err(BuildError::TooManyUsers(MAX_USERS + 1)) + ); + assert_eq!( + st.observe("lab", 0, population(0, n + 1)), + Err(BuildError::TooManyGroups(MAX_GROUPS + 1)) + ); +} + +#[test] +fn a_full_population_refuses_a_create_and_the_other_one_does_not() { + let n = MAX_USERS as u32; + let mut st = VersionStore::new(); + let g0 = st.observe("lab", 0, population(n, 1)).unwrap(); + let state = |kind| NodeState { + kind, + active: true, + upn: None, + primary_smtp: None, + dn: None, + }; + let more_users = Propose(vec![Change::CreateNode { + node: user(n), + state: state(NodeKind::User), + }]); + assert_eq!( + apply_err(st.simulate(g0, &more_users, &[])), + ApplyError::PopulationFull(NodeKind::User) + ); + let more_groups = Propose(vec![Change::CreateNode { + node: group(1), + state: state(NodeKind::Group), + }]); + let v = st.simulate(g0, &more_groups, &[]).unwrap(); + assert_eq!(st.view(v).unwrap().groups_len(), 2); + // A delete frees no ordinal for reuse: the slot stays, the bound holds. + let gone = Propose(vec![Change::DeleteNode { + node: user(0), + state: state(NodeKind::User), + }]); + let d = st.simulate(g0, &gone, &[]).unwrap(); + assert_eq!( + apply_err(st.simulate(d, &more_users, &[])), + ApplyError::PopulationFull(NodeKind::User) + ); +} + +// ---- 2. sparse membership ------------------------------------------------ + +#[test] +fn an_unresolved_membership_is_kept_by_identity_never_as_a_lane_value() { + let mut obs = population(2, 2); + obs.members.push((user(0), group(9))); // no such group + obs.members.push((group(0), group(1))); // a group as member + obs.members.push((user(1), group(1))); + let mut st = VersionStore::new(); + let v = st.observe("lab", 0, obs).unwrap(); + // Only the resolved row is in the lanes. + assert_eq!(st.snapshot(v).unwrap().membership_rows(), 1); + let view = st.view(v).unwrap(); + assert!(view.is_member(&user(0), &group(9)), "still observed"); + assert_eq!(st.validate(v).unwrap().len(), 2); + // And it can be removed like any other membership. + let fix = Propose(vec![ + Change::RemoveMembership { + user: user(0), + group: group(9), + }, + Change::RemoveMembership { + user: group(0), + group: group(1), + }, + ]); + let w = st.simulate(v, &fix, &[]).unwrap(); + assert!(st.validate(w).unwrap().is_empty()); + assert_eq!(st.diff(v, w).unwrap().len(), 2); +} + +// ---- 3. Dn128 hierarchy -------------------------------------------------- + +#[test] +fn dn128_subtree_at_depth_1_4_and_16() { + let dn = |l: &[u8]| Dn128::new(l).unwrap(); + let path16: Vec = (0..16).map(|k| (k * 17) as u8).collect(); + let mut sibling16 = path16.clone(); + sibling16[15] ^= 1; + let located = [ + dn(&path16), // user 0: the full depth-16 path + dn(&sibling16), // user 1: same parent, different leaf + dn(&path16[..4]), // user 2: an ancestor at depth 4 + dn(&path16[..1]), // user 3: an ancestor at depth 1 + dn(&[path16[0] ^ 1]), // user 4: a sibling at depth 1 + // user 5: the depth-3 ancestor. Its zero tail equals the byte a + // `[0, 17, 34, 0]` query compares at level 3. + dn(&path16[..3]), + ]; + let mut obs = population(located.len() as u32 + 1, 0); // last user unlocated + for (i, d) in located.iter().enumerate() { + obs.nodes[i].1.dn = Some(*d); + } + let mut st = VersionStore::new(); + let v = st.observe("lab", 0, obs).unwrap(); + let view = st.view(v).unwrap(); + let pick = |p: &Dn128| subtree(&view, SCOPE, NodeKind::User, p).unwrap().rows(); + assert_eq!( + pick(&dn(&path16)), + vec![0], + "depth 16: the leaf, not its sibling" + ); + assert_eq!(pick(&dn(&path16[..15])), vec![0, 1], "the shared parent"); + assert_eq!( + pick(&dn(&path16[..4])), + vec![0, 1, 2], + "depth 4: self and below" + ); + // Anti-vacuity: the 16-byte compare alone accepts user 5 here … + let (pattern, care) = dn(&[0, 17, 34, 0]).subtree_mask(); + let bytes = located[5].bytes(); + assert!((0..16).all(|k| (bytes[k] ^ pattern[k]) & care[k] == 0)); + // … and the depth gate is what keeps it out. + assert_eq!(pick(&dn(&[0, 17, 34, 0])), Vec::::new()); + assert_eq!(pick(&dn(&path16[..3])), vec![0, 1, 2, 5], "depth 3"); + assert_eq!(pick(&dn(&path16[..1])), vec![0, 1, 2, 3, 5], "depth 1"); + assert_eq!( + pick(&Dn128::ROOT).len(), + located.len(), + "every located user" + ); + // The ancestor is answered from the node's own bytes, by OGAR's rule. + for i in pick(&dn(&path16[..4])) { + assert!(dn(&path16[..4]).is_ancestor_of(&located[i])); + } +} + +#[test] +fn a_seventeenth_level_and_a_257th_child_are_refused() { + assert_eq!(Dn128::new(&[0; 17]), Err(Dn128Error::TooDeep(17))); + // 257 OUs under one parent, through the real ingress path. + let mut ldif = String::new(); + for i in 0..257u32 { + let mut guid = [0u8; 16]; + guid[..4].copy_from_slice(&i.to_be_bytes()); + guid[15] = 1; + ldif.push_str(&format!( + "dn: CN=U{i},OU=O{i},DC=example,DC=test\nobjectGUID:: {}\nobjectClass: user\n\n", + b64(&guid) + )); + } + let (mut d, mut p) = ( + ogar_dir_core::OuDictionary::new(), + ogar_dir_core::ValuePool::new(), + ); + let recs: Vec<_> = ogar_ad::ldif::parse(&ldif) + .unwrap() + .iter() + .map(|e| { + ogar_ad::encode(e, SCOPE.0, &mut d, &mut p, 0) + .unwrap() + .record + }) + .collect(); + // 256 children still convert… + assert!(observe::from_ad(SCOPE, &recs[..256], &p).is_ok()); + // …the 257th fails closed, naming the record, and nothing is observed. + assert_eq!( + observe::from_ad(SCOPE, &recs, &p).unwrap_err(), + observe::ObserveError::Location { + node: recs[256].node_guid(), + error: Dn128Error::ChildCodeOverflow { + level: 0, + segment: 257 + } + } + ); +} + +/// A `Dn128` holds no scope, so the scope is enforced at the edges: a record +/// from another directory is refused at ingress, and two versions from +/// different directories are neither diffed nor planned against each other. +#[test] +fn a_foreign_scope_is_refused_at_ingress_diff_and_plan() { + let other = DirectoryScope(Guid128([0x0D; 16])); + let ldif = |n: u8| { + format!( + "dn: CN=U,OU=Same,DC=example,DC=test\nobjectGUID:: {}\nobjectClass: user\n\n", + b64(&[n; 16]) + ) + }; + let (mut d, mut p) = ( + ogar_dir_core::OuDictionary::new(), + ogar_dir_core::ValuePool::new(), + ); + let mut rec = |n: u8, scope: DirectoryScope| { + let e = &ogar_ad::ldif::parse(&ldif(n)).unwrap()[0]; + ogar_ad::encode(e, scope.0, &mut d, &mut p, 0) + .unwrap() + .record + }; + let (local, foreign) = (rec(1, SCOPE), rec(2, other)); + // Same OU path, so the same Dn128: only the scope tells them apart. + assert_eq!(local.ou_hhtl(), foreign.ou_hhtl()); + assert_eq!( + observe::from_ad(SCOPE, &[local, foreign], &p).unwrap_err(), + (observe::ObserveError::ForeignScope { + node: foreign.node_guid(), + scope: other, + }) + ); + + let mut st = VersionStore::new(); + let here = st + .observe("ad", 0, observe::from_ad(SCOPE, &[local], &p).unwrap()) + .unwrap(); + let there = st + .observe("ad", 1, observe::from_ad(other, &[foreign], &p).unwrap()) + .unwrap(); + assert_eq!( + st.diff(here, there), + Err(SimError::ScopeMismatch(here, there)) + ); + // Desired lineage in one scope, latest observation in another: no plan. + let mut st = VersionStore::new(); + let mut obs = population(1, 1); + let g0 = st.observe("lab", 0, obs.clone()).unwrap(); + let add = Propose(vec![Change::AddMembership { + user: user(0), + group: group(0), + }]); + let g1 = st.simulate(g0, &add, &[]).unwrap(); + st.promote_desired(g1).unwrap(); + assert!(st.plan(g1).is_ok()); + obs.scope = other; + let o = st.observe("lab", 1, obs).unwrap(); + assert_eq!( + st.plan(g1), + Err(ogar_dir_sim::PlanError::ScopeMismatch { + basis: o, + target: g1 + }) + ); +} + +fn b64(bytes: &[u8]) -> String { + const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::new(); + for c in bytes.chunks(3) { + let n = c + .iter() + .enumerate() + .fold(0u32, |n, (i, b)| n | u32::from(*b) << (16 - 8 * i)); + for k in 0..4 { + if k <= c.len() { + out.push(T[(n >> (18 - 6 * k) & 63) as usize] as char); + } else { + out.push('='); + } + } + } + out +} + +// ---- 4. stable ValueId boundary ------------------------------------------ + +#[test] +fn a_value_id_keeps_its_meaning_from_ingress_to_plan_across_observations() { + let mut st = VersionStore::new(); + // Ingress first: the id is fixed before anything is observed. + let alice = st.intern("Alice@Example.test"); + let renamed = st.intern("a.smith@example.test"); + let mut obs = population(1, 0); + obs.nodes[0].1.primary_smtp = Some("Alice@Example.test".into()); + let g0 = st.observe("lab", 0, obs.clone()).unwrap(); + let view = st.view(g0).unwrap(); + assert_eq!(view.attr(&user(0), Attribute::PrimarySmtp), Some(alice)); + drop(view); + let rule = SetPrimarySmtp { + rule: RuleId { + name: "Rename", + version: 1, + }, + user: user(0), + to: renamed, + }; + let g1 = st.simulate(g0, &rule, &[]).unwrap(); + st.promote_desired(g1).unwrap(); + // A second observation re-interns the same string: same id. + st.observe("lab", 1, obs).unwrap(); + let plan = st.plan(g1).unwrap(); + assert_eq!( + plan.ops[0].op, + Operation::SetAttribute { + object: user(0), + attribute: Attribute::PrimarySmtp, + value: Some(renamed), + } + ); + assert_eq!( + plan.ops[0].precondition, + Precondition::AttributeEquals(Some(alice)) + ); + // Strings come back only at egress. + assert_eq!(st.value(alice), Some("Alice@Example.test")); + assert_eq!(st.value(renamed), Some("a.smith@example.test")); + // Comparison happens on keys: different raw values, one key. + let shout = st.intern("ALICE@EXAMPLE.TEST"); + assert_ne!(shout, alice); + assert_eq!(st.key_of(shout), st.key_of(alice)); + assert_eq!( + st.key_label(st.key_of(alice).unwrap()), + Some("alice@example.test") + ); +} + +#[test] +fn a_value_id_the_store_never_issued_is_refused() { + let mut st = VersionStore::new(); + let mut obs = population(1, 0); + obs.nodes[0].1.upn = Some("u@example.test".into()); + let g0 = st.observe("lab", 0, obs).unwrap(); + let from = st.view(g0).unwrap().attr(&user(0), Attribute::Upn); + let bogus = ValueId(4_000_000); + let set = Propose(vec![Change::SetAttribute { + node: user(0), + attribute: Attribute::Upn, + from, + to: Some(bogus), + }]); + assert_eq!( + apply_err(st.simulate(g0, &set, &[])), + ApplyError::Uninterned(bogus) + ); +} + +/// An observed membership whose group was missing resolves once a version +/// creates that group: it is no longer dangling, it is counted, and a +/// population rule sees it. +#[test] +fn an_unresolved_membership_resolves_when_its_endpoint_is_created() { + let mut obs = population(2, 1); + obs.members.push((user(0), group(7))); // group 7 not observed yet + let mut st = VersionStore::new(); + let g0 = st.observe("lab", 0, obs).unwrap(); + assert_eq!(st.validate(g0).unwrap().len(), 1, "dangling while missing"); + let create = Propose(vec![Change::CreateNode { + node: group(7), + state: NodeState { + kind: NodeKind::Group, + active: true, + upn: None, + primary_smtp: None, + dn: None, + }, + }]); + let g1 = st.simulate(g0, &create, &[]).unwrap(); + assert!( + st.validate(g1).unwrap().is_empty(), + "resolved by the create" + ); + let view = st.view(g1).unwrap(); + let counts = member_counts(&view, &group(7)); + assert_eq!(counts, vec![1, 0], "counted like any membership"); + // A rule over the now-resolved edge: members of 7 should be in 0. + let imply = ImplyGroup { + rule: RuleId { + name: "Imply", + version: 1, + }, + source: group(7), + target: group(0), + }; + assert_eq!( + imply.propose(&view, &[]), + vec![Change::AddMembership { + user: user(0), + group: group(0) + }] + ); + drop(view); + // Removing and re-adding the edge round-trips. + let edge = |add| { + let c = if add { + Change::AddMembership { + user: user(0), + group: group(7), + } + } else { + Change::RemoveMembership { + user: user(0), + group: group(7), + } + }; + Propose(vec![c]) + }; + let g2 = st.simulate(g1, &edge(false), &[]).unwrap(); + assert!(!st.view(g2).unwrap().is_member(&user(0), &group(7))); + assert_eq!(member_counts(&st.view(g2).unwrap(), &group(7)), vec![0, 0]); + let g3 = st.simulate(g2, &edge(true), &[]).unwrap(); + assert_eq!(member_counts(&st.view(g3).unwrap(), &group(7)), vec![1, 0]); + assert!(st.diff(g1, g3).unwrap().is_empty()); +} diff --git a/crates/lance-graph-dir-sim/tests/nodes.rs b/crates/lance-graph-dir-sim/tests/nodes.rs index 1e8ad07e2..c15d9a795 100644 --- a/crates/lance-graph-dir-sim/tests/nodes.rs +++ b/crates/lance-graph-dir-sim/tests/nodes.rs @@ -1,7 +1,7 @@ //! Node creation and deletion, and reconciliation against a re-observation. use lance_graph_dir_sim::*; -use ogar_dir_core::{Guid128, OuHhtl}; +use ogar_dir_core::{DirectoryScope, Dn128, Guid128}; use ogar_dir_sim::*; fn g(n: u8) -> Guid128 { @@ -32,13 +32,44 @@ impl Rule for Propose { } } +/// Every raw value these tests use, interned at ingress in this order before +/// anything is observed. The store is append-only, so `val(s)` — the index +/// here — is the id the store issues for `s` for its whole lifetime. +const VOCAB: &[&str] = &[ + "alice@example.test", + "bob@example.test", + "carol@example.test", + "new@example.test", + "other@example.test", + "ghost@example.test", + "heir@example.test", + "ALICE@example.test", + "a.smith@example.test", + "carol.renamed@example.test", + "old@example.test", + "typo@example.test", +]; +fn val(s: &str) -> ValueId { + let i = VOCAB.iter().position(|v| *v == s).expect("in VOCAB"); + ValueId(i as u32) +} +fn new_store() -> VersionStore { + let mut st = VersionStore::new(); + for (i, v) in VOCAB.iter().enumerate() { + assert_eq!(st.intern(v), ValueId(i as u32)); + } + st +} +const SCOPE: DirectoryScope = DirectoryScope(Guid128([0x5C; 16])); + fn user_state(name: &str) -> NodeState { + let v = val(&format!("{name}@example.test")); NodeState { kind: NodeKind::User, active: true, - upn: Some(format!("{name}@example.test")), - primary_smtp: Some(format!("{name}@example.test")), - ou: None, + upn: Some(v), + primary_smtp: Some(v), + dn: None, } } fn group_state() -> NodeState { @@ -47,11 +78,12 @@ fn group_state() -> NodeState { active: true, upn: None, primary_smtp: None, - ou: None, + dn: None, } } fn observed() -> Observation { Observation { + scope: SCOPE, nodes: vec![ ( g(ALICE), @@ -73,7 +105,7 @@ fn observed() -> Observation { } } fn store() -> (VersionStore, VersionId) { - let mut st = VersionStore::new(); + let mut st = new_store(); let g0 = st.observe("lab", 1_000, observed()).unwrap(); (st, g0) } @@ -115,12 +147,16 @@ fn create_user_grows_the_version_by_one_delta() { view.snapshot(), st.view(g0).unwrap().snapshot() )); - assert_eq!(view.len(), 6); - let o = view.ordinal(&g(NEW_USER)).unwrap(); - assert_eq!(o, 5, "created nodes follow the base ordinals"); - assert_eq!(view.guid(o), Some(g(NEW_USER))); + assert_eq!((view.users_len(), view.groups_len()), (4, 2)); + let o = view.user_ordinal(&g(NEW_USER)).unwrap(); + assert_eq!( + o, + UserOrdinal(3), + "created users follow the base user ordinals" + ); + assert_eq!(view.user_guid(o), Some(g(NEW_USER))); assert_eq!(view.node_state(&g(NEW_USER)), Some(user_state("new"))); - assert!(view.active_users()[0] >> o & 1 == 1); + assert!(view.active_users()[0] >> o.0 & 1 == 1); assert!(st.validate(v).unwrap().is_empty()); assert_eq!( st.diff(g0, v).unwrap(), @@ -181,10 +217,11 @@ fn delete_user_and_group() { assert_eq!(view.delta_len(), 2); assert!(!view.exists(&g(CAROL)) && !view.exists(&g(EXCHANGE))); // The slot remains, cleared from every live plane. - assert_eq!(view.len(), 5); - let carol_ord = st.view(g0).unwrap().ordinal(&g(CAROL)).unwrap(); - assert_eq!(view.guid(carol_ord), None); - assert!(view.active_users()[0] >> carol_ord & 1 == 0); + assert_eq!(view.users_len(), 3, "a deleted user keeps its slot"); + let carol_ord = st.view(g0).unwrap().user_ordinal(&g(CAROL)).unwrap(); + assert_eq!(view.user_guid(carol_ord), None); + assert_eq!(view.user_ordinal(&g(CAROL)), None); + assert!(view.active_users()[0] >> carol_ord.0 & 1 == 0); assert!(st.validate(v).unwrap().is_empty()); st.promote_desired(v).unwrap(); let plan = st.plan(v).unwrap(); @@ -261,7 +298,7 @@ fn create_is_refused_for_an_existing_identity() { fn delete_is_compare_and_set() { let (mut st, g0) = store(); let stale = NodeState { - primary_smtp: Some("old@example.test".into()), + primary_smtp: Some(val("old@example.test")), ..st.view(g0).unwrap().node_state(&g(CAROL)).unwrap() }; assert!(matches!( @@ -344,12 +381,12 @@ fn created_and_deleted_nodes_take_part_in_uniqueness() { let (mut st, g0) = store(); // A new user taking Alice's address collides… let mut clash = user_state("new"); - clash.primary_smtp = Some("ALICE@example.test".into()); + clash.primary_smtp = Some(val("ALICE@example.test")); let v = sim(&mut st, g0, vec![create(NEW_USER, clash)]).unwrap(); assert_eq!( st.validate(v).unwrap(), vec![Violation::DuplicateSmtp { - address: "alice@example.test".into(), + key: st.key_of(val("alice@example.test")).unwrap(), // Sorted by identity: 0x51.. < 0xA1.. owners: vec![g(NEW_USER), g(ALICE)], }] @@ -357,8 +394,8 @@ fn created_and_deleted_nodes_take_part_in_uniqueness() { // …but the address of a deleted user is free. let carol = st.view(g0).unwrap().node_state(&g(CAROL)).unwrap(); let mut reuse = user_state("new"); - reuse.primary_smtp = carol.primary_smtp.clone(); - reuse.upn = carol.upn.clone(); + reuse.primary_smtp = carol.primary_smtp; + reuse.upn = carol.upn; let w = sim( &mut st, g0, @@ -370,18 +407,14 @@ fn created_and_deleted_nodes_take_part_in_uniqueness() { #[test] fn subtree_sees_created_and_not_deleted_nodes() { - let ou = |l: &[u16]| { - let mut h = OuHhtl::ROOT; - h.0[..l.len()].copy_from_slice(l); - h - }; + let dn = |l: &[u8]| Dn128::new(l).unwrap(); let mut obs = observed(); - obs.nodes[2].1.ou = Some(ou(&[1, 2])); - let mut st = VersionStore::new(); + obs.nodes[2].1.dn = Some(dn(&[0, 1])); + let mut st = new_store(); let g0 = st.observe("lab", 0, obs).unwrap(); let carol = st.view(g0).unwrap().node_state(&g(CAROL)).unwrap(); let mut located = user_state("new"); - located.ou = Some(ou(&[1, 3])); + located.dn = Some(dn(&[0, 2])); let v = sim( &mut st, g0, @@ -389,8 +422,11 @@ fn subtree_sees_created_and_not_deleted_nodes() { ) .unwrap(); let view = st.view(v).unwrap(); - let rows = subtree(&view, &ou(&[1])).unwrap().rows(); - assert_eq!(rows, vec![view.ordinal(&g(NEW_USER)).unwrap() as usize]); + let rows = subtree(&view, SCOPE, NodeKind::User, &dn(&[0])) + .unwrap() + .rows(); + let new = view.user_ordinal(&g(NEW_USER)).unwrap(); + assert_eq!(rows, vec![usize::from(new.0)]); } #[test] @@ -410,7 +446,7 @@ fn input_order_does_not_change_the_output() { obs.nodes.reverse(); obs.members.reverse(); } - let mut st = VersionStore::new(); + let mut st = new_store(); let g0 = st.observe("lab", 0, obs).unwrap(); let v = sim(&mut st, g0, cs).unwrap(); st.promote_desired(v).unwrap(); @@ -469,8 +505,8 @@ fn desired(st: &mut VersionStore, g0: VersionId) -> VersionId { Change::SetAttribute { node: g(ALICE), attribute: Attribute::PrimarySmtp, - from: Some("alice@example.test".into()), - to: Some("a.smith@example.test".into()), + from: Some(val("alice@example.test")), + to: Some(val("a.smith@example.test")), }, ], ) @@ -537,9 +573,9 @@ fn reconcile_a_partial_observation_plans_only_the_rest() { op: Operation::SetAttribute { object: g(NEW_USER), attribute: Attribute::PrimarySmtp, - value: Some("new@example.test".into()), + value: Some(val("new@example.test")), }, - precondition: Precondition::AttributeEquals(Some("typo@example.test".into())), + precondition: Precondition::AttributeEquals(Some(val("typo@example.test"))), }, PlannedOp { op: Operation::AddGroupMember { @@ -568,7 +604,7 @@ fn reconcile_a_delete_reads_its_precondition_from_the_latest_observation() { assert_eq!( del.precondition, Precondition::ObjectRemovable(NodeState { - upn: Some("carol.renamed@example.test".into()), + upn: Some(val("carol.renamed@example.test")), ..user_state("carol") }) ); @@ -601,7 +637,7 @@ fn a_freed_address_is_released_before_it_is_claimed() { let (mut st, g0) = store(); let carol = st.view(g0).unwrap().node_state(&g(CAROL)).unwrap(); let mut heir = user_state("heir"); - heir.primary_smtp = carol.primary_smtp.clone(); + heir.primary_smtp = carol.primary_smtp; let v = sim( &mut st, g0, diff --git a/crates/lance-graph-dir-sim/tests/sim.rs b/crates/lance-graph-dir-sim/tests/sim.rs index 5cff64dad..a0c9aa4c9 100644 --- a/crates/lance-graph-dir-sim/tests/sim.rs +++ b/crates/lance-graph-dir-sim/tests/sim.rs @@ -3,7 +3,7 @@ use lance_graph_dir_sim::validate::{dangling, dangling_program, validate}; use lance_graph_dir_sim::*; use lance_graph_mask_risc::MaskOp; -use ogar_dir_core::{Guid128, OuHhtl}; +use ogar_dir_core::{DirectoryScope, Dn128, Guid128}; use ogar_dir_sim::*; use std::sync::Arc; @@ -28,8 +28,11 @@ const RENAME_MAIL: RuleId = RuleId { version: 1, }; +const SCOPE: DirectoryScope = DirectoryScope(Guid128([0x5C; 16])); + fn observed() -> Observation { Observation { + scope: SCOPE, nodes: vec![ ( g(ALICE), @@ -69,6 +72,11 @@ fn chain_from(obs: Observation) -> (VersionStore, VersionId, VersionId, VersionI let g2 = st.simulate(g1, &imply(), &ev("POLICY-7")).unwrap(); (st, g0, g1, g2) } +/// The comparison key of a value — interned at ingress like any value. +fn key_of(st: &mut VersionStore, s: &str) -> KeyId { + let v = st.intern(s); + st.key_of(v).unwrap() +} fn chain() -> (VersionStore, VersionId, VersionId, VersionId) { chain_from(observed()) } @@ -213,7 +221,7 @@ fn t07_duplicate_upn() { assert_eq!( st.validate(v0).unwrap(), vec![Violation::DuplicateUpn { - upn: "alice@example.test".into(), + key: key_of(&mut st, "alice@example.test"), owners: vec![g(0x77), g(ALICE)] }] ); @@ -234,10 +242,11 @@ fn t08_t09_smtp_collision_rejected() { .collect(); let snap_before = Arc::clone(st.snapshot(g0).unwrap()); + let to = st.intern("Alice@Example.test"); let rule = SetPrimarySmtp { rule: RENAME_MAIL, user: g(BOB), - to: "Alice@Example.test".into(), + to, }; let g3 = st .simulate(g2, &rule, &ev("REQ-2")) @@ -246,7 +255,7 @@ fn t08_t09_smtp_collision_rejected() { assert_eq!( rej.violations, vec![Violation::DuplicateSmtp { - address: "alice@example.test".into(), + key: key_of(&mut st, "alice@example.test"), owners: vec![g(ALICE), g(BOB)] }] ); @@ -261,17 +270,16 @@ fn t08_t09_smtp_collision_rejected() { "the base was never copied" ); assert_eq!(st.verdict(g3), Some(rej.violations.as_slice())); - let bob = st.view(g3).unwrap().ordinal(&g(BOB)).unwrap(); - assert_eq!( - st.view(g3).unwrap().attr(bob, Attribute::PrimarySmtp), - Some("Alice@Example.test") - ); + let got = st.view(g3).unwrap().attr(&g(BOB), Attribute::PrimarySmtp); + assert_eq!(got, Some(to)); + // Egress formatting is the only place the string comes back. + assert_eq!(st.value(to), Some("Alice@Example.test")); } // A stale compare-and-set creates no version. #[test] fn stale_change_creates_no_version() { - struct Stale; + struct Stale(ValueId, ValueId); impl Rule for Stale { fn id(&self) -> RuleId { RENAME_MAIL @@ -280,15 +288,19 @@ fn stale_change_creates_no_version() { vec![Change::SetAttribute { node: g(BOB), attribute: Attribute::PrimarySmtp, - from: Some("not-it@example.test".into()), - to: Some("x@example.test".into()), + from: Some(self.0), + to: Some(self.1), }] } } let mut st = VersionStore::new(); let g0 = st.observe("lab", 0, observed()).unwrap(); + let stale = Stale( + st.intern("not-it@example.test"), + st.intern("x@example.test"), + ); assert!(matches!( - st.simulate(g0, &Stale, &[]), + st.simulate(g0, &stale, &[]), Err(SimError::Apply(ApplyError::Stale { .. })) )); assert!(st.version(VersionId(1)).is_none()); @@ -323,13 +335,15 @@ fn t11_plan() { ); let mut s2 = VersionStore::new(); let b0 = s2.observe("lab", 0, observed()).unwrap(); + let robert = s2.intern("robert@example.test"); + let bob = s2.intern("bob@example.test"); let b1 = s2 .simulate( b0, &SetPrimarySmtp { rule: RENAME_MAIL, user: g(BOB), - to: "robert@example.test".into(), + to: robert, }, &[], ) @@ -341,9 +355,9 @@ fn t11_plan() { op: Operation::SetAttribute { object: g(BOB), attribute: Attribute::PrimarySmtp, - value: Some("robert@example.test".into()) + value: Some(robert) }, - precondition: Precondition::AttributeEquals(Some("bob@example.test".into())), + precondition: Precondition::AttributeEquals(Some(bob)), }] ); } @@ -371,12 +385,17 @@ fn t13_guid_ordinal_round_trip() { let mut st = VersionStore::new(); let v = st.observe("lab", 0, obs).unwrap(); let view = st.view(v).unwrap(); - for id in [a, b, g(ALICE), g(BOB), g(EMPLOYEES), g(EXCHANGE)] { - let o = view.ordinal(&id).unwrap(); - assert_eq!(view.guid(o), Some(id)); + for id in [a, b, g(EMPLOYEES), g(EXCHANGE)] { + let o = view.group_ordinal(&id).unwrap(); + assert_eq!(view.group_guid(o), Some(id)); + assert_eq!(view.user_ordinal(&id), None, "groups are not users"); } - assert_ne!(view.ordinal(&a), view.ordinal(&b)); - assert_eq!(view.ordinal(&g(0x99)), None); + for id in [g(ALICE), g(BOB)] { + let o = view.user_ordinal(&id).unwrap(); + assert_eq!(view.user_guid(o), Some(id)); + } + assert_ne!(view.group_ordinal(&a), view.group_ordinal(&b)); + assert_eq!(view.user_ordinal(&g(0x99)), None); } // 14. Membership validation reads only membership lanes and kind planes: it @@ -388,9 +407,10 @@ fn t14_membership_validation_reads_no_attributes() { active: true, upn: None, primary_smtp: None, - ou: None, + dn: None, }; let obs = Observation { + scope: SCOPE, nodes: vec![(g(1), node(NodeKind::User)), (g(2), node(NodeKind::Group))], members: vec![(g(1), g(2)), (g(1), g(3))], }; @@ -442,10 +462,11 @@ fn t17_ordering_invariance() { let outputs = |obs| { let (mut st, g0, _, g2) = chain_from(obs); st.promote_desired(g2).unwrap(); + let to = st.intern("alice@example.test"); let rule = SetPrimarySmtp { rule: RENAME_MAIL, user: g(BOB), - to: "alice@example.test".into(), + to, }; let g3 = st.simulate(g2, &rule, &[]).unwrap(); ( @@ -602,43 +623,46 @@ fn removal_round_trip() { ); } -// HHTL as executable geometry: subtree selection is one prefix match. +// Dn128 as executable geometry: subtree selection is one 16-byte prefix +// match plus a depth gate. #[test] -fn ou_subtree_is_a_prefix_match() { - let ou = |l: &[u16]| { - let mut h = OuHhtl::ROOT; - h.0[..l.len()].copy_from_slice(l); - h - }; +fn dn_subtree_is_a_prefix_match() { + let dn = |l: &[u8]| Dn128::new(l).unwrap(); let mut obs = observed(); - obs.nodes[0].1.ou = Some(ou(&[1, 1, 1])); // Stuttgart/Infrastructure/Exchange - obs.nodes[1].1.ou = Some(ou(&[2])); // Berlin + obs.nodes[0].1.dn = Some(dn(&[0, 0, 0])); // Stuttgart/Infrastructure/Exchange + obs.nodes[1].1.dn = Some(dn(&[1])); // Berlin + obs.nodes[2].1.dn = Some(dn(&[0])); // a group in Stuttgart obs.nodes.push(( g(0x55), ObservedNode { - ou: Some(ou(&[1, 2])), + dn: Some(dn(&[0, 1])), ..ObservedNode::user("c@x", "c@x") }, )); let mut st = VersionStore::new(); let v = st.observe("lab", 0, obs).unwrap(); let view = st.view(v).unwrap(); - let pick = |p: &OuHhtl| -> Vec { - subtree(&view, p) + let pick = |kind, p: &Dn128| -> Vec { + subtree(&view, SCOPE, kind, p) .unwrap() .rows() .into_iter() - .map(|o| view.guid(o as u32).unwrap()) + .map(|o| match kind { + NodeKind::User => view.user_guid(UserOrdinal(o as u16)).unwrap(), + NodeKind::Group => view.group_guid(GroupOrdinal(o as u16)).unwrap(), + }) .collect() }; - assert_eq!(pick(&ou(&[1])), vec![g(0x55), g(ALICE)]); - assert_eq!(pick(&ou(&[1, 1])), vec![g(ALICE)]); - assert_eq!(pick(&ou(&[2])), vec![g(BOB)]); - assert_eq!(pick(&OuHhtl::ROOT).len(), 3, "root = every located node"); - assert_eq!( - subtree(&view, &ou(&[1, 1, 1, 1, 1])), - Err(SubtreeTooDeep(5)) - ); + let u = NodeKind::User; + assert_eq!(pick(u, &dn(&[0])), vec![g(0x55), g(ALICE)]); + assert_eq!(pick(u, &dn(&[0, 0])), vec![g(ALICE)]); + assert_eq!(pick(u, &dn(&[1])), vec![g(BOB)]); + assert_eq!(pick(u, &Dn128::ROOT).len(), 3, "root = every located user"); + // The two populations are queried separately. + assert_eq!(pick(NodeKind::Group, &dn(&[0])), vec![g(EMPLOYEES)]); + // Codes from another directory are refused, not compared. + let other = DirectoryScope(Guid128([1; 16])); + assert!(subtree(&view, other, u, &dn(&[0])).is_err()); } // Observation from OGAR PR #313 records. @@ -651,18 +675,18 @@ fn observe_from_ogar_ad() { .unwrap() .iter() .map(|e| { - ogar_ad::encode(e, Guid128::NIL, &mut d, &mut p, 0) + ogar_ad::encode(e, SCOPE.0, &mut d, &mut p, 0) .unwrap() .record }) .collect(); - let obs = observe::from_ad(&recs, &p); + let obs = observe::from_ad(SCOPE, &recs, &p).unwrap(); assert_eq!( obs.nodes[0].1.primary_smtp.as_deref(), Some("alice@example.test") ); assert!(!obs.nodes[0].1.active, "UAC 514 = disabled"); - assert!(obs.nodes[0].1.ou.is_some()); + assert!(obs.nodes[0].1.dn.is_some()); assert_eq!(obs.nodes[1].1.kind, NodeKind::Group); let mut st = VersionStore::new(); let v = st.observe("ogar-ad:ldif", 0, obs).unwrap(); @@ -685,14 +709,15 @@ fn renaming_away_resolves_an_observed_collision() { assert_eq!( st.validate(g0).unwrap(), vec![Violation::DuplicateSmtp { - address: "bob@example.test".into(), + key: key_of(&mut st, "bob@example.test"), owners: vec![g(BOB), carol] }] ); + let to = st.intern("carol@example.test"); let fix = SetPrimarySmtp { rule: RENAME_MAIL, user: carol, - to: "carol@example.test".into(), + to, }; let g1 = st.simulate(g0, &fix, &[]).unwrap(); assert!(st.validate(g1).unwrap().is_empty()); diff --git a/crates/lance-graph-dir-sim/tests/string_fence.rs b/crates/lance-graph-dir-sim/tests/string_fence.rs new file mode 100644 index 000000000..7c0fc342c --- /dev/null +++ b/crates/lance-graph-dir-sim/tests/string_fence.rs @@ -0,0 +1,50 @@ +//! Source fence: the execution modules contain no text types. Text belongs +//! to ingress and egress — `snapshot.rs` (observation, the label/value +//! table), `observe.rs` (records) and `store.rs` (tags, `intern`/`value`). +//! Same shape as `lance-graph-report/tests/string_fence.rs`. + +const EXECUTION: &[&str] = &["exec.rs", "view.rs", "validate.rs", "rule.rs", "lib.rs"]; +const FORBIDDEN: &[&str] = &[ + "String", + "&str", + "str::", + "format!", + "to_string", + "char", + "normalize", +]; + +fn violations(src: &str) -> Vec { + src.lines() + .enumerate() + .filter(|(_, l)| { + let t = l.trim_start(); + !t.starts_with("//") && !t.starts_with("///") && !t.starts_with("//!") + }) + .filter(|(_, l)| FORBIDDEN.iter().any(|t| l.contains(t))) + .map(|(i, l)| format!("{}: {}", i + 1, l.trim())) + .collect() +} + +#[test] +fn execution_modules_are_string_free() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/"); + let mut bad = Vec::new(); + for f in EXECUTION { + let src = std::fs::read_to_string(format!("{dir}{f}")).unwrap(); + bad.extend(violations(&src).into_iter().map(|v| format!("{f}:{v}"))); + } + assert!( + bad.is_empty(), + "text in the execution modules:\n{}", + bad.join("\n") + ); +} + +#[test] +fn the_fence_can_fire() { + assert_eq!(violations(" pub to: String,").len(), 1); + assert_eq!(violations(" let k = normalize(s);").len(), 1); + assert!(violations(" /// a String is presentation").is_empty()); + assert!(violations(" pub to: ValueId,").is_empty()); +} diff --git a/crates/lance-graph-dir-sim/tests/where_eq.rs b/crates/lance-graph-dir-sim/tests/where_eq.rs new file mode 100644 index 000000000..7254d38d8 --- /dev/null +++ b/crates/lance-graph-dir-sim/tests/where_eq.rs @@ -0,0 +1,111 @@ +//! `WHERE smtp = 'Alice@X.de'` as a contract, not a frontend: the literal is +//! resolved once at the boundary, the executable query holds only numbers, +//! and execution performs no text operation. + +use lance_graph_dir_sim::*; +use lance_graph_mask_risc::{MaskOp, Pred}; +use ogar_dir_core::Guid128; +use ogar_dir_sim::{Attribute, Change, EvidenceRef, NodeState, RuleId}; + +fn g(n: u8) -> Guid128 { + Guid128([n; 16]) +} + +struct Propose(Vec); +impl Rule for Propose { + fn id(&self) -> RuleId { + RuleId { + name: "Propose", + version: 1, + } + } + fn propose(&self, _: &View<'_>, _: &[EvidenceRef]) -> Vec { + self.0.clone() + } +} + +#[test] +fn a_text_literal_is_resolved_once_and_execution_is_numeric() { + let mut st = VersionStore::new(); + let obs = Observation { + nodes: vec![ + (g(1), ObservedNode::user("a@x.de", "alice@x.de")), + (g(2), ObservedNode::user("b@x.de", "bob@x.de")), + (g(3), ObservedNode::user("c@x.de", "ALICE@X.DE")), // same key + (g(4), ObservedNode::user("d@x.de", "carol@x.de")), + ], + ..Observation::default() + }; + let g0 = st.observe("lab", 0, obs).unwrap(); + // A version that moves one owner off the key, moves another onto it, + // and creates a third: the query must see base, override and created. + let carol = st.lookup("carol@x.de"); + let alice2 = st.intern("Alice@X.de"); + let newcomer = st.intern("n@x.de"); + let g1 = st + .simulate( + g0, + &Propose(vec![ + Change::SetAttribute { + node: g(3), + attribute: Attribute::PrimarySmtp, + from: st.view(g0).unwrap().attr(&g(3), Attribute::PrimarySmtp), + to: Some(newcomer), + }, + Change::SetAttribute { + node: g(4), + attribute: Attribute::PrimarySmtp, + from: carol, + to: Some(alice2), + }, + Change::CreateNode { + node: g(5), + state: NodeState { + kind: NodeKind::User, + active: true, + upn: None, + primary_smtp: Some(alice2), + dn: None, + }, + }, + ]), + &[], + ) + .unwrap(); + let view = st.view(g1).unwrap(); + let counters = &view.dicts().counters; + + // 1. Boundary: one text operation turns the literal into a number. + let before = counters.snapshot(); + let key = view.dicts().key_lookup("Alice@X.de").expect("key observed"); + assert_eq!(counters.snapshot()[1], before[1] + 1, "exactly one lookup"); + + // 2. The executable query holds only that number. + let p = key_eq_program(key); + let eqs: Vec = p + .ops + .iter() + .filter_map(|op| match op { + MaskOp::Pred { + pred: Pred::EqU32 { v, .. }, + .. + } => Some(*v), + _ => None, + }) + .collect(); + assert_eq!(eqs, vec![key.0]); + + // 3. Execution: no intern, lookup or resolution. + let at = counters.snapshot(); + let rows = users_with_key(&view, Attribute::PrimarySmtp, key).rows(); + assert_eq!(counters.snapshot(), at, "execution did no text work"); + let owners: Vec = rows + .into_iter() + .map(|o| view.user_guid(UserOrdinal(o as u16)).unwrap()) + .collect(); + // g(1) observed; g(3) moved off; g(4) moved on; g(5) created. + assert_eq!(owners, vec![g(1), g(4), g(5)]); + + // An unknown literal is refused at the boundary, before any program. + assert_eq!(view.dicts().key_lookup("nobody@x.de"), None); +} diff --git a/crates/lance-graph-quack/src/lib.rs b/crates/lance-graph-quack/src/lib.rs index 29f043904..b20457ece 100644 --- a/crates/lance-graph-quack/src/lib.rs +++ b/crates/lance-graph-quack/src/lib.rs @@ -263,6 +263,16 @@ pub enum Cmp { /// Which bits of each byte participate; zero means "don't care". care: [u8; 12], }, + /// `((field_i[k] ^ pattern[k]) & care[k]) == 0` for every `k < 16` over a + /// 16-byte strided field view (`Pred::MatchFacet16Strided`): the same + /// ternary match over all 16 bytes of the field. The `Col` must name a + /// `LaneRef::Strided` lane whose field is 16 bytes wide. + MatchFacet16Strided { + /// The byte values to compare. + pattern: [u8; 16], + /// Which bits of each byte participate; zero means "don't care". + care: [u8; 16], + }, /// `lo <= row < hi` — a predicate on the ROW ORDINAL, reading no lane /// (`Pred::Range`, `mask_set_range`). The `Col` it is attached to is the /// ORDERED lane the range was bound on, kept for provenance so the leaf @@ -2250,6 +2260,11 @@ fn pred_of(col: Col, cmp: Cmp) -> Pred { pattern, care, }, + Cmp::MatchFacet16Strided { pattern, care } => Pred::MatchFacet16Strided { + lane, + pattern, + care, + }, // Reads no lane: `lane` is provenance only (see `Cmp::Range`). Cmp::Range { lo, hi } => Pred::Range { lo, hi }, } @@ -2570,7 +2585,10 @@ mod tests { (self.u64_at(*col, row) ^ pattern) & care == 0 } Cmp::Range { lo, hi } => (lo as usize) <= row && row < (hi as usize), - Cmp::EqU32Strided(_) | Cmp::NeU32Strided(_) | Cmp::MatchFacetStrided { .. } => { + Cmp::EqU32Strided(_) + | Cmp::NeU32Strided(_) + | Cmp::MatchFacetStrided { .. } + | Cmp::MatchFacet16Strided { .. } => { panic!("this fixture has no strided lane (see strided_leaf_tests)") } }, @@ -4757,3 +4775,79 @@ mod strided_leaf_tests { assert_eq!(fx.rows_of(&f), fx.oracle(|k| a.matches(k))); } } + +/// `Cmp::MatchFacet16Strided` lowers to `Pred::MatchFacet16Strided` and runs +/// over a 16-byte strided view, bytes `12..16` included. +#[cfg(test)] +mod facet16_tests { + use super::*; + use lance_graph_mask_risc::{ + execute_into, materialize_rows, words_for, Foreign, LaneRef, Out, Planes, Scratch, + StridedRef, + }; + + #[test] + fn match_facet16_lowers_and_reads_all_sixteen_bytes() { + let n = 300usize; + // Record i, byte k: a multiplicative hash mod 5, so bytes vary + // independently of one another. + let mut bytes = vec![0u8; n * 16]; + for i in 0..n { + for k in 0..16 { + // splitmix64 finalizer: no linear relation between bytes. + let mut z = ((i * 16 + k) as u64).wrapping_add(0x9E37_79B9_7F4A_7C15); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + bytes[i * 16 + k] = ((z ^ (z >> 31)) % 5) as u8; + } + } + let mut pattern = [0u8; 16]; + let mut care = [0u8; 16]; + // Constrain one leading byte and one byte past 12, the part a + // 12-byte match would never read. + pattern[0] = 3; + care[0] = 0xFF; + pattern[14] = 2; + care[14] = 0xFF; + let p = lower(&Query { + filter: Filter::cmp(Col(0), Cmp::MatchFacet16Strided { pattern, care }), + agg: Agg::Rows, + }) + .unwrap(); + assert!(p.ops.iter().any(|op| matches!( + op, + MaskOp::Pred { + pred: Pred::MatchFacet16Strided { .. }, + .. + } + ))); + let lanes = [LaneRef::Strided(StridedRef { + bytes: &bytes, + first_offset: 0, + stride: 16, + records: n, + })]; + let planes = Planes { + n_rows: n, + masks: &[], + lanes: &lanes, + }; + let mut scratch = Scratch::for_program(&p, n).unwrap(); + let mut out = vec![0u64; words_for(n)]; + execute_into( + &p, + &planes, + &Foreign::NONE, + &mut scratch, + Out::Mask(&mut out), + ) + .unwrap(); + let got = materialize_rows(&out, n); + let want: Vec = (0..n) + .filter(|&i| bytes[i * 16] == 3 && bytes[i * 16 + 14] == 2) + .collect(); + let only_first: usize = (0..n).filter(|&i| bytes[i * 16] == 3).count(); + assert!(!want.is_empty() && want.len() < only_first, "anti-vacuity"); + assert_eq!(got, want); + } +}