From 981e97a03754f30475f8d8db8a390e1f5ff9c7c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 13:43:51 +0000 Subject: [PATCH 1/5] quack: strided leaves and an in-place aperture sweep Cmp::{EqU32Strided, NeU32Strided, MatchFacetStrided} lower one-to-one to the mask-risc strided predicates, so a facet stored inside a wider record (a NodeRow) is queried where it sits. Filter::aperture_facet_strided sweeps an aperture over two views of the same bytes (classid at +0, tier bytes at +4) instead of the extracted semantic planes; the bound path is shared with aperture_facet. An aperture caring about part of the classid has no strided spelling and is refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- crates/lance-graph-contract/src/facet.rs | 12 + crates/lance-graph-quack/src/lib.rs | 368 +++++++++++++++++++++++ 2 files changed, 380 insertions(+) diff --git a/crates/lance-graph-contract/src/facet.rs b/crates/lance-graph-contract/src/facet.rs index a6f3a6963..09bada9bc 100644 --- a/crates/lance-graph-contract/src/facet.rs +++ b/crates/lance-graph-contract/src/facet.rs @@ -676,6 +676,18 @@ impl SemanticAperture { Self::new(p.lo_key(), FacetCascade::from_semantic_tiles(care)) } + /// The pattern, with every bit outside the care cleared. + #[must_use] + pub const fn pattern(self) -> FacetCascade { + self.pattern + } + + /// The care mask: set bits are the ones the aperture consults. + #[must_use] + pub const fn care(self) -> FacetCascade { + self.care + } + /// The lens the aperture is stated under (same as [`SemanticPrefix`]). #[must_use] pub const fn lens(self) -> SemanticLens { diff --git a/crates/lance-graph-quack/src/lib.rs b/crates/lance-graph-quack/src/lib.rs index 1d5dc8dd3..4498128ef 100644 --- a/crates/lance-graph-quack/src/lib.rs +++ b/crates/lance-graph-quack/src/lib.rs @@ -246,6 +246,23 @@ pub enum Cmp { /// Which bits participate; zero means "don't care". care: u64, }, + /// `u32_le(field_i) == v` over a strided field view — the classid of a + /// `NodeRow` read where it sits, no extracted column + /// (`Pred::EqU32Strided`). The `Col` must name a `LaneRef::Strided` lane. + EqU32Strided(u32), + /// `u32_le(field_i) != v` over a strided field view (`Pred::NeU32Strided`). + NeU32Strided(u32), + /// `((field_i[k] ^ pattern[k]) & care[k]) == 0` for every `k < 12` over a + /// strided view of the 12 facet tier bytes, in their stored order + /// (`[t0.lo, t0.hi, …, t5.lo, t5.hi]`), in place + /// (`Pred::MatchFacetStrided`). The `Col` must name a `LaneRef::Strided` + /// lane. + MatchFacetStrided { + /// The byte values to compare. + pattern: [u8; 12], + /// Which bits of each byte participate; zero means "don't care". + care: [u8; 12], + }, /// `lo <= row < hi` — a predicate on the ROW ORDINAL, reading no lane /// (`Pred::Range`, `mask_set_range`). The `Col` it is attached to is the /// ORDERED lane the range was bound on, kept for provenance so the leaf @@ -771,6 +788,68 @@ impl Filter { (true, true) => Filter::And(vec![leg(hi_col, p_hi, c_hi), leg(lo_col, p_lo, c_lo)]), } }; + Self::aperture_with(witnessed, lane_col, aperture, sweep) + } + + /// [`Filter::aperture_facet`] with the sweep read IN PLACE from the stored + /// facet bytes, so a caller that holds only `NodeRow` bytes (or any record + /// carrying a facet at a fixed offset) never extracts the semantic planes. + /// + /// `classid_col` must name a `LaneRef::Strided` view at the facet's first + /// byte (the little-endian `facet_classid`), and `tiers_col` one four bytes + /// later (the 12 tier bytes, `[t0.lo, t0.hi, …]`). Two views over the same + /// buffer. The sweep is `EqU32Strided` on the classid plus + /// `MatchFacetStrided` on the tiers, each included only when the aperture + /// cares about that part. + /// + /// Returns `None` when the aperture cares about only PART of the classid: + /// the strided classid reader is an equality, and there is no strided u32 + /// ternary match to express the rest. Such an aperture lowers through + /// [`Filter::aperture_facet`] instead. The bound path and its report are + /// the same as there. + #[must_use] + pub fn aperture_facet_strided( + witnessed: Option<(&SealedFacetLane, &OrderedLaneWitness)>, + lane_col: Col, + classid_col: Col, + tiers_col: Col, + aperture: &SemanticAperture, + ) -> Option<(Self, ApertureLowering)> { + let p = aperture.pattern().to_bytes(); + let c = aperture.care().to_bytes(); + let class_care = u32::from_le_bytes([c[0], c[1], c[2], c[3]]); + let class_leg = match class_care { + 0 => None, + u32::MAX => Some(Filter::Cmp( + classid_col, + Cmp::EqU32Strided(u32::from_le_bytes([p[0], p[1], p[2], p[3]])), + )), + _ => return None, + }; + let mut pattern = [0u8; 12]; + let mut care = [0u8; 12]; + pattern.copy_from_slice(&p[4..16]); + care.copy_from_slice(&c[4..16]); + let tier_leg = Filter::Cmp(tiers_col, Cmp::MatchFacetStrided { pattern, care }); + let sweep = || match (&class_leg, care.iter().any(|&b| b != 0)) { + (Some(cl), true) => Filter::And(vec![cl.clone(), tier_leg.clone()]), + (Some(cl), false) => cl.clone(), + // An empty care matches every row: the tier match with no cared + // byte is that predicate. + (None, _) => tier_leg.clone(), + }; + Some(Self::aperture_with(witnessed, lane_col, aperture, sweep)) + } + + /// The bound-or-sweep decision shared by the aperture lowerings: a prefix + /// aperture under a validated witness becomes one `Cmp::Range`; anything + /// else takes `sweep()`, and the report says why. + fn aperture_with( + witnessed: Option<(&SealedFacetLane, &OrderedLaneWitness)>, + lane_col: Col, + aperture: &SemanticAperture, + sweep: impl Fn() -> Filter, + ) -> (Self, ApertureLowering) { if aperture.prefix_bits().is_none() { return (sweep(), ApertureLowering::SweepNotAPrefix); } @@ -2164,6 +2243,13 @@ fn pred_of(col: Col, cmp: Cmp) -> Pred { pattern, care, }, + Cmp::EqU32Strided(v) => Pred::EqU32Strided { lane, v }, + Cmp::NeU32Strided(v) => Pred::NeU32Strided { lane, v }, + Cmp::MatchFacetStrided { pattern, care } => Pred::MatchFacetStrided { + lane, + pattern, + care, + }, // Reads no lane: `lane` is provenance only (see `Cmp::Range`). Cmp::Range { lo, hi } => Pred::Range { lo, hi }, } @@ -2484,6 +2570,9 @@ mod tests { (self.u64_at(*col, row) ^ pattern) & care == 0 } Cmp::Range { lo, hi } => (lo as usize) <= row && row < (hi as usize), + Cmp::EqU32Strided(_) | Cmp::NeU32Strided(_) | Cmp::MatchFacetStrided { .. } => { + panic!("this fixture has no strided lane (see strided_leaf_tests)") + } }, Filter::Plane(m) => self.bit(*m, row), Filter::EqU32Via { fk, key, v } => { @@ -4389,3 +4478,282 @@ mod diamond_lowering_tests { } } } + +/// The strided leaves (`EqU32Strided`, `NeU32Strided`, `MatchFacetStrided`) +/// and the in-place aperture sweep, over facets stored inside wider records. +#[cfg(test)] +mod strided_leaf_tests { + use super::*; + use lance_graph_contract::facet::FacetCascade; + use lance_graph_mask_risc::{ + execute_into, materialize_rows, words_for, Foreign, LaneRef, Out, Planes, Scratch, + StridedRef, Value, + }; + + const RANGE_COL: Col = Col(0); // provenance only + const HI: Col = Col(0); + const LO: Col = Col(1); + const CLASSID: Col = Col(2); + const TIERS: Col = Col(3); + const AT: usize = 8; // the facet's offset inside each record + + struct Rng(u64); + impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + } + + /// Small alphabets at the coarse tiles, so apertures at every depth cut + /// the population into non-trivial parts, and several classids. + fn skewed(n: usize, seed: u64) -> Vec { + let mut r = Rng(seed); + (0..n) + .map(|_| { + let t = [ + (r.next() % 3) as u16, + (r.next() % 2) as u16, + (r.next() % 4) as u16, + (r.next() % 3) as u16, + (r.next() % 300) as u16, + (r.next() % 3) as u16, + (r.next() % 7) as u16, + (r.next() % 2) as u16, + ]; + FacetCascade::from_semantic_tiles(t) + }) + .collect() + } + + /// Records of `stride` bytes, each carrying its facet at `AT` and junk + /// everywhere else, plus the two semantic planes, all in sealed order. + struct Fx { + lane: SealedFacetLane, + records: Vec, + stride: usize, + hi: Vec, + lo: Vec, + alpha: Vec, + } + + impl Fx { + fn new(n: usize, seed: u64, stride: usize) -> Self { + let lane = SealedFacetLane::seal(skewed(n, seed), 1).expect("seals"); + let mut r = Rng(seed ^ 0xA5A5); + let mut records: Vec = (0..n * stride).map(|_| r.next() as u8).collect(); + for (i, k) in lane.keys().iter().enumerate() { + records[i * stride + AT..i * stride + AT + 16].copy_from_slice(&k.to_bytes()); + } + let (hi, lo) = lane.keys().iter().map(|k| k.semantic_u64_halves()).unzip(); + let mut alpha = vec![0u64; words_for(n)]; + for i in 0..n { + alpha[i / 64] |= 1u64 << (i % 64); + } + Fx { + lane, + records, + stride, + hi, + lo, + alpha, + } + } + + fn n(&self) -> usize { + self.lane.keys().len() + } + + fn rows_of(&self, f: &Filter) -> Vec { + let q = Query { + filter: Filter::And(vec![Filter::Plane(Mask(0)), f.clone()]), + agg: Agg::Rows, + }; + let program = lower(&q).expect("lowers"); + let view = |off| { + LaneRef::Strided(StridedRef { + bytes: &self.records, + first_offset: off, + stride: self.stride, + records: self.n(), + }) + }; + let lanes = [ + LaneRef::U64(&self.hi), + LaneRef::U64(&self.lo), + view(AT), + view(AT + 4), + ]; + let masks: Vec<&[u64]> = vec![&self.alpha]; + let planes = Planes { + n_rows: self.n(), + masks: &masks, + lanes: &lanes, + }; + let mut scratch = Scratch::for_program(&program, self.n()).expect("carves"); + let mut mask = vec![0u64; words_for(self.n())]; + match execute_into( + &program, + &planes, + &Foreign::NONE, + &mut scratch, + Out::Mask(&mut mask), + ) + .expect("runs") + { + Value::Mask(_) => {} + other => panic!("not a mask: {other:?}"), + } + materialize_rows(&mask, self.n()) + } + + fn oracle(&self, keep: impl Fn(FacetCascade) -> bool) -> Vec { + (0..self.n()) + .filter(|&i| keep(self.lane.keys()[i])) + .collect() + } + } + + /// Random care over the tiers (whole bytes, empty bytes, and bytes with + /// holes) and a classid care that is either empty or full. + fn random_aperture(fx: &Fx, r: &mut Rng) -> SemanticAperture { + let key = fx.lane.keys()[(r.next() as usize) % fx.n()]; + let mut care = [0u8; 16]; + if r.next() % 2 == 0 { + care[..4].fill(0xFF); + } + for b in care[4..].iter_mut() { + *b = match r.next() % 4 { + 0 => 0, + 1 => 0xFF, + _ => r.next() as u8, + }; + } + SemanticAperture::new(key, FacetCascade::from_bytes(&care)) + } + + /// The three leaves select exactly the rows a plain reading of the stored + /// bytes selects, at a 40-byte and at the real 512-byte stride. + #[test] + fn strided_leaves_agree_with_a_plain_reading() { + for stride in [40usize, 512] { + let fx = Fx::new(1500, 41, stride); + let mut r = Rng(42); + let mut nontrivial = 0; + for _ in 0..60 { + let key = fx.lane.keys()[(r.next() as usize) % fx.n()]; + let class = key.facet_classid; + assert_eq!( + fx.rows_of(&Filter::Cmp(CLASSID, Cmp::EqU32Strided(class))), + fx.oracle(|k| k.facet_classid == class), + "eq, stride {stride}" + ); + assert_eq!( + fx.rows_of(&Filter::Cmp(CLASSID, Cmp::NeU32Strided(class))), + fx.oracle(|k| k.facet_classid != class), + "ne, stride {stride}" + ); + let mut care = [0u8; 12]; + for b in &mut care { + *b = r.next() as u8 & r.next() as u8; + } + let kb = key.to_bytes(); + let mut pattern = [0u8; 12]; + pattern.copy_from_slice(&kb[4..16]); + let truth = fx.oracle(|k| { + let b = k.to_bytes(); + (0..12).all(|i| (b[4 + i] ^ pattern[i]) & care[i] == 0) + }); + assert_eq!( + fx.rows_of(&Filter::Cmp( + TIERS, + Cmp::MatchFacetStrided { pattern, care } + )), + truth, + "match, stride {stride}" + ); + if !truth.is_empty() && truth.len() < fx.n() { + nontrivial += 1; + } + } + assert!(nontrivial > 20, "anti-vacuity: {nontrivial}"); + } + } + + /// The in-place aperture sweep selects the same rows as the semantic-plane + /// sweep and as the aperture itself, for apertures with and without holes. + #[test] + fn the_in_place_aperture_sweep_equals_the_plane_sweep() { + for stride in [40usize, 512] { + let fx = Fx::new(1500, 43, stride); + let mut r = Rng(44); + let (mut holes, mut nontrivial) = (0, 0); + for _ in 0..80 { + let a = random_aperture(&fx, &mut r); + let (strided, how_s) = + Filter::aperture_facet_strided(None, RANGE_COL, CLASSID, TIERS, &a) + .expect("classid care is empty or full"); + let (planes, how_p) = Filter::aperture_facet(None, RANGE_COL, HI, LO, &a); + assert_eq!(how_s, how_p); + assert!(!matches!(strided, Filter::Cmp(_, Cmp::Range { .. }))); + let truth = fx.oracle(|k| a.matches(k)); + assert_eq!(fx.rows_of(&strided), truth, "strided, stride {stride}"); + assert_eq!(fx.rows_of(&planes), truth, "planes, stride {stride}"); + holes += usize::from(how_s == ApertureLowering::SweepNotAPrefix); + nontrivial += usize::from(!truth.is_empty() && truth.len() < fx.n()); + } + assert!(holes > 40 && nontrivial > 20, "{holes} {nontrivial}"); + } + } + + /// Under a witness, a prefix aperture still becomes the same `Range` + /// whichever sweep the caller would otherwise have taken. + #[test] + fn a_witnessed_prefix_is_a_range_on_either_sweep() { + let fx = Fx::new(1500, 45, 512); + let w = fx.lane.witness(); + let key = fx.lane.keys()[700]; + for bits in [32u32, 36, 40, 48, 64, 100] { + let care: u128 = u128::MAX << (128 - bits); + let (h, l) = ((care >> 64) as u64, care as u64); + let care = FacetCascade::from_semantic_tiles([ + (h >> 48) as u16, + (h >> 32) as u16, + (h >> 16) as u16, + h as u16, + (l >> 48) as u16, + (l >> 32) as u16, + (l >> 16) as u16, + l as u16, + ]); + let a = SemanticAperture::new(key, care); + let strided = + Filter::aperture_facet_strided(Some((&fx.lane, &w)), RANGE_COL, CLASSID, TIERS, &a) + .expect("classid care is full"); + let planes = Filter::aperture_facet(Some((&fx.lane, &w)), RANGE_COL, HI, LO, &a); + assert_eq!(strided, planes, "bits {bits}"); + assert!(matches!(strided.0, Filter::Cmp(_, Cmp::Range { .. }))); + assert_eq!(fx.rows_of(&strided.0), fx.oracle(|k| a.matches(k))); + } + } + + /// Caring about part of the classid has no strided spelling: refused, and + /// the plane lowering still answers it. + #[test] + fn a_partial_classid_care_is_refused_in_place() { + let fx = Fx::new(800, 46, 40); + let key = fx.lane.keys()[100]; + // canon half cared, app half free: 16 of the classid's 32 bits. + let care = FacetCascade::from_semantic_tiles([0xFFFF, 0, 0, 0, 0, 0, 0, 0]); + let a = SemanticAperture::new(key, care); + assert_eq!( + Filter::aperture_facet_strided(None, RANGE_COL, CLASSID, TIERS, &a), + None + ); + let (f, _) = Filter::aperture_facet(None, RANGE_COL, HI, LO, &a); + assert_eq!(fx.rows_of(&f), fx.oracle(|k| a.matches(k))); + } +} From 5bb0340fe8b9091c97f872f933b6239c3d5d2dc1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 13:45:29 +0000 Subject: [PATCH 2/5] board: the in-place aperture sweep landed; clippy nit in the new test Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- ...6-09-25-aperture-prefix-lowers-to-range.md | 22 ++++++++++++++++++- crates/lance-graph-quack/src/lib.rs | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.claude/board/entries/2026-09-25-aperture-prefix-lowers-to-range.md b/.claude/board/entries/2026-09-25-aperture-prefix-lowers-to-range.md index d59301c36..4c1488182 100644 --- a/.claude/board/entries/2026-09-25-aperture-prefix-lowers-to-range.md +++ b/.claude/board/entries/2026-09-25-aperture-prefix-lowers-to-range.md @@ -16,6 +16,26 @@ - `ApertureLowering` reports which fold was used, and why. - A tile-aligned aperture lowers to exactly the same `Filter` as `prefix_facet`. +## The in-place sweep +- New quack leaves `Cmp::{EqU32Strided, NeU32Strided, MatchFacetStrided}` lower one-to-one to the mask-risc strided predicates. A facet stored inside a wider record (a `NodeRow`) is queried where it sits. +- `Filter::aperture_facet_strided` sweeps an aperture over two views of the same bytes: + - the classid at +0, compared with `EqU32Strided`; + - the 12 tier bytes at +4, compared with `MatchFacetStrided`. +- A caller holding only record bytes therefore never extracts the semantic planes. +- The bound decision is shared with `aperture_facet`: a witnessed prefix becomes the same `Range` on either sweep. +- Tests at a 40-byte and at the 512-byte stride, with junk around every facet: + - the three leaves match a plain reading of the stored bytes; + - the in-place sweep, the plane sweep and the aperture agree on random apertures, with and without holes; + - a partial classid care is refused. +- Disable runs, each red then green: + +| disable | tests that went red | +|---|---| +| leaf pattern/care swapped | 2 tests | +| tier bytes read at +0 | 1 test | +| classid leg dropped | 2 tests | +| partial classid accepted | 1 test | + ## Evidence - Every bit-prefix aperture from 0 to 128 bits bounds to exactly its matches, over 3000 sealed keys at four probe keys. Prefixes that end inside a tile are included; an anti-vacuity count requires such sub-tile cuts to actually split the population. - At the lowering, the Range, the sweep and the row oracle agree for bit prefixes. A hole aperture never becomes a Range, even under a valid witness. A stale witness sweeps and reports `VersionMismatch`. @@ -30,5 +50,5 @@ ## OPEN - The `Range` is ordinals in the sealed lane's order. The precondition on `prefix_facet` (`ISS-WITNESSED-RANGE-DOES-NOT-ATTEST-PLANE-ORDER`) applies unchanged. -- The sweep reads the two semantic `u64` planes. The strided in-place form (`Pred::MatchFacetStrided` over the 12-byte payload) is not yet a quack leaf, so a caller that has only the `NodeRow` bytes cannot sweep an aperture without extracting the planes. +- An aperture that cares about only PART of the classid has no in-place spelling: the strided classid reader is an equality, and no strided u32 ternary match exists. `aperture_facet_strided` refuses it (`None`), and it lowers through the plane sweep. - No caller mints apertures yet. The HHTL cascade and the traversal frontier are the intended producers. diff --git a/crates/lance-graph-quack/src/lib.rs b/crates/lance-graph-quack/src/lib.rs index 4498128ef..29f043904 100644 --- a/crates/lance-graph-quack/src/lib.rs +++ b/crates/lance-graph-quack/src/lib.rs @@ -4622,7 +4622,7 @@ mod strided_leaf_tests { fn random_aperture(fx: &Fx, r: &mut Rng) -> SemanticAperture { let key = fx.lane.keys()[(r.next() as usize) % fx.n()]; let mut care = [0u8; 16]; - if r.next() % 2 == 0 { + if r.next().is_multiple_of(2) { care[..4].fill(0xFF); } for b in care[4..].iter_mut() { From 4461afd36a5d437077f228ed26c6270dcddeb366 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 13:46:55 +0000 Subject: [PATCH 3/5] board: regenerate entries index on the new base Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- .claude/board/entries/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 59e082182..8fd1aea97 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -171 entries, 2026-08-06 .. 2026-09-25. +172 entries, 2026-08-06 .. 2026-09-25. | date | entry id | finding | file | |---|---|---|---| From 2ec6a4880380a3c2ee959d50bd9372f705b2a4db Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 14:04:23 +0000 Subject: [PATCH 4/5] mask-risc probe: HHTL-ordered vs random access for a partial mask Ordered (ascending address) access over the packed key lane stays at 3-4 ns per node from L1 to 64 MB; random climbs to 144-172 ns. Visiting 256-row subtrees in random order but rows in order costs almost nothing extra. Sparsity (1 member in 16 rows) and the 512-byte in-place stride defeat the prefetcher: ordering only halves those. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- .../entries/2026-09-25-hhtl-ordered-access.md | 41 ++++ .claude/board/entries/README.md | 3 +- .../examples/hhtl_order_probe.rs | 204 ++++++++++++++++++ 3 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-09-25-hhtl-ordered-access.md create mode 100644 crates/lance-graph-mask-risc/examples/hhtl_order_probe.rs diff --git a/.claude/board/entries/2026-09-25-hhtl-ordered-access.md b/.claude/board/entries/2026-09-25-hhtl-ordered-access.md new file mode 100644 index 000000000..265738a02 --- /dev/null +++ b/.claude/board/entries/2026-09-25-hhtl-ordered-access.md @@ -0,0 +1,41 @@ +# 2026-09-25 — HHTL-ordered access keeps a partial mask at ~3–4 ns per node at any working set + +**Status:** MEASURED (`crates/lance-graph-mask-risc/examples/hhtl_order_probe.rs`; 3 runs, median of 5) · OPEN (see below) +**D-ids:** none new. Answers the "HHTL-ordered access has not been measured" item of `2026-09-25-aperture-masks-and-kernel-gap.md`. + +## What was measured +The query is the one from `mask_cache_hit_probe`: one node's 6-byte tier path compared under a care mask. Only the visit order changes: +- **random:** a random permutation of the members; +- **ordered:** ascending address order, which is what a prefix walk over a sealed lane yields; +- **clustered:** 256-row blocks in random order, with rows in address order inside each block. + +Latency is a pointer chase. Each member's record stores the next row in the chosen order, so only the load sits on the chain. + +## Result (latency, ns per node) + +| members, 16-byte records | random | ordered | clustered | +|---|---|---|---| +| 32k rows, packed (512 KB) | 7.8–8.7 | 3.2–3.3 | 3.2–3.3 | +| 32k scattered over a 64k tile | 12.3–16.5 | 4.2–4.6 | 4.3–4.5 | +| one tile (64k, 1 MB) | 13.1–14.6 | 3.2–3.3 | 3.3–3.5 | +| four tiles (256k, 4 MB) | 28.5–30.5 (one run 84) | 3.2–3.4 | 3.4 | +| 64 tiles (4M, 64 MB) | 144–172 | 3.8–4.0 | 4.4–4.9 | +| 256k scattered over 4M (1 in 16) | 133–161 | 94–99 | 114–121 | + +| members, 512-byte `NodeRow` in place | random | ordered | clustered | +|---|---|---|---| +| 4k rows (2 MB span) | 26–28 | 14.0–14.6 | 14.1–16.7 | +| 32k rows (16 MB span) | 65–125 | 22–24 | 18–27 | +| 64k rows (32 MB span) | 125–152 | 57–71 | 54–70 | +| 4k scattered over 64k | 40–42 | 36–40 | 36–41 | + +## What it says +- **Ordered access over the packed key lane is flat, at ~3–4 ns per node, from L1 up to 64 MB.** The hardware prefetcher streams it. That is 37–43× below random at 64 MB, and 4× below random within one tile. The ~12 / ~25 ns regimes are the cost of random order, not of the working set. +- **Only order within a subtree matters.** Jumping between 256-row blocks in random order costs almost nothing extra, so a frontier that finishes one subtree before moving on gets the ordered cost. +- **Sparsity defeats the prefetcher.** At 1 member in 16 rows (members about 4 cache lines apart), the ordered walk is still ~95 ns over 64 MB. At 32k members scattered over one tile (one in two), ordered holds at ~4 ns. +- **The in-place 512-byte stride gains less.** Consecutive rows are 8 cache lines apart, so ordering roughly halves the cost (2 MB span: 27 → 14 ns; 32 MB span: ~140 → ~60 ns) rather than flattening it. This favours the packed key lane for traversal even more strongly than the random-access numbers did. +- **Consequence for the positive/negative selection argument:** capping members at 32k matters much less than visiting them in address order. A mask is address-ordered by construction, so a walk that follows its set bits in word order is already the ordered case. + +## OPEN +- The sparse case (1 in 16 and sparser) is where neither order helps. The alternatives (compacting survivors, software prefetch a few members ahead, or reading the bitplane instead of the records) are unmeasured. +- The measurement uses one core and no competing traffic. Prefetcher behaviour under multi-core load is untested. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 461e54459..39a0b0e44 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -173 entries, 2026-08-06 .. 2026-09-25. +174 entries, 2026-08-06 .. 2026-09-25. | date | entry id | finding | file | |---|---|---|---| @@ -38,6 +38,7 @@ exactly one of them, never both. | 2026-09-25 | `llvm-whole-stack-fold-ceiling` | | [2026-09-25-llvm-whole-stack-fold-ceiling.md](2026-09-25-llvm-whole-stack-fold-ceiling.md) | | 2026-09-25 | `lance12-lancedb039-sweep` | | [2026-09-25-lance12-lancedb039-sweep.md](2026-09-25-lance12-lancedb039-sweep.md) | | 2026-09-25 | `keep-fold` | | [2026-09-25-keep-fold.md](2026-09-25-keep-fold.md) | +| 2026-09-25 | `hhtl-ordered-access` | | [2026-09-25-hhtl-ordered-access.md](2026-09-25-hhtl-ordered-access.md) | | 2026-09-25 | `argon2-in-register-compress-per-tier` | | [2026-09-25-argon2-in-register-compress-per-tier.md](2026-09-25-argon2-in-register-compress-per-tier.md) | | 2026-09-25 | `aperture-prefix-lowers-to-range` | | [2026-09-25-aperture-prefix-lowers-to-range.md](2026-09-25-aperture-prefix-lowers-to-range.md) | | 2026-09-25 | `aperture-masks-and-kernel-gap` | | [2026-09-25-aperture-masks-and-kernel-gap.md](2026-09-25-aperture-masks-and-kernel-gap.md) | diff --git a/crates/lance-graph-mask-risc/examples/hhtl_order_probe.rs b/crates/lance-graph-mask-risc/examples/hhtl_order_probe.rs new file mode 100644 index 000000000..85a80faa3 --- /dev/null +++ b/crates/lance-graph-mask-risc/examples/hhtl_order_probe.rs @@ -0,0 +1,204 @@ +//! Does HHTL-ordered access beat random access for a vertical partial mask? +//! +//! Same query as `mask_cache_hit_probe`: one node's 6-byte tier path (half the +//! facet) compared under a care mask. What changes is the ORDER the member +//! nodes are visited in: +//! +//! - `random`: a uniformly random permutation of the members; +//! - `ordered`: ascending address order, which is what a prefix walk over a +//! sealed (address-ordered) lane produces; +//! - `clustered`: 256-row blocks (one HEEL.hi cell) visited in random order, +//! rows in address order inside each block — a frontier that finishes one +//! subtree before it jumps. +//! +//! Latency is a pointer chase: bytes 12..16 of each member's record hold the +//! next row in the chosen order, so nothing but the load is on the chain. +//! Throughput walks the same order from a list drawn before the clock starts. +//! +//! Measured 2026-09-25 (Xeon @ 2.8 GHz, median of 5, 3 runs), ns per node: +//! +//! | members (16-byte records) | random lat | ordered lat | clustered lat | +//! |---|---|---|---| +//! | 32k rows, packed (512 KB) | 7.8-8.7 | 3.2-3.3 | 3.2-3.3 | +//! | 32k scattered over a 64k tile | 12.3-16.5 | 4.2-4.6 | 4.3-4.5 | +//! | one tile (64k, 1 MB) | 13.1-14.6 | 3.2-3.3 | 3.3-3.5 | +//! | four tiles (256k, 4 MB) | 28.5-30.5 (one run 84) | 3.2-3.4 | 3.4 | +//! | 64 tiles (4M, 64 MB) | 144-172 | 3.8-4.0 | 4.4-4.9 | +//! | 256k scattered over 4M (1/16) | 133-161 | 94-99 | 114-121 | +//! +//! | members (512-byte `NodeRow`, in place) | random lat | ordered lat | clustered lat | +//! |---|---|---|---| +//! | 4k rows (2 MB span) | 26-28 | 14.0-14.6 | 14.1-16.7 | +//! | 32k rows (16 MB span) | 65-125 | 22-24 | 18-27 | +//! | 64k rows (32 MB span) | 125-152 | 57-71 | 54-70 | +//! | 4k scattered over 64k | 40-42 | 36-40 | 36-41 | +//! +//! Ordered access over the packed key lane stays at ~3-4 ns per node at every +//! working-set size up to 64 MB: the hardware prefetcher streams it. Walking +//! 256-row subtrees in random order costs almost nothing extra. What defeats +//! the prefetcher is sparsity: at 1 member per 16 rows (4 cache lines apart) +//! the ordered walk is still ~95 ns, and on the 512-byte stride a row is 8 +//! lines from the next, so ordered access only roughly halves the cost. +//! +//! `RUSTFLAGS="-C target-cpu=native" cargo run --release -p lance-graph-mask-risc --example hhtl_order_probe` + +use std::hint::black_box; +use std::time::Instant; + +const Q: usize = 1 << 20; + +fn mix(x: u64) -> u64 { + let x = (x ^ (x >> 33)).wrapping_mul(0xff51_afd7_ed55_8ccd); + (x ^ (x >> 29)).wrapping_mul(0xc4ce_b9fe_1a85_ec53) +} + +fn shuffle(v: &mut [T], seed: u64) { + let mut x = seed; + for i in (1..v.len()).rev() { + x = mix(x); + v.swap(i, (x as usize) % (i + 1)); + } +} + +#[derive(Clone, Copy, PartialEq)] +enum Order { + Random, + Ordered, + Clustered, +} + +/// The visit order over sorted `members`. +fn visit_order(members: &[u32], order: Order, seed: u64) -> Vec { + let mut v = members.to_vec(); + match order { + Order::Random => shuffle(&mut v, seed), + Order::Ordered => {} + Order::Clustered => { + let mut blocks: Vec<&[u32]> = v.chunk_by(|a, b| a >> 8 == b >> 8).collect(); + shuffle(&mut blocks, seed); + v = blocks.concat(); + } + } + v +} + +/// Link `order` into one cycle through bytes 12..16 of each record. +fn link(store: &mut [u8], rec: usize, order: &[u32]) { + for k in 0..order.len() { + let from = order[k] as usize * rec; + let to = order[(k + 1) % order.len()]; + store[from + 12..from + 16].copy_from_slice(&to.to_le_bytes()); + } +} + +#[inline(always)] +fn half_hit(store: &[u8], o: usize) -> u64 { + const CARE: u64 = 0x0000_FFFF_FFFF; + const PAT: u64 = 0x0000_1234_5678; + let mut b = [0u8; 8]; + b[..6].copy_from_slice(&store[o + 4..o + 10]); + u64::from((u64::from_le_bytes(b) ^ PAT) & CARE == 0) +} + +fn latency(store: &[u8], rec: usize, start: u32) -> f64 { + let (mut r, mut acc) = (start as usize, 0u64); + let t = Instant::now(); + for _ in 0..Q { + let o = r * rec; + acc = acc.wrapping_add(half_hit(store, o)); + r = u32::from_le_bytes(store[o + 12..o + 16].try_into().unwrap()) as usize; + } + black_box(acc); + t.elapsed().as_nanos() as f64 / Q as f64 +} + +fn throughput(store: &[u8], rec: usize, queries: &[u32]) -> f64 { + let mut acc = 0u64; + let t = Instant::now(); + for &r in queries { + acc = acc.wrapping_add(half_hit(store, r as usize * rec)); + } + black_box(acc); + t.elapsed().as_nanos() as f64 / queries.len() as f64 +} + +fn median(mut v: Vec) -> f64 { + v.sort_by(|a, b| a.total_cmp(b)); + v[v.len() / 2] +} + +/// `n` distinct rows of `0..domain`, sorted. +fn members(n: usize, domain: usize, seed: u64) -> Vec { + if n == domain { + return (0..domain as u32).collect(); + } + let mut all: Vec = (0..domain as u32).collect(); + shuffle(&mut all, seed); + all.truncate(n); + all.sort_unstable(); + all +} + +fn run(store: &mut [u8], rec: usize, label: &str, m: &[u32]) { + let mut row = format!("{label:>40}"); + for order in [Order::Random, Order::Ordered, Order::Clustered] { + let o = visit_order(m, order, 0x5A77); + link(store, rec, &o); + // Q queries walking the order, wrapping around the cycle. + let qs: Vec = (0..Q).map(|i| o[i % o.len()]).collect(); + let l = median((0..5).map(|_| latency(store, rec, o[0])).collect()); + let t = median((0..5).map(|_| throughput(store, rec, &qs)).collect()); + row += &format!(" {:>7.2} {:>6.2}", l, t); + } + println!("{row}"); +} + +fn header(title: &str) { + println!("\n{title}"); + println!( + "{:>40} {:>14} {:>14} {:>14}", + "", "random", "ordered", "clustered" + ); + println!( + "{:>40} {:>7} {:>6} {:>7} {:>6} {:>7} {:>6}", + "members", "lat", "thru", "lat", "thru", "lat", "thru" + ); +} + +fn main() { + println!("ns per node, median of 5; lat = pointer chase, thru = independent"); + let max_rows = 1usize << 22; + let mut store: Vec = (0..max_rows * 16).map(|i| mix(i as u64) as u8).collect(); + header("16-byte records (packed key lane)"); + for (label, n, domain) in [ + ("2k rows (L1d)", 1usize << 11, 1usize << 11), + ("32k rows, packed (512 KB)", 1 << 15, 1 << 15), + ("32k scattered over a 64k tile", 1 << 15, 1 << 16), + ("4k scattered over a 64k tile", 1 << 12, 1 << 16), + ("one tile (64k, 1 MB)", 1 << 16, 1 << 16), + ("four tiles (256k, 4 MB)", 1 << 18, 1 << 18), + ("64 tiles (4M, 64 MB)", 1 << 22, 1 << 22), + ("256k scattered over 4M (1/16)", 1 << 18, 1 << 22), + ] { + run( + &mut store, + 16, + label, + &members(n, domain, 0xC0DE ^ n as u64), + ); + } + drop(store); + + let rows = 1usize << 16; + let mut big: Vec = (0..rows * 512).map(|i| mix(i as u64) as u8).collect(); + header("512-byte NodeRow stride, read in place (one line per row)"); + for (label, n, domain) in [ + ("512 rows (256 KB span)", 1usize << 9, 1usize << 9), + ("4k rows (2 MB span)", 1 << 12, 1 << 12), + ("32k rows (16 MB span)", 1 << 15, 1 << 15), + ("64k rows (32 MB span)", 1 << 16, 1 << 16), + ("4k scattered over 64k", 1 << 12, 1 << 16), + ] { + run(&mut big, 512, label, &members(n, domain, 0xC0DE ^ n as u64)); + } +} From f738fef854f056dddd6199dc7761f5de57b4823f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 25 Sep 2026 14:11:11 +0000 Subject: [PATCH 5/5] hhtl_order_probe: walk every member of each cycle, not the first 2^20 CodeRabbit on #1292: the 4M-member cases were capped at 2^20 visits, so they measured only their first 16 MB. Every case now visits max(Q, members). Re-measured over 3 runs; the ordered 64 MB result is unchanged (3.8-3.9 ns), the sparse ordered case is 77-91 ns. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GXUahz73MZxtxWcfpHp9dG --- .../entries/2026-09-25-hhtl-ordered-access.md | 11 ++++++---- .../examples/hhtl_order_probe.rs | 22 +++++++++++-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.claude/board/entries/2026-09-25-hhtl-ordered-access.md b/.claude/board/entries/2026-09-25-hhtl-ordered-access.md index 265738a02..192db35ba 100644 --- a/.claude/board/entries/2026-09-25-hhtl-ordered-access.md +++ b/.claude/board/entries/2026-09-25-hhtl-ordered-access.md @@ -19,8 +19,8 @@ Latency is a pointer chase. Each member's record stores the next row in the chos | 32k scattered over a 64k tile | 12.3–16.5 | 4.2–4.6 | 4.3–4.5 | | one tile (64k, 1 MB) | 13.1–14.6 | 3.2–3.3 | 3.3–3.5 | | four tiles (256k, 4 MB) | 28.5–30.5 (one run 84) | 3.2–3.4 | 3.4 | -| 64 tiles (4M, 64 MB) | 144–172 | 3.8–4.0 | 4.4–4.9 | -| 256k scattered over 4M (1 in 16) | 133–161 | 94–99 | 114–121 | +| 64 tiles (4M, 64 MB), walked end to end | 157–163 | 3.8–3.9 | 4.4–4.6 | +| 256k scattered over 4M (1 in 16) | 138–151 | 77–91 | 99–105 | | members, 512-byte `NodeRow` in place | random | ordered | clustered | |---|---|---|---| @@ -30,12 +30,15 @@ Latency is a pointer chase. Each member's record stores the next row in the chos | 4k scattered over 64k | 40–42 | 36–40 | 36–41 | ## What it says -- **Ordered access over the packed key lane is flat, at ~3–4 ns per node, from L1 up to 64 MB.** The hardware prefetcher streams it. That is 37–43× below random at 64 MB, and 4× below random within one tile. The ~12 / ~25 ns regimes are the cost of random order, not of the working set. +- **Ordered access over the packed key lane is flat, at ~3–4 ns per node, from L1 up to 64 MB.** The hardware prefetcher streams it. That is ~40× below random at 64 MB, and 4× below random within one tile. The ~12 / ~25 ns regimes are the cost of random order, not of the working set. - **Only order within a subtree matters.** Jumping between 256-row blocks in random order costs almost nothing extra, so a frontier that finishes one subtree before moving on gets the ordered cost. -- **Sparsity defeats the prefetcher.** At 1 member in 16 rows (members about 4 cache lines apart), the ordered walk is still ~95 ns over 64 MB. At 32k members scattered over one tile (one in two), ordered holds at ~4 ns. +- **Sparsity defeats the prefetcher.** At 1 member in 16 rows (members about 4 cache lines apart), the ordered walk is still ~80–90 ns over 64 MB. At 32k members scattered over one tile (one in two), ordered holds at ~4 ns. - **The in-place 512-byte stride gains less.** Consecutive rows are 8 cache lines apart, so ordering roughly halves the cost (2 MB span: 27 → 14 ns; 32 MB span: ~140 → ~60 ns) rather than flattening it. This favours the packed key lane for traversal even more strongly than the random-access numbers did. - **Consequence for the positive/negative selection argument:** capping members at 32k matters much less than visiting them in address order. A mask is address-ordered by construction, so a walk that follows its set bits in word order is already the ordered case. +## Probe correction (review) +A first version capped every case at 2^20 visits, so the 4M-member cases walked only their first 16 MB, which could sit partly in L3. Every case now visits at least all of its members. The rows over 4M are the full-walk figures from 3 runs. The ordered 64 MB result did not change (3.8–3.9 ns). + ## OPEN - The sparse case (1 in 16 and sparser) is where neither order helps. The alternatives (compacting survivors, software prefetch a few members ahead, or reading the bitplane instead of the records) are unmeasured. - The measurement uses one core and no competing traffic. Prefetcher behaviour under multi-core load is untested. diff --git a/crates/lance-graph-mask-risc/examples/hhtl_order_probe.rs b/crates/lance-graph-mask-risc/examples/hhtl_order_probe.rs index 85a80faa3..54a73f644 100644 --- a/crates/lance-graph-mask-risc/examples/hhtl_order_probe.rs +++ b/crates/lance-graph-mask-risc/examples/hhtl_order_probe.rs @@ -11,6 +11,7 @@ //! rows in address order inside each block — a frontier that finishes one //! subtree before it jumps. //! +//! Every case visits max(Q, members) nodes, so the whole cycle is walked. //! Latency is a pointer chase: bytes 12..16 of each member's record hold the //! next row in the chosen order, so nothing but the load is on the chain. //! Throughput walks the same order from a list drawn before the clock starts. @@ -23,8 +24,8 @@ //! | 32k scattered over a 64k tile | 12.3-16.5 | 4.2-4.6 | 4.3-4.5 | //! | one tile (64k, 1 MB) | 13.1-14.6 | 3.2-3.3 | 3.3-3.5 | //! | four tiles (256k, 4 MB) | 28.5-30.5 (one run 84) | 3.2-3.4 | 3.4 | -//! | 64 tiles (4M, 64 MB) | 144-172 | 3.8-4.0 | 4.4-4.9 | -//! | 256k scattered over 4M (1/16) | 133-161 | 94-99 | 114-121 | +//! | 64 tiles (4M, 64 MB), walked end to end | 157-163 | 3.8-3.9 | 4.4-4.6 | +//! | 256k scattered over 4M (1/16) | 138-151 | 77-91 | 99-105 | //! //! | members (512-byte `NodeRow`, in place) | random lat | ordered lat | clustered lat | //! |---|---|---|---| @@ -37,7 +38,7 @@ //! working-set size up to 64 MB: the hardware prefetcher streams it. Walking //! 256-row subtrees in random order costs almost nothing extra. What defeats //! the prefetcher is sparsity: at 1 member per 16 rows (4 cache lines apart) -//! the ordered walk is still ~95 ns, and on the 512-byte stride a row is 8 +//! the ordered walk is still ~80-90 ns, and on the 512-byte stride a row is 8 //! lines from the next, so ordered access only roughly halves the cost. //! //! `RUSTFLAGS="-C target-cpu=native" cargo run --release -p lance-graph-mask-risc --example hhtl_order_probe` @@ -100,16 +101,16 @@ fn half_hit(store: &[u8], o: usize) -> u64 { u64::from((u64::from_le_bytes(b) ^ PAT) & CARE == 0) } -fn latency(store: &[u8], rec: usize, start: u32) -> f64 { +fn latency(store: &[u8], rec: usize, start: u32, visits: usize) -> f64 { let (mut r, mut acc) = (start as usize, 0u64); let t = Instant::now(); - for _ in 0..Q { + for _ in 0..visits { let o = r * rec; acc = acc.wrapping_add(half_hit(store, o)); r = u32::from_le_bytes(store[o + 12..o + 16].try_into().unwrap()) as usize; } black_box(acc); - t.elapsed().as_nanos() as f64 / Q as f64 + t.elapsed().as_nanos() as f64 / visits as f64 } fn throughput(store: &[u8], rec: usize, queries: &[u32]) -> f64 { @@ -144,9 +145,12 @@ fn run(store: &mut [u8], rec: usize, label: &str, m: &[u32]) { for order in [Order::Random, Order::Ordered, Order::Clustered] { let o = visit_order(m, order, 0x5A77); link(store, rec, &o); - // Q queries walking the order, wrapping around the cycle. - let qs: Vec = (0..Q).map(|i| o[i % o.len()]).collect(); - let l = median((0..5).map(|_| latency(store, rec, o[0])).collect()); + // At least Q visits, and never fewer than the whole cycle: a case with + // more members than Q must be walked end to end, or a big working set + // is measured by its first Q members only. + let visits = Q.max(o.len()); + let qs: Vec = (0..visits).map(|i| o[i % o.len()]).collect(); + let l = median((0..5).map(|_| latency(store, rec, o[0], visits)).collect()); let t = median((0..5).map(|_| throughput(store, rec, &qs)).collect()); row += &format!(" {:>7.2} {:>6.2}", l, t); }