From d5a120b472507e1b6c3d8025910e8a9630b01f3e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 23:18:10 +0000 Subject: [PATCH 1/4] D-PUZZLE-ATTN-0: entropy as the focus of attention on crosswords Move the #1387 crossword core (codebook, lexicon, grids, compile, token masks + fixed point, creation) unchanged into examples/shared/ crossword_core.rs; crossword_mask_propagation_probe includes it (its 20 tests pass unchanged). New crossword_attention_probe (test = true) varies only the search's slot choice: fixed, random, widest, popcount (log2 count), Shannon under the COCA frequency prior, dom/wdeg. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- crates/cognitive-shader-driver/Cargo.toml | 6 + .../examples/crossword_attention_probe.rs | 624 +++++++++++++++ .../crossword_mask_propagation_probe.rs | 713 +---------------- .../examples/shared/crossword_core.rs | 733 ++++++++++++++++++ 4 files changed, 1368 insertions(+), 708 deletions(-) create mode 100644 crates/cognitive-shader-driver/examples/crossword_attention_probe.rs create mode 100644 crates/cognitive-shader-driver/examples/shared/crossword_core.rs diff --git a/crates/cognitive-shader-driver/Cargo.toml b/crates/cognitive-shader-driver/Cargo.toml index f5894478a..a71965afb 100644 --- a/crates/cognitive-shader-driver/Cargo.toml +++ b/crates/cognitive-shader-driver/Cargo.toml @@ -171,6 +171,12 @@ test = true name = "crossword_real_words_probe" test = true +# D-PUZZLE-ATTN-0: entropy as the focus of attention on crosswords (search +# slot policies over the shared crossword core); tests run under `cargo test`. +[[example]] +name = "crossword_attention_probe" +test = true + [[example]] name = "crossword_mask_propagation_probe" test = true diff --git a/crates/cognitive-shader-driver/examples/crossword_attention_probe.rs b/crates/cognitive-shader-driver/examples/crossword_attention_probe.rs new file mode 100644 index 000000000..3050f459c --- /dev/null +++ b/crates/cognitive-shader-driver/examples/crossword_attention_probe.rs @@ -0,0 +1,624 @@ +//! **D-PUZZLE-ATTN-0 — entropy as the focus of attention, on crosswords.** +//! +//! #1387 showed that crossword solving is a fixed point: placing a word ANDs +//! its letters into every crossing slot, and every order of propagation +//! reaches the same state. Deduction needs no attention. Attention starts +//! where deduction runs out: when no slot is forced, a search has to choose +//! WHERE to look next. This probe changes only that choice and holds +//! everything else fixed: the same puzzles, the same propagation +//! (`shared/crossword_core.rs`, moved unchanged from #1387), the same value +//! order (most frequent word first). +//! +//! # The slot policies +//! +//! | policy | next slot | reads | +//! |---|---|---| +//! | `Fixed` | the first unplaced slot | nothing | +//! | `Random` | a seeded random unplaced slot | nothing | +//! | `Widest` | the most candidates | popcount (negative control) | +//! | `Popcount` | the fewest candidates: `log2(popcount)`, uniform entropy | popcount | +//! | `Shannon` | the lowest `H = -Σ p log2 p`, `p ∝` COCA frequency | the frequency prior | +//! | `DomWdeg` | the fewest candidates per accumulated failure | popcount + past contradictions | +//! +//! `Popcount` is #1387's `min_by_key(count)`: entropy under a uniform prior. +//! `Shannon` differs only when candidates carry unequal weight, which the +//! frequency-ranked vocabulary gives them. `DomWdeg` is Boussemart et al.'s +//! weighted-degree heuristic (2004) in its simplest form: every contradiction +//! adds one to the weight of the slot that ran out of candidates, and later +//! choices divide a slot's count by that weight. It is the one policy whose +//! choices depend on what earlier folds did. +//! +//! A slot with no candidate is a contradiction: it has no entropy and is never +//! read as settled. +//! +//! # Workloads +//! +//! - **prove**: created puzzles (unique by construction); count fills up to 2. +//! - **fill**: an empty grid; find one fill. +//! - **open**: created puzzles with only the first half of their givens; count +//! fills up to 50. +//! +//! Run: `cargo run --release -p cognitive-shader-driver --example crossword_attention_probe` +//! Tests (CI): `cargo test -p cognitive-shader-driver --example crossword_attention_probe` + +use std::time::Instant; + +use deepnsm_v2::vocab::WordId; + +#[path = "shared/population_fold.rs"] +mod population_fold; +use population_fold::Rng; + +#[path = "shared/crossword_core.rs"] +mod crossword_core; +use crossword_core::*; + +const ACADEMIC: &str = include_str!("../../deepnsm/word_frequency/academic_20k.csv"); + +/// The frequency prior: per `WordId`, its COCA-All count `z` and `z ln z`. +/// Surfaces that collapse when lowercased sum. +struct Prior { + z: Vec, + zlnz: Vec, +} + +impl Prior { + fn coca(cold: &Cold) -> Self { + let n = cold.vocab.len(); + let mut z = vec![0.0f64; n]; + for line in ACADEMIC.lines().skip(1) { + let c: Vec<&str> = line.split(',').collect(); + let (Some(w), Some(f)) = (c.get(3), c.get(5)) else { + continue; + }; + if let (Some(id), Ok(f)) = (cold.vocab.id(&w.to_lowercase()), f.parse::()) { + z[id as usize] += f; + } + } + // A word with no count still counts as a candidate. + for v in z.iter_mut() { + *v = v.max(1.0); + } + let zlnz = z.iter().map(|v| v * v.ln()).collect(); + Prior { z, zlnz } + } + + /// Shannon entropy in bits of the candidates in `mask` under this prior; + /// `None` for an empty mask (a contradiction, not certainty). + fn entropy(&self, mask: &[u64]) -> Option { + let (mut zs, mut s) = (0.0, 0.0); + let mut any = false; + for (b, &word) in mask.iter().enumerate() { + let mut w = word; + while w != 0 { + let i = b * 64 + w.trailing_zeros() as usize; + zs += self.z[i]; + s += self.zlnz[i]; + any = true; + w &= w - 1; + } + } + any.then(|| ((zs.ln() - s / zs) / std::f64::consts::LN_2).max(0.0)) + } +} + +/// Uniform entropy: `log2(popcount)`; `None` for an empty mask. +#[cfg(test)] +fn uniform_entropy(count: u64) -> Option { + (count > 0).then(|| (count as f64).log2()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Policy { + Fixed, + Random(u64), + Widest, + Popcount, + Shannon, + DomWdeg, +} + +const POLICIES: [Policy; 6] = [ + Policy::Fixed, + Policy::Random(0x5EED), + Policy::Widest, + Policy::Popcount, + Policy::Shannon, + Policy::DomWdeg, +]; + +/// What one search did. +#[derive(Debug, Clone, Default, PartialEq)] +struct Stats { + /// Words placed and propagated. + nodes: u64, + /// Placements whose propagation emptied a slot. + contradictions: u64, + /// Fills found (capped). + fills: u64, + /// The first fill found. + first: Option>, + /// Nodes spent before the first fill. + nodes_to_first: Option, + /// Ran out of budget. + exhausted: bool, + /// Per node, in order: did the placement contradict? (Bounded.) + fail_trace: Vec, + /// Decisions where Shannon and popcount would pick different slots. + disagreements: u64, + decisions: u64, +} + +struct Search<'a> { + hot: &'a Hot, + puz: &'a Puzzle, + prior: &'a Prior, + policy: Policy, + rng: Rng, + /// DomWdeg: per slot, contradictions it caused by running dry. + weight: Vec, + budget: u64, + cap: u64, + stats: Stats, +} + +const TRACE_CAP: usize = 1 << 20; + +impl<'a> Search<'a> { + fn new( + hot: &'a Hot, + puz: &'a Puzzle, + prior: &'a Prior, + policy: Policy, + budget: u64, + cap: u64, + ) -> Self { + let seed = match policy { + Policy::Random(s) => s, + _ => 0, + }; + Search { + hot, + puz, + prior, + policy, + rng: Rng(seed), + weight: vec![0; puz.slots()], + budget, + cap, + stats: Stats::default(), + } + } + + /// The slot to branch on, or `None` when every slot is placed. + fn select(&mut self, st: &State) -> Option { + let open: Vec = (0..self.puz.slots()) + .filter(|&s| st.placed[s] == UNSET) + .collect(); + if open.is_empty() { + return None; + } + let by_count = |f: &dyn Fn(usize) -> f64| { + open.iter() + .copied() + .min_by(|&a, &b| f(a).total_cmp(&f(b)).then(a.cmp(&b))) + }; + // How often the two entropy readings disagree, measured on the + // popcount search's own decisions. + let pc = by_count(&|s| st.count(s) as f64); + if self.policy == Policy::Popcount { + let sh = by_count(&|s| self.prior.entropy(st.mask(s)).unwrap_or(-1.0)); + self.stats.decisions += 1; + self.stats.disagreements += u64::from(pc != sh); + } + match self.policy { + Policy::Fixed => Some(open[0]), + Policy::Random(_) => Some(open[self.rng.below(open.len() as u64) as usize]), + Policy::Widest => by_count(&|s| -(st.count(s) as f64)), + Policy::Popcount => pc, + Policy::Shannon => by_count(&|s| self.prior.entropy(st.mask(s)).unwrap_or(-1.0)), + Policy::DomWdeg => by_count(&|s| st.count(s) as f64 / (1 + self.weight[s]) as f64), + } + } + + /// Depth-first: fills under `st`, up to the cap. `false` once the + /// budget or the cap stops the search. + fn go(&mut self, st: State) -> bool { + let Some(s) = self.select(&st) else { + if no_repeats(&st.placed) { + self.stats.fills += 1; + if self.stats.first.is_none() { + self.stats.first = Some(st.placed.clone()); + self.stats.nodes_to_first = Some(self.stats.nodes); + } + } + return self.stats.fills < self.cap; + }; + for w in bits(st.mask(s)) { + if st.placed.contains(&w) { + continue; + } + if self.stats.nodes >= self.budget { + self.stats.exhausted = true; + return false; + } + self.stats.nodes += 1; + let mut next = st.clone(); + next.place(s, w); + let mut q = vec![s as u8]; + let failed = match propagate_token(self.hot, self.puz, &mut next, &mut q) { + Err(Stop::Contradiction(t)) => { + self.weight[t as usize] += 1; + true + } + Err(Stop::Language) => unreachable!("one language"), + Ok(()) => !no_repeats(&next.placed), + }; + self.stats.contradictions += u64::from(failed); + if self.stats.fail_trace.len() < TRACE_CAP { + self.stats.fail_trace.push(failed); + } + if !failed && !self.go(next) { + return false; + } + } + true + } +} + +/// Search from the fixed point of `givens`. +fn search( + hot: &Hot, + puz: &Puzzle, + prior: &Prior, + givens: &[(u8, WordId)], + policy: Policy, + budget: u64, + cap: u64, +) -> Stats { + let mut s = Search::new(hot, puz, prior, policy, budget, cap); + if let Ok((mut st, mut q)) = start(hot, puz, givens) { + if propagate_token(hot, puz, &mut st, &mut q).is_ok() { + s.go(st); + } + } + s.stats +} + +/// The workloads, built once. +struct Bench { + hot: Hot, + prior: Prior, + created: Vec, + empty: Vec, +} + +fn bench(sides: &[usize], per_side: usize, seed: u64) -> Bench { + let (hot, cold) = build_lexicon(Lang::En, &english_ranked()); + let prior = Prior::coca(&cold); + let mut rng = Rng(seed); + let mut created = Vec::new(); + let mut empty = Vec::new(); + for &side in sides { + let mut made = 0; + while made < per_side { + if let Ok(c) = create(&hot, side, &mut rng, 100_000) { + created.push(c); + made += 1; + } + } + for _ in 0..per_side { + empty.push(compile(&Grid::random_nyt(side, &mut rng), Lang::En)); + } + } + Bench { + hot, + prior, + created, + empty, + } +} + +#[derive(Debug, Clone, Copy, Default)] +struct Row { + runs: u64, + solved: u64, + nodes: u64, + contradictions: u64, + nodes_to_first: u64, + disagreements: u64, + decisions: u64, +} + +impl Row { + fn add(&mut self, s: &Stats, solved: bool) { + self.runs += 1; + self.solved += u64::from(solved); + self.nodes += s.nodes; + self.contradictions += s.contradictions; + self.nodes_to_first += s.nodes_to_first.unwrap_or(s.nodes); + self.disagreements += s.disagreements; + self.decisions += s.decisions; + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Work { + Prove, + Fill, + Open, +} + +fn run_work(b: &Bench, work: Work, policy: Policy, budget: u64) -> (Row, Vec) { + let mut row = Row::default(); + let mut all = Vec::new(); + match work { + Work::Prove => { + for c in &b.created { + let s = search(&b.hot, &c.puz, &b.prior, &c.givens, policy, budget, 2); + row.add(&s, !s.exhausted); + all.push(s); + } + } + Work::Fill => { + for p in &b.empty { + let s = search(&b.hot, p, &b.prior, &[], policy, budget, 1); + row.add(&s, s.fills > 0); + all.push(s); + } + } + Work::Open => { + for c in &b.created { + let half = &c.givens[..c.givens.len() / 2]; + let s = search(&b.hot, &c.puz, &b.prior, half, policy, budget, 50); + row.add(&s, !s.exhausted); + all.push(s); + } + } + } + (row, all) +} + +/// Contradiction rate per tenth of a search's node sequence, pooled. +fn fail_deciles(all: &[Stats]) -> [f64; 10] { + let mut n = [0u64; 10]; + let mut f = [0u64; 10]; + for s in all { + let len = s.fail_trace.len(); + for (i, &x) in s.fail_trace.iter().enumerate() { + let d = (i * 10 / len.max(1)).min(9); + n[d] += 1; + f[d] += u64::from(x); + } + } + core::array::from_fn(|d| f[d] as f64 / n[d].max(1) as f64) +} + +fn main() { + println!("D-PUZZLE-ATTN-0: entropy as the focus of attention on crosswords"); + let t0 = Instant::now(); + let b = bench(&[5, 7], 10, 0xA77E); + println!( + " workload: {} created puzzles, {} empty grids (sides 5/7), built in {:.1} s", + b.created.len(), + b.empty.len(), + t0.elapsed().as_secs_f64() + ); + let budget = 200_000; + for (name, work) in [ + ("prove (count to 2)", Work::Prove), + ("fill (empty grid)", Work::Fill), + ("open (half the givens, count to 50)", Work::Open), + ] { + println!("\n{name}, budget {budget} nodes per run:"); + println!( + " {:<10} {:>6} {:>10} {:>10} {:>12} {:>8} {:>13}", + "policy", "solved", "nodes", "contra", "to first", "ms", "Sh≠Pc" + ); + for p in POLICIES { + let t = Instant::now(); + let (r, all) = run_work(&b, work, p, budget); + let ms = t.elapsed().as_secs_f64() * 1e3; + println!( + " {:<10} {:>3}/{:<2} {:>10} {:>10} {:>12} {:>8.1} {:>13}", + format!("{p:?}").split('(').next().unwrap_or(""), + r.solved, + r.runs, + r.nodes, + r.contradictions, + r.nodes_to_first, + ms, + if p == Policy::Popcount { + format!("{:.3}", r.disagreements as f64 / r.decisions.max(1) as f64) + } else { + "-".into() + }, + ); + if matches!(p, Policy::Popcount | Policy::DomWdeg) { + let d = fail_deciles(&all); + println!( + " contradiction rate by tenth of the search: {}", + d.iter() + .map(|x| format!("{x:.2}")) + .collect::>() + .join(" ") + ); + } + } + } +} + +// ── Falsifiers ──────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::OnceLock; + + /// Small and shared: sides 5 and 7, a few of each. + fn bench_small() -> &'static Bench { + static B: OnceLock = OnceLock::new(); + B.get_or_init(|| bench(&[5], 6, 0xA77E)) + } + + const BUDGET: u64 = 50_000; + + /// Measurement helper (not an assertion): the small-fixture table. + #[test] + #[ignore] + fn dump_small() { + let b = bench_small(); + for work in [Work::Prove, Work::Fill, Work::Open] { + for p in POLICIES { + let (r, _) = run_work(b, work, p, BUDGET); + println!( + "{work:?} {p:?} solved {}/{} nodes {} first {} contra {}", + r.solved, r.runs, r.nodes, r.nodes_to_first, r.contradictions + ); + } + } + } + + /// A contradiction is not certainty: an empty slot has no entropy under + /// either reading; a single candidate has zero. + #[test] + fn a_contradiction_has_no_entropy() { + let b = bench_small(); + assert_eq!(b.prior.entropy(&[0, 0]), None); + assert_eq!(uniform_entropy(0), None); + assert_eq!(b.prior.entropy(&[1, 0]), Some(0.0)); + assert_eq!(uniform_entropy(1), Some(0.0)); + } + + /// Shannon under the frequency prior is not log2(popcount): two slots + /// with the same 128 candidates, one of them holding "the" (the most + /// frequent word), differ by more than a bit while log2(popcount) reads + /// both as exactly 7. + #[test] + fn shannon_is_not_log2_popcount() { + let b = bench_small(); + let n = b.prior.z.len(); + let mut flat = vec![0u64; b.hot.blocks]; + for id in n - 200..n - 72 { + flat[id / 64] |= 1 << (id % 64); + } + let mut skewed = flat.clone(); + let first = n - 200; + skewed[first / 64] &= !(1 << (first % 64)); + skewed[0] |= 1; // id 0 = "the" + let count = |m: &[u64]| m.iter().map(|w| w.count_ones()).sum::(); + assert_eq!((count(&flat), count(&skewed)), (128, 128)); + let (hf, hs) = ( + b.prior.entropy(&flat).unwrap(), + b.prior.entropy(&skewed).unwrap(), + ); + assert!(hs < hf - 1.0, "{hs} vs {hf}"); + assert!((uniform_entropy(128).unwrap() - 7.0).abs() < 1e-12); + } + + /// Attention never changes the answer: on unique puzzles every policy + /// that finishes finds exactly one fill, and it is the solution. + #[test] + fn attention_changes_cost_never_the_answer() { + let b = bench_small(); + for p in POLICIES { + let (_, all) = run_work(b, Work::Prove, p, BUDGET); + for (c, s) in b.created.iter().zip(&all) { + if s.exhausted { + continue; + } + assert_eq!(s.fills, 1, "{p:?}"); + assert_eq!(s.first.as_ref(), Some(&c.solution), "{p:?}"); + } + } + } + + /// Focus matters: most-constrained-first spends far fewer nodes than + /// a fixed order and than widest-first, and solves at least as many. + #[test] + fn most_constrained_first_beats_fixed_order_and_widest_first() { + let b = bench_small(); + for work in [Work::Prove, Work::Fill, Work::Open] { + let pc = run_work(b, work, Policy::Popcount, BUDGET).0; + for base in [Policy::Fixed, Policy::Widest] { + let r = run_work(b, work, base, BUDGET).0; + assert!(pc.solved >= r.solved, "{work:?} {base:?}"); + assert!( + 2 * pc.nodes < r.nodes, + "{work:?} {base:?}: {} vs {}", + pc.nodes, + r.nodes + ); + } + } + } + + /// The measured loss: frequency-weighted Shannon attention spends more + /// nodes than popcount. Search cost is the number of candidates to try, + /// not how predictable the slot is. If this ever flips, re-read it. + #[test] + fn shannon_attention_costs_more_than_popcount() { + let b = bench_small(); + for work in [Work::Fill, Work::Open] { + let pc = run_work(b, work, Policy::Popcount, BUDGET).0; + let sh = run_work(b, work, Policy::Shannon, BUDGET).0; + assert!( + sh.nodes > pc.nodes, + "{work:?}: {} vs {}", + sh.nodes, + pc.nodes + ); + assert!(sh.solved <= pc.solved, "{work:?}"); + } + } + + /// DomWdeg's weights only grow from contradictions, and a search with + /// none behaves exactly like popcount. + #[test] + fn dom_wdeg_is_popcount_until_something_fails() { + let b = bench_small(); + let c = &b.created[0]; + // From the full givens propagation finishes the puzzle: no choice, + // no contradiction, and the two policies are identical. + let pc = search( + &b.hot, + &c.puz, + &b.prior, + &c.givens, + Policy::Popcount, + BUDGET, + 2, + ); + let dw = search( + &b.hot, + &c.puz, + &b.prior, + &c.givens, + Policy::DomWdeg, + BUDGET, + 2, + ); + if pc.contradictions == 0 { + assert_eq!(pc, dw); + } + // Can fire: on empty grids contradictions happen and the weights + // move the search somewhere else. + let (_, a) = run_work(b, Work::Fill, Policy::Popcount, BUDGET); + let (_, d) = run_work(b, Work::Fill, Policy::DomWdeg, BUDGET); + assert!(a.iter().map(|s| s.contradictions).sum::() > 0); + assert_ne!(a, d); + } + + /// Same puzzles, same seed, same policy: the same search. + #[test] + fn replay_is_deterministic() { + let b = bench_small(); + for p in POLICIES { + assert_eq!( + run_work(b, Work::Fill, p, 5_000).1, + run_work(b, Work::Fill, p, 5_000).1, + "{p:?}" + ); + } + } +} diff --git a/crates/cognitive-shader-driver/examples/crossword_mask_propagation_probe.rs b/crates/cognitive-shader-driver/examples/crossword_mask_propagation_probe.rs index cae4103d2..55c6c51e3 100644 --- a/crates/cognitive-shader-driver/examples/crossword_mask_propagation_probe.rs +++ b/crates/cognitive-shader-driver/examples/crossword_mask_propagation_probe.rs @@ -70,10 +70,8 @@ use std::collections::HashMap; use std::hint::black_box; use std::time::{Duration, Instant}; -use deepnsm_v2::vocab::{PaletteVocab, WordId}; +use deepnsm_v2::vocab::WordId; use lance_graph_contract::class_view::ClassId; -use lance_graph_contract::facet::FacetTier; -use lance_graph_contract::morton8x8::Morton8x8; use ndarray::simd::{mask_and_assign, popcount_batch_u64}; #[path = "shared/population_fold.rs"] @@ -83,439 +81,15 @@ use population_fold::{ FORCED, GIVEN, }; +#[path = "shared/crossword_core.rs"] +mod crossword_core; +use crossword_core::*; + /// Same class as steps 2 and 2b: the same domain. const CROSSWORD_CLASS: ClassId = 0x0907; -/// NYT rules: shortest word; most black squares (a sixth). -const MIN_WORD: usize = 3; -const MAX_BLACK_DIVISOR: usize = 6; -/// Longest slot (21×21 Sunday board) and the per-slot stride of position lanes. -const MAX_LEN: usize = 21; -const STRIDE: usize = 32; -/// No crossing / no occupant. -const NONE: u16 = u16::MAX; -/// No word placed in this slot. -const UNSET: WordId = WordId::MAX; -/// German forms kept. -const GERMAN_VOCAB: usize = 20_000; - -const ACADEMIC: &str = include_str!("../../deepnsm/word_frequency/academic_20k.csv"); - -// ─────────────────────────────── symbols ─────────────────────────────── - -/// The declared letter reading of one Moore-local byte: `0` unknown, `1..=26` -/// a–z, `27` ä, `28` ö, `29` ü, `30` ß, `31..=255` reserved. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] -#[repr(transparent)] -struct MooreSymbol8(u8); - -impl MooreSymbol8 { - const UNKNOWN: Self = Self(0); - /// Codes in use, `0..=30`. - const COUNT: usize = 31; - - /// The symbol of a lowercase character after accent folding. - fn of(c: char) -> Option { - match fold(c) { - c @ 'a'..='z' => Some(Self(c as u8 - b'a' + 1)), - 'ä' => Some(Self(27)), - 'ö' => Some(Self(28)), - 'ü' => Some(Self(29)), - 'ß' => Some(Self(30)), - _ => None, - } - } - - /// The letter of a symbol; `None` for unknown and reserved codes. - fn letter(self) -> Option { - match self.0 { - 1..=26 => Some((b'a' + self.0 - 1) as char), - 27 => Some('ä'), - 28 => Some('ö'), - 29 => Some('ü'), - 30 => Some('ß'), - _ => None, - } - } -} - -/// Crossword spelling ignores accents: an accented Latin letter folds to its -/// base. ä ö ü ß are letters of their own and do not fold. -fn fold(c: char) -> char { - match c { - 'à' | 'á' | 'â' | 'ã' | 'å' | 'ā' => 'a', - 'ç' => 'c', - 'è' | 'é' | 'ê' | 'ë' | 'ē' => 'e', - 'ì' | 'í' | 'î' | 'ï' => 'i', - 'ñ' => 'n', - 'ò' | 'ó' | 'ô' | 'õ' => 'o', - 'ù' | 'ú' | 'û' => 'u', - 'ý' | 'ÿ' => 'y', - c => c, - } -} - -/// A word's crossword spelling, or `None` if it is not a crossword word -/// (wrong length, or a character with no symbol). -fn spelling(w: &str) -> Option> { - let s: Option> = w.chars().map(MooreSymbol8::of).collect(); - s.filter(|s| (MIN_WORD..=MAX_LEN).contains(&s.len())) -} - -// ─────────────────────────────── vocabulary ─────────────────────────────── - -/// The language a lexicon and a puzzle belong to. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Lang { - En, - De, -} - -/// Cold side: the strings. Construction, the literal arm and checks only. -struct Cold { - vocab: PaletteVocab, -} - -impl Cold { - fn word(&self, w: WordId) -> &str { - self.vocab.word(w).expect("word id in range") - } -} - -/// Hot side: integers and masks only. No `String`, no `char`. -struct Hot { - lang: Lang, - blocks: usize, - /// Per `WordId`: crossword length, 0 = not a crossword word. - len: Vec, - /// `[id * STRIDE + offset]` = the symbol. - spell: Vec, - /// By length: every crossword word of that length. - all: Vec>, - /// By length: `[(offset * COUNT + symbol) * blocks ..]` = `P(L, offset, symbol)`. - at: Vec>, -} - -impl Hot { - fn pop(&self, len: usize, offset: usize, x: MooreSymbol8) -> &[u64] { - let i = (offset * MooreSymbol8::COUNT + x.0 as usize) * self.blocks; - &self.at[len][i..i + self.blocks] - } - - fn letter(&self, w: WordId, offset: usize) -> MooreSymbol8 { - self.spell[w as usize * STRIDE + offset] - } - - /// Bytes held by the populations (the fixed per-language cost). - fn population_bytes(&self) -> usize { - 8 * (self.all.iter().map(Vec::len).sum::() - + self.at.iter().map(Vec::len).sum::()) - } -} - -/// Both sides from a frequency-ranked word list (most frequent first). -fn build_lexicon(lang: Lang, ranked: &[String]) -> (Hot, Cold) { - let mut vocab = PaletteVocab::new(); - vocab.from_frequency_ranked(ranked.iter().map(String::as_str)); - let n = vocab.len(); - assert!(n < UNSET as usize, "WordId::MAX is the empty marker"); - let blocks = n.div_ceil(64); - let mut hot = Hot { - lang, - blocks, - len: vec![0; n], - spell: vec![MooreSymbol8::UNKNOWN; n * STRIDE], - all: vec![Vec::new(); MAX_LEN + 1], - at: vec![Vec::new(); MAX_LEN + 1], - }; - for l in MIN_WORD..=MAX_LEN { - hot.all[l] = vec![0; blocks]; - hot.at[l] = vec![0; l * MooreSymbol8::COUNT * blocks]; - } - for id in 0..n { - let Some(sp) = spelling(vocab.word(id as WordId).expect("in range")) else { - continue; - }; - let l = sp.len(); - hot.len[id] = l as u8; - let bit = 1u64 << (id % 64); - hot.all[l][id / 64] |= bit; - for (i, &x) in sp.iter().enumerate() { - hot.spell[id * STRIDE + i] = x; - hot.at[l][(i * MooreSymbol8::COUNT + x.0 as usize) * blocks + id / 64] |= bit; - } - } - (hot, Cold { vocab }) -} - -/// English: DeepNSM-v2's academic vocabulary, built as `genre_shapes` builds it. -fn english_ranked() -> Vec { - ACADEMIC - .lines() - .skip(1) - .filter_map(|l| l.split(',').nth(3)) - .map(str::to_lowercase) - .filter(|w| !w.is_empty()) - .collect() -} - -/// German: the `GERMAN_VOCAB` most frequent DeReKo forms (`form, lemma, tag, -/// freq`), lowercased, `NE` and non-alphabetic forms dropped, summed. -fn german_ranked(text: &str) -> Vec { - let mut f: HashMap = HashMap::new(); - for line in text.lines() { - let c: Vec<&str> = line.split('\t').collect(); - let [form, _, tag, freq] = c.as_slice() else { - continue; - }; - if *tag == "NE" { - continue; - } - let w = form.to_lowercase(); - if w.is_empty() || !w.chars().all(char::is_alphabetic) { - continue; - } - *f.entry(w).or_insert(0.0) += freq.trim().parse::().unwrap_or(0.0); - } - let mut r: Vec<(String, f64)> = f.into_iter().collect(); - r.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); - r.truncate(GERMAN_VOCAB); - r.into_iter().map(|(w, _)| w).collect() -} - -// ─────────────────────────────── layout ─────────────────────────────── - -/// A board as row masks: bit `c` of `rows[r]` = cell `(r, c)` is white. -#[derive(Clone, Debug, PartialEq)] -struct Grid { - side: usize, - rows: Vec, -} - -/// Every white bit of a line lies in a run of at least three: the OR of the -/// three-wide windows covers the line. -fn line_runs_ok(w: u32) -> bool { - let t = w & (w >> 1) & (w >> 2); - (t | (t << 1) | (t << 2)) == w -} - -impl Grid { - #[cfg(test)] - fn from_rows(rows: &[&str]) -> Self { - let rows: Vec = rows - .iter() - .map(|r| { - r.bytes() - .enumerate() - .fold(0u32, |m, (c, b)| m | (u32::from(b == b'.') << c)) - }) - .collect(); - Self { - side: rows.len(), - rows, - } - } - - fn full_line(side: usize) -> u32 { - ((1u64 << side) - 1) as u32 - } - - fn cols(&self) -> Vec { - (0..self.side) - .map(|c| (0..self.side).fold(0u32, |m, r| m | (((self.rows[r] >> c) & 1) << r))) - .collect() - } - - fn blacks(&self) -> usize { - self.side * self.side - - self - .rows - .iter() - .map(|r| r.count_ones() as usize) - .sum::() - } - - /// 180° symmetry: row `r` reversed is row `side - 1 - r`. - fn symmetric(&self) -> bool { - let rev = |w: u32| w.reverse_bits() >> (32 - self.side); - (0..self.side).all(|r| rev(self.rows[r]) == self.rows[self.side - 1 - r]) - } - - /// Every across and down run is at least `MIN_WORD` long. - fn runs_ok(&self) -> bool { - self.rows - .iter() - .chain(self.cols().iter()) - .all(|&w| line_runs_ok(w)) - } - - /// One white region: grow from the first white cell by row-mask dilation. - fn connected(&self) -> bool { - let Some(r0) = self.rows.iter().position(|&w| w != 0) else { - return false; - }; - let full = Self::full_line(self.side); - let mut reach = vec![0u32; self.side]; - reach[r0] = self.rows[r0] & self.rows[r0].wrapping_neg(); - loop { - let mut changed = false; - for r in 0..self.side { - let up = if r > 0 { reach[r - 1] } else { 0 }; - let down = if r + 1 < self.side { reach[r + 1] } else { 0 }; - let grown = (reach[r] | (reach[r] << 1) | (reach[r] >> 1) | up | down) - & full - & self.rows[r]; - if grown != reach[r] { - reach[r] = grown; - changed = true; - } - } - if !changed { - return reach == self.rows; - } - } - } - - fn is_nyt_valid(&self) -> bool { - self.symmetric() - && self.runs_ok() - && self.connected() - && self.blacks() * MAX_BLACK_DIVISOR <= self.side * self.side - } - - /// Symmetric black pairs drawn until the grid is NYT-valid with at least - /// one black square. - fn random_nyt(side: usize, rng: &mut Rng) -> Self { - let cells = side * side; - loop { - let mut g = Self { - side, - rows: vec![Self::full_line(side); side], - }; - for i in 0..cells.div_ceil(2) { - if rng.below(8) == 0 { - for j in [i, cells - 1 - i] { - g.rows[j / side] &= !(1 << (j % side)); - } - } - } - if g.blacks() > 0 && g.is_nyt_valid() { - return g; - } - } - } -} - -/// A compiled puzzle: everything the runtime may read. -#[derive(Clone)] -struct Puzzle { - lang: Lang, - /// Per slot: its length. - len: Vec, - /// `[slot * STRIDE + offset]`: the crossing `(slot:offset)` tile, or `NONE`. - cross: Vec, - /// `[slot * STRIDE + offset]`: the Morton cell code. - cell: Vec, - /// `[cell]`: the (at most two) `(slot:offset)` tiles on a cell. - occupant: Vec<[u16; 2]>, -} - -impl Puzzle { - fn slots(&self) -> usize { - self.len.len() - } -} - -fn tile(s: usize, o: usize) -> u16 { - FacetTier { - hi: s as u8, - lo: o as u8, - } - .as_u16() -} - -fn untile(t: u16) -> (usize, usize) { - ((t >> 8) as usize, (t & 0xFF) as usize) -} - -/// Compile a grid: slots from the row and column masks, each position's cell -/// from `Morton8x8::checked_offset`, crossings from equal cell codes. -fn compile(grid: &Grid, lang: Lang) -> Puzzle { - let mut len = Vec::new(); - let mut cell = Vec::new(); - for (across, lines) in [(true, grid.rows.clone()), (false, grid.cols())] { - for (line, &w) in lines.iter().enumerate() { - let mut c = 0; - while c < grid.side { - if w >> c & 1 == 0 { - c += 1; - continue; - } - let start = c; - while c < grid.side && w >> c & 1 == 1 { - c += 1; - } - let l = c - start; - if l < 2 { - continue; - } - let (x, y) = if across { (start, line) } else { (line, start) }; - let (dx, dy) = if across { (1i8, 0i8) } else { (0, 1) }; - let origin = Morton8x8::from_xy(x as u8, y as u8); - let mut codes = [NONE; STRIDE]; - for (off, code) in codes.iter_mut().enumerate().take(l) { - *code = origin - .checked_offset(dx * off as i8, dy * off as i8) - .expect("on the board") - .code(); - } - len.push(l as u8); - cell.extend_from_slice(&codes); - } - } - } - assert!(len.len() < 255, "slot ids are one byte"); - let tiles = cell - .iter() - .filter(|&&c| c != NONE) - .max() - .map_or(0, |&m| m as usize + 1); - let mut occupant = vec![[NONE; 2]; tiles]; - let mut cross = vec![NONE; len.len() * STRIDE]; - for (s, &l) in len.iter().enumerate() { - for o in 0..l as usize { - let c = cell[s * STRIDE + o] as usize; - let slot = if occupant[c][0] == NONE { 0 } else { 1 }; - assert_eq!(occupant[c][slot], NONE, "at most two positions per cell"); - occupant[c][slot] = tile(s, o); - if slot == 1 { - let other = occupant[c][0]; - cross[s * STRIDE + o] = other; - let (t, j) = untile(other); - cross[t * STRIDE + j] = tile(s, o); - } - } - } - Puzzle { - lang, - len, - cross, - cell, - occupant, - } -} - // ─────────────────────────────── arms ─────────────────────────────── -/// Why a solve did not start or did not finish. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Stop { - /// The puzzle and the populations are different languages. - Language, - /// A slot was left with no candidate, or two letters disagree on a cell. - Contradiction(u8), -} - /// What a solve leaves, and what it cost. #[derive(Clone, Default)] struct Solved { @@ -533,146 +107,6 @@ struct Solved { /// One mask arm: puzzle and givens in, fixed point out. type Arm = fn(&Hot, &Puzzle, &[(u8, WordId)]) -> Result; -/// Per-slot candidate masks plus the placed word, shared by arms A and D. -#[derive(Clone)] -struct State { - blocks: usize, - cand: Vec, - placed: Vec, - ands: usize, -} - -impl State { - fn new(hot: &Hot, puz: &Puzzle) -> Result { - if hot.lang != puz.lang { - return Err(Stop::Language); - } - let b = hot.blocks; - let mut cand = vec![0u64; puz.slots() * b]; - for (s, &l) in puz.len.iter().enumerate() { - cand[s * b..(s + 1) * b].copy_from_slice(&hot.all[l as usize]); - } - Ok(Self { - blocks: b, - cand, - placed: vec![UNSET; puz.slots()], - ands: 0, - }) - } - - fn mask(&self, s: usize) -> &[u64] { - &self.cand[s * self.blocks..(s + 1) * self.blocks] - } - - fn count(&self, s: usize) -> u64 { - popcount_batch_u64(self.mask(s)) - } - - fn place(&mut self, s: usize, w: WordId) { - let b = self.blocks; - self.cand[s * b..(s + 1) * b].fill(0); - self.cand[s * b + w as usize / 64] = 1 << (w % 64); - self.placed[s] = w; - } - - /// Promote every unplaced slot already down to one candidate. - fn seed(&mut self, queue: &mut Vec) -> Result<(), Stop> { - for s in 0..self.placed.len() { - if self.placed[s] == UNSET { - match self.count(s) { - 0 => return Err(Stop::Contradiction(s as u8)), - 1 => { - let w = first_bit(self.mask(s)); - self.place(s, w); - queue.push(s as u8); - } - _ => {} - } - } - } - Ok(()) - } - - /// AND one population into slot `t`; promote at popcount 1. - fn narrow(&mut self, t: usize, p: &[u64], queue: &mut Vec) -> Result<(), Stop> { - let b = self.blocks; - let m = &mut self.cand[t * b..(t + 1) * b]; - mask_and_assign(m, p); - self.ands += 1; - match popcount_batch_u64(m) { - 0 => Err(Stop::Contradiction(t as u8)), - 1 => { - self.placed[t] = first_bit(m); - queue.push(t as u8); - Ok(()) - } - _ => Ok(()), - } - } -} - -fn first_bit(m: &[u64]) -> WordId { - let (b, w) = m - .iter() - .enumerate() - .find(|(_, &w)| w != 0) - .expect("non-empty mask"); - (b * 64 + w.trailing_zeros() as usize) as WordId -} - -fn bits(m: &[u64]) -> Vec { - let mut out = Vec::new(); - for (b, &word) in m.iter().enumerate() { - let mut w = word; - while w != 0 { - out.push((b * 64 + w.trailing_zeros() as usize) as WordId); - w &= w - 1; - } - } - out -} - -/// Arm A's fixed point: each queued slot hands its letter at every crossed -/// offset to the crossing `(slot:offset)` read from `cross`. -fn propagate_token( - hot: &Hot, - puz: &Puzzle, - st: &mut State, - queue: &mut Vec, -) -> Result<(), Stop> { - while let Some(s) = queue.pop() { - let s = s as usize; - let w = st.placed[s]; - for off in 0..puz.len[s] as usize { - let c = puz.cross[s * STRIDE + off]; - if c == NONE { - continue; - } - let (t, j) = untile(c); - let x = hot.letter(w, off); - if st.placed[t] != UNSET { - if hot.letter(st.placed[t], j) != x { - return Err(Stop::Contradiction(t as u8)); - } - continue; - } - st.narrow(t, hot.pop(puz.len[t] as usize, j, x), queue)?; - } - } - Ok(()) -} - -fn start(hot: &Hot, puz: &Puzzle, givens: &[(u8, WordId)]) -> Result<(State, Vec), Stop> { - let mut st = State::new(hot, puz)?; - let mut queue = Vec::new(); - for &(s, w) in givens { - st.place(s as usize, w); - queue.push(s); - } - st.seed(&mut queue)?; - Ok((st, queue)) -} - /// Arm A: token masks over the `cross` lane. fn solve_token(hot: &Hot, puz: &Puzzle, givens: &[(u8, WordId)]) -> Result { let (mut st, mut queue) = start(hot, puz, givens)?; @@ -903,143 +337,6 @@ fn solve_literal(lit: &Literal, puz: &Puzzle, givens: &[(u8, WordId)]) -> Option } } -// ─────────────────────────────── creation ─────────────────────────────── - -/// Why creation gave up on one attempt. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Miss { - /// No complete fill found within the budget. - Fill, - /// A fill was found, but uniqueness could not be decided within the budget. - Unique, -} - -/// Crossword fills never repeat an entry: no two placed slots hold the same -/// word. Both creation searches apply this rule, so the generator and the -/// uniqueness count agree on what a fill is. -fn no_repeats(placed: &[WordId]) -> bool { - let mut seen: Vec = placed.iter().copied().filter(|&w| w != UNSET).collect(); - let n = seen.len(); - seen.sort_unstable(); - seen.dedup(); - seen.len() == n -} - -/// Depth-first fill over arm A's masks, most-constrained slot first, random -/// order within it. `budget` counts tried words. Creation only. -fn random_fill(hot: &Hot, puz: &Puzzle, rng: &mut Rng, budget: &mut usize) -> Option> { - fn go(hot: &Hot, puz: &Puzzle, st: State, rng: &mut Rng, budget: &mut usize) -> Option { - let Some(s) = (0..puz.slots()) - .filter(|&s| st.placed[s] == UNSET) - .min_by_key(|&s| st.count(s)) - else { - return no_repeats(&st.placed).then_some(st); - }; - let mut cs = bits(st.mask(s)); - cs.retain(|w| !st.placed.contains(w)); - rng.shuffle(&mut cs); - for w in cs { - if *budget == 0 { - return None; - } - *budget -= 1; - let mut next = st.clone(); - next.place(s, w); - let mut q = vec![s as u8]; - if propagate_token(hot, puz, &mut next, &mut q).is_ok() && no_repeats(&next.placed) { - if let Some(done) = go(hot, puz, next, rng, budget) { - return Some(done); - } - } - } - None - } - let (mut st, mut q) = start(hot, puz, &[]).ok()?; - propagate_token(hot, puz, &mut st, &mut q).ok()?; - go(hot, puz, st, rng, budget).map(|st| st.placed) -} - -/// Complete fills consistent with `givens`, up to `cap`; `None` when the -/// budget runs out. The uniqueness oracle for creation. -fn count_fills( - hot: &Hot, - puz: &Puzzle, - givens: &[(u8, WordId)], - cap: usize, - budget: &mut usize, -) -> Option { - fn go(hot: &Hot, puz: &Puzzle, st: State, cap: usize, budget: &mut usize) -> Option { - let Some(s) = (0..puz.slots()) - .filter(|&s| st.placed[s] == UNSET) - .min_by_key(|&s| st.count(s)) - else { - return Some(usize::from(no_repeats(&st.placed))); - }; - let mut n = 0; - for w in bits(st.mask(s)) - .into_iter() - .filter(|w| !st.placed.contains(w)) - { - if *budget == 0 { - return None; - } - *budget -= 1; - let mut next = st.clone(); - next.place(s, w); - let mut q = vec![s as u8]; - if propagate_token(hot, puz, &mut next, &mut q).is_ok() && no_repeats(&next.placed) { - n += go(hot, puz, next, cap - n, budget)?; - if n >= cap { - break; - } - } - } - Some(n) - } - let Ok((mut st, mut q)) = start(hot, puz, givens) else { - return Some(0); - }; - if propagate_token(hot, puz, &mut st, &mut q).is_err() { - return Some(0); - } - go(hot, puz, st, cap, budget) -} - -/// A created puzzle: its compiled layout, its solution and the givens that -/// make the solution unique. -struct Created { - puz: Puzzle, - solution: Vec, - givens: Vec<(u8, WordId)>, - compile_ns: f64, -} - -/// Grid, compile, random fill, then givens in random order until unique. -fn create(hot: &Hot, side: usize, rng: &mut Rng, budget: usize) -> Result { - let grid = Grid::random_nyt(side, rng); - let t = Instant::now(); - let puz = compile(&grid, hot.lang); - let compile_ns = t.elapsed().as_nanos() as f64; - let mut b = budget; - let solution = random_fill(hot, &puz, rng, &mut b).ok_or(Miss::Fill)?; - let mut order: Vec = (0..puz.slots() as u8).collect(); - rng.shuffle(&mut order); - let mut givens = Vec::new(); - for s in order { - givens.push((s, solution[s as usize])); - let mut b = budget; - if count_fills(hot, &puz, &givens, 2, &mut b).ok_or(Miss::Unique)? == 1 { - return Ok(Created { - puz, - solution, - givens, - compile_ns, - }); - } - } - unreachable!("every slot given is unique") -} - // ─────────────────────────────── ablation ─────────────────────────────── /// What removing one given does: the fixed point re-run without it, and diff --git a/crates/cognitive-shader-driver/examples/shared/crossword_core.rs b/crates/cognitive-shader-driver/examples/shared/crossword_core.rs new file mode 100644 index 000000000..ee6570e95 --- /dev/null +++ b/crates/cognitive-shader-driver/examples/shared/crossword_core.rs @@ -0,0 +1,733 @@ +//! The crossword core shared by the D-PUZZLE-0 crossword probes: the +//! `MooreSymbol8` codebook, lexicons and `P(L, i, x)` populations, NYT grids, +//! compilation into slot / crossing lanes, the token-mask state with its fixed +//! point, and puzzle creation. Moved unchanged from +//! `crossword_mask_propagation_probe.rs` (#1387) so the attention probe +//! (D-PUZZLE-ATTN-0) runs on the same code; included with `#[path]`. +#![allow(dead_code)] + +use std::collections::HashMap; +use std::time::Instant; + +use deepnsm_v2::vocab::{PaletteVocab, WordId}; +use lance_graph_contract::facet::FacetTier; +use lance_graph_contract::morton8x8::Morton8x8; +use ndarray::simd::{mask_and_assign, popcount_batch_u64}; + +use super::population_fold::Rng; + +/// NYT rules: shortest word; most black squares (a sixth). +pub const MIN_WORD: usize = 3; +pub const MAX_BLACK_DIVISOR: usize = 6; +/// Longest slot (21×21 Sunday board) and the per-slot stride of position lanes. +pub const MAX_LEN: usize = 21; +pub const STRIDE: usize = 32; +/// No crossing / no occupant. +pub const NONE: u16 = u16::MAX; +/// No word placed in this slot. +pub const UNSET: WordId = WordId::MAX; +/// German forms kept. +pub const GERMAN_VOCAB: usize = 20_000; + +pub const ACADEMIC: &str = include_str!("../../../deepnsm/word_frequency/academic_20k.csv"); + +// ─────────────────────────────── symbols ─────────────────────────────── + +/// The declared letter reading of one Moore-local byte: `0` unknown, `1..=26` +/// a–z, `27` ä, `28` ö, `29` ü, `30` ß, `31..=255` reserved. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +#[repr(transparent)] +pub struct MooreSymbol8(pub u8); + +impl MooreSymbol8 { + pub const UNKNOWN: Self = Self(0); + /// Codes in use, `0..=30`. + pub const COUNT: usize = 31; + + /// The symbol of a lowercase character after accent folding. + pub fn of(c: char) -> Option { + match fold(c) { + c @ 'a'..='z' => Some(Self(c as u8 - b'a' + 1)), + 'ä' => Some(Self(27)), + 'ö' => Some(Self(28)), + 'ü' => Some(Self(29)), + 'ß' => Some(Self(30)), + _ => None, + } + } + + /// The letter of a symbol; `None` for unknown and reserved codes. + pub fn letter(self) -> Option { + match self.0 { + 1..=26 => Some((b'a' + self.0 - 1) as char), + 27 => Some('ä'), + 28 => Some('ö'), + 29 => Some('ü'), + 30 => Some('ß'), + _ => None, + } + } +} + +/// Crossword spelling ignores accents: an accented Latin letter folds to its +/// base. ä ö ü ß are letters of their own and do not fold. +pub fn fold(c: char) -> char { + match c { + 'à' | 'á' | 'â' | 'ã' | 'å' | 'ā' => 'a', + 'ç' => 'c', + 'è' | 'é' | 'ê' | 'ë' | 'ē' => 'e', + 'ì' | 'í' | 'î' | 'ï' => 'i', + 'ñ' => 'n', + 'ò' | 'ó' | 'ô' | 'õ' => 'o', + 'ù' | 'ú' | 'û' => 'u', + 'ý' | 'ÿ' => 'y', + c => c, + } +} + +/// A word's crossword spelling, or `None` if it is not a crossword word +/// (wrong length, or a character with no symbol). +pub fn spelling(w: &str) -> Option> { + let s: Option> = w.chars().map(MooreSymbol8::of).collect(); + s.filter(|s| (MIN_WORD..=MAX_LEN).contains(&s.len())) +} + +// ─────────────────────────────── vocabulary ─────────────────────────────── + +/// The language a lexicon and a puzzle belong to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Lang { + En, + De, +} + +/// Cold side: the strings. Construction, the literal arm and checks only. +pub struct Cold { + pub vocab: PaletteVocab, +} + +impl Cold { + pub fn word(&self, w: WordId) -> &str { + self.vocab.word(w).expect("word id in range") + } +} + +/// Hot side: integers and masks only. No `String`, no `char`. +pub struct Hot { + pub lang: Lang, + pub blocks: usize, + /// Per `WordId`: crossword length, 0 = not a crossword word. + pub len: Vec, + /// `[id * STRIDE + offset]` = the symbol. + pub spell: Vec, + /// By length: every crossword word of that length. + pub all: Vec>, + /// By length: `[(offset * COUNT + symbol) * blocks ..]` = `P(L, offset, symbol)`. + pub at: Vec>, +} + +impl Hot { + pub fn pop(&self, len: usize, offset: usize, x: MooreSymbol8) -> &[u64] { + let i = (offset * MooreSymbol8::COUNT + x.0 as usize) * self.blocks; + &self.at[len][i..i + self.blocks] + } + + pub fn letter(&self, w: WordId, offset: usize) -> MooreSymbol8 { + self.spell[w as usize * STRIDE + offset] + } + + /// Bytes held by the populations (the fixed per-language cost). + pub fn population_bytes(&self) -> usize { + 8 * (self.all.iter().map(Vec::len).sum::() + + self.at.iter().map(Vec::len).sum::()) + } +} + +/// Both sides from a frequency-ranked word list (most frequent first). +pub fn build_lexicon(lang: Lang, ranked: &[String]) -> (Hot, Cold) { + let mut vocab = PaletteVocab::new(); + vocab.from_frequency_ranked(ranked.iter().map(String::as_str)); + let n = vocab.len(); + assert!(n < UNSET as usize, "WordId::MAX is the empty marker"); + let blocks = n.div_ceil(64); + let mut hot = Hot { + lang, + blocks, + len: vec![0; n], + spell: vec![MooreSymbol8::UNKNOWN; n * STRIDE], + all: vec![Vec::new(); MAX_LEN + 1], + at: vec![Vec::new(); MAX_LEN + 1], + }; + for l in MIN_WORD..=MAX_LEN { + hot.all[l] = vec![0; blocks]; + hot.at[l] = vec![0; l * MooreSymbol8::COUNT * blocks]; + } + for id in 0..n { + let Some(sp) = spelling(vocab.word(id as WordId).expect("in range")) else { + continue; + }; + let l = sp.len(); + hot.len[id] = l as u8; + let bit = 1u64 << (id % 64); + hot.all[l][id / 64] |= bit; + for (i, &x) in sp.iter().enumerate() { + hot.spell[id * STRIDE + i] = x; + hot.at[l][(i * MooreSymbol8::COUNT + x.0 as usize) * blocks + id / 64] |= bit; + } + } + (hot, Cold { vocab }) +} + +/// English: DeepNSM-v2's academic vocabulary, built as `genre_shapes` builds it. +pub fn english_ranked() -> Vec { + ACADEMIC + .lines() + .skip(1) + .filter_map(|l| l.split(',').nth(3)) + .map(str::to_lowercase) + .filter(|w| !w.is_empty()) + .collect() +} + +/// German: the `GERMAN_VOCAB` most frequent DeReKo forms (`form, lemma, tag, +/// freq`), lowercased, `NE` and non-alphabetic forms dropped, summed. +pub fn german_ranked(text: &str) -> Vec { + let mut f: HashMap = HashMap::new(); + for line in text.lines() { + let c: Vec<&str> = line.split('\t').collect(); + let [form, _, tag, freq] = c.as_slice() else { + continue; + }; + if *tag == "NE" { + continue; + } + let w = form.to_lowercase(); + if w.is_empty() || !w.chars().all(char::is_alphabetic) { + continue; + } + *f.entry(w).or_insert(0.0) += freq.trim().parse::().unwrap_or(0.0); + } + let mut r: Vec<(String, f64)> = f.into_iter().collect(); + r.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + r.truncate(GERMAN_VOCAB); + r.into_iter().map(|(w, _)| w).collect() +} + +// ─────────────────────────────── layout ─────────────────────────────── + +/// A board as row masks: bit `c` of `rows[r]` = cell `(r, c)` is white. +#[derive(Clone, Debug, PartialEq)] +pub struct Grid { + pub side: usize, + pub rows: Vec, +} + +/// Every white bit of a line lies in a run of at least three: the OR of the +/// three-wide windows covers the line. +pub fn line_runs_ok(w: u32) -> bool { + let t = w & (w >> 1) & (w >> 2); + (t | (t << 1) | (t << 2)) == w +} + +impl Grid { + #[cfg(test)] + pub fn from_rows(rows: &[&str]) -> Self { + let rows: Vec = rows + .iter() + .map(|r| { + r.bytes() + .enumerate() + .fold(0u32, |m, (c, b)| m | (u32::from(b == b'.') << c)) + }) + .collect(); + Self { + side: rows.len(), + rows, + } + } + + pub fn full_line(side: usize) -> u32 { + ((1u64 << side) - 1) as u32 + } + + pub fn cols(&self) -> Vec { + (0..self.side) + .map(|c| (0..self.side).fold(0u32, |m, r| m | (((self.rows[r] >> c) & 1) << r))) + .collect() + } + + pub fn blacks(&self) -> usize { + self.side * self.side + - self + .rows + .iter() + .map(|r| r.count_ones() as usize) + .sum::() + } + + /// 180° symmetry: row `r` reversed is row `side - 1 - r`. + pub fn symmetric(&self) -> bool { + let rev = |w: u32| w.reverse_bits() >> (32 - self.side); + (0..self.side).all(|r| rev(self.rows[r]) == self.rows[self.side - 1 - r]) + } + + /// Every across and down run is at least `MIN_WORD` long. + pub fn runs_ok(&self) -> bool { + self.rows + .iter() + .chain(self.cols().iter()) + .all(|&w| line_runs_ok(w)) + } + + /// One white region: grow from the first white cell by row-mask dilation. + pub fn connected(&self) -> bool { + let Some(r0) = self.rows.iter().position(|&w| w != 0) else { + return false; + }; + let full = Self::full_line(self.side); + let mut reach = vec![0u32; self.side]; + reach[r0] = self.rows[r0] & self.rows[r0].wrapping_neg(); + loop { + let mut changed = false; + for r in 0..self.side { + let up = if r > 0 { reach[r - 1] } else { 0 }; + let down = if r + 1 < self.side { reach[r + 1] } else { 0 }; + let grown = (reach[r] | (reach[r] << 1) | (reach[r] >> 1) | up | down) + & full + & self.rows[r]; + if grown != reach[r] { + reach[r] = grown; + changed = true; + } + } + if !changed { + return reach == self.rows; + } + } + } + + pub fn is_nyt_valid(&self) -> bool { + self.symmetric() + && self.runs_ok() + && self.connected() + && self.blacks() * MAX_BLACK_DIVISOR <= self.side * self.side + } + + /// Symmetric black pairs drawn until the grid is NYT-valid with at least + /// one black square. + pub fn random_nyt(side: usize, rng: &mut Rng) -> Self { + let cells = side * side; + loop { + let mut g = Self { + side, + rows: vec![Self::full_line(side); side], + }; + for i in 0..cells.div_ceil(2) { + if rng.below(8) == 0 { + for j in [i, cells - 1 - i] { + g.rows[j / side] &= !(1 << (j % side)); + } + } + } + if g.blacks() > 0 && g.is_nyt_valid() { + return g; + } + } + } +} + +/// A compiled puzzle: everything the runtime may read. +#[derive(Clone)] +pub struct Puzzle { + pub lang: Lang, + /// Per slot: its length. + pub len: Vec, + /// `[slot * STRIDE + offset]`: the crossing `(slot:offset)` tile, or `NONE`. + pub cross: Vec, + /// `[slot * STRIDE + offset]`: the Morton cell code. + pub cell: Vec, + /// `[cell]`: the (at most two) `(slot:offset)` tiles on a cell. + pub occupant: Vec<[u16; 2]>, +} + +impl Puzzle { + pub fn slots(&self) -> usize { + self.len.len() + } +} + +pub fn tile(s: usize, o: usize) -> u16 { + FacetTier { + hi: s as u8, + lo: o as u8, + } + .as_u16() +} + +pub fn untile(t: u16) -> (usize, usize) { + ((t >> 8) as usize, (t & 0xFF) as usize) +} + +/// Compile a grid: slots from the row and column masks, each position's cell +/// from `Morton8x8::checked_offset`, crossings from equal cell codes. +pub fn compile(grid: &Grid, lang: Lang) -> Puzzle { + let mut len = Vec::new(); + let mut cell = Vec::new(); + for (across, lines) in [(true, grid.rows.clone()), (false, grid.cols())] { + for (line, &w) in lines.iter().enumerate() { + let mut c = 0; + while c < grid.side { + if w >> c & 1 == 0 { + c += 1; + continue; + } + let start = c; + while c < grid.side && w >> c & 1 == 1 { + c += 1; + } + let l = c - start; + if l < 2 { + continue; + } + let (x, y) = if across { (start, line) } else { (line, start) }; + let (dx, dy) = if across { (1i8, 0i8) } else { (0, 1) }; + let origin = Morton8x8::from_xy(x as u8, y as u8); + let mut codes = [NONE; STRIDE]; + for (off, code) in codes.iter_mut().enumerate().take(l) { + *code = origin + .checked_offset(dx * off as i8, dy * off as i8) + .expect("on the board") + .code(); + } + len.push(l as u8); + cell.extend_from_slice(&codes); + } + } + } + assert!(len.len() < 255, "slot ids are one byte"); + let tiles = cell + .iter() + .filter(|&&c| c != NONE) + .max() + .map_or(0, |&m| m as usize + 1); + let mut occupant = vec![[NONE; 2]; tiles]; + let mut cross = vec![NONE; len.len() * STRIDE]; + for (s, &l) in len.iter().enumerate() { + for o in 0..l as usize { + let c = cell[s * STRIDE + o] as usize; + let slot = if occupant[c][0] == NONE { 0 } else { 1 }; + assert_eq!(occupant[c][slot], NONE, "at most two positions per cell"); + occupant[c][slot] = tile(s, o); + if slot == 1 { + let other = occupant[c][0]; + cross[s * STRIDE + o] = other; + let (t, j) = untile(other); + cross[t * STRIDE + j] = tile(s, o); + } + } + } + Puzzle { + lang, + len, + cross, + cell, + occupant, + } +} + +/// Why a solve did not start or did not finish. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Stop { + /// The puzzle and the populations are different languages. + Language, + /// A slot was left with no candidate, or two letters disagree on a cell. + Contradiction(u8), +} + +/// Per-slot candidate masks plus the placed word, shared by arms A and D. +#[derive(Clone)] +pub struct State { + pub blocks: usize, + pub cand: Vec, + pub placed: Vec, + pub ands: usize, +} + +impl State { + pub fn new(hot: &Hot, puz: &Puzzle) -> Result { + if hot.lang != puz.lang { + return Err(Stop::Language); + } + let b = hot.blocks; + let mut cand = vec![0u64; puz.slots() * b]; + for (s, &l) in puz.len.iter().enumerate() { + cand[s * b..(s + 1) * b].copy_from_slice(&hot.all[l as usize]); + } + Ok(Self { + blocks: b, + cand, + placed: vec![UNSET; puz.slots()], + ands: 0, + }) + } + + pub fn mask(&self, s: usize) -> &[u64] { + &self.cand[s * self.blocks..(s + 1) * self.blocks] + } + + pub fn count(&self, s: usize) -> u64 { + popcount_batch_u64(self.mask(s)) + } + + pub fn place(&mut self, s: usize, w: WordId) { + let b = self.blocks; + self.cand[s * b..(s + 1) * b].fill(0); + self.cand[s * b + w as usize / 64] = 1 << (w % 64); + self.placed[s] = w; + } + + /// Promote every unplaced slot already down to one candidate. + pub fn seed(&mut self, queue: &mut Vec) -> Result<(), Stop> { + for s in 0..self.placed.len() { + if self.placed[s] == UNSET { + match self.count(s) { + 0 => return Err(Stop::Contradiction(s as u8)), + 1 => { + let w = first_bit(self.mask(s)); + self.place(s, w); + queue.push(s as u8); + } + _ => {} + } + } + } + Ok(()) + } + + /// AND one population into slot `t`; promote at popcount 1. + pub fn narrow(&mut self, t: usize, p: &[u64], queue: &mut Vec) -> Result<(), Stop> { + let b = self.blocks; + let m = &mut self.cand[t * b..(t + 1) * b]; + mask_and_assign(m, p); + self.ands += 1; + match popcount_batch_u64(m) { + 0 => Err(Stop::Contradiction(t as u8)), + 1 => { + self.placed[t] = first_bit(m); + queue.push(t as u8); + Ok(()) + } + _ => Ok(()), + } + } +} + +pub fn first_bit(m: &[u64]) -> WordId { + let (b, w) = m + .iter() + .enumerate() + .find(|(_, &w)| w != 0) + .expect("non-empty mask"); + (b * 64 + w.trailing_zeros() as usize) as WordId +} + +pub fn bits(m: &[u64]) -> Vec { + let mut out = Vec::new(); + for (b, &word) in m.iter().enumerate() { + let mut w = word; + while w != 0 { + out.push((b * 64 + w.trailing_zeros() as usize) as WordId); + w &= w - 1; + } + } + out +} + +/// Arm A's fixed point: each queued slot hands its letter at every crossed +/// offset to the crossing `(slot:offset)` read from `cross`. +pub fn propagate_token( + hot: &Hot, + puz: &Puzzle, + st: &mut State, + queue: &mut Vec, +) -> Result<(), Stop> { + while let Some(s) = queue.pop() { + let s = s as usize; + let w = st.placed[s]; + for off in 0..puz.len[s] as usize { + let c = puz.cross[s * STRIDE + off]; + if c == NONE { + continue; + } + let (t, j) = untile(c); + let x = hot.letter(w, off); + if st.placed[t] != UNSET { + if hot.letter(st.placed[t], j) != x { + return Err(Stop::Contradiction(t as u8)); + } + continue; + } + st.narrow(t, hot.pop(puz.len[t] as usize, j, x), queue)?; + } + } + Ok(()) +} + +pub fn start(hot: &Hot, puz: &Puzzle, givens: &[(u8, WordId)]) -> Result<(State, Vec), Stop> { + let mut st = State::new(hot, puz)?; + let mut queue = Vec::new(); + for &(s, w) in givens { + st.place(s as usize, w); + queue.push(s); + } + st.seed(&mut queue)?; + Ok((st, queue)) +} + +// ─────────────────────────────── creation ─────────────────────────────── + +/// Why creation gave up on one attempt. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Miss { + /// No complete fill found within the budget. + Fill, + /// A fill was found, but uniqueness could not be decided within the budget. + Unique, +} + +/// Crossword fills never repeat an entry: no two placed slots hold the same +/// word. Both creation searches apply this rule, so the generator and the +/// uniqueness count agree on what a fill is. +pub fn no_repeats(placed: &[WordId]) -> bool { + let mut seen: Vec = placed.iter().copied().filter(|&w| w != UNSET).collect(); + let n = seen.len(); + seen.sort_unstable(); + seen.dedup(); + seen.len() == n +} + +/// Depth-first fill over arm A's masks, most-constrained slot first, random +/// order within it. `budget` counts tried words. Creation only. +pub fn random_fill( + hot: &Hot, + puz: &Puzzle, + rng: &mut Rng, + budget: &mut usize, +) -> Option> { + pub fn go( + hot: &Hot, + puz: &Puzzle, + st: State, + rng: &mut Rng, + budget: &mut usize, + ) -> Option { + let Some(s) = (0..puz.slots()) + .filter(|&s| st.placed[s] == UNSET) + .min_by_key(|&s| st.count(s)) + else { + return no_repeats(&st.placed).then_some(st); + }; + let mut cs = bits(st.mask(s)); + cs.retain(|w| !st.placed.contains(w)); + rng.shuffle(&mut cs); + for w in cs { + if *budget == 0 { + return None; + } + *budget -= 1; + let mut next = st.clone(); + next.place(s, w); + let mut q = vec![s as u8]; + if propagate_token(hot, puz, &mut next, &mut q).is_ok() && no_repeats(&next.placed) { + if let Some(done) = go(hot, puz, next, rng, budget) { + return Some(done); + } + } + } + None + } + let (mut st, mut q) = start(hot, puz, &[]).ok()?; + propagate_token(hot, puz, &mut st, &mut q).ok()?; + go(hot, puz, st, rng, budget).map(|st| st.placed) +} + +/// Complete fills consistent with `givens`, up to `cap`; `None` when the +/// budget runs out. The uniqueness oracle for creation. +pub fn count_fills( + hot: &Hot, + puz: &Puzzle, + givens: &[(u8, WordId)], + cap: usize, + budget: &mut usize, +) -> Option { + pub fn go(hot: &Hot, puz: &Puzzle, st: State, cap: usize, budget: &mut usize) -> Option { + let Some(s) = (0..puz.slots()) + .filter(|&s| st.placed[s] == UNSET) + .min_by_key(|&s| st.count(s)) + else { + return Some(usize::from(no_repeats(&st.placed))); + }; + let mut n = 0; + for w in bits(st.mask(s)) + .into_iter() + .filter(|w| !st.placed.contains(w)) + { + if *budget == 0 { + return None; + } + *budget -= 1; + let mut next = st.clone(); + next.place(s, w); + let mut q = vec![s as u8]; + if propagate_token(hot, puz, &mut next, &mut q).is_ok() && no_repeats(&next.placed) { + n += go(hot, puz, next, cap - n, budget)?; + if n >= cap { + break; + } + } + } + Some(n) + } + let Ok((mut st, mut q)) = start(hot, puz, givens) else { + return Some(0); + }; + if propagate_token(hot, puz, &mut st, &mut q).is_err() { + return Some(0); + } + go(hot, puz, st, cap, budget) +} + +/// A created puzzle: its compiled layout, its solution and the givens that +/// make the solution unique. +pub struct Created { + pub puz: Puzzle, + pub solution: Vec, + pub givens: Vec<(u8, WordId)>, + pub compile_ns: f64, +} + +/// Grid, compile, random fill, then givens in random order until unique. +pub fn create(hot: &Hot, side: usize, rng: &mut Rng, budget: usize) -> Result { + let grid = Grid::random_nyt(side, rng); + let t = Instant::now(); + let puz = compile(&grid, hot.lang); + let compile_ns = t.elapsed().as_nanos() as f64; + let mut b = budget; + let solution = random_fill(hot, &puz, rng, &mut b).ok_or(Miss::Fill)?; + let mut order: Vec = (0..puz.slots() as u8).collect(); + rng.shuffle(&mut order); + let mut givens = Vec::new(); + for s in order { + givens.push((s, solution[s as usize])); + let mut b = budget; + if count_fills(hot, &puz, &givens, 2, &mut b).ok_or(Miss::Unique)? == 1 { + return Ok(Created { + puz, + solution, + givens, + compile_ns, + }); + } + } + unreachable!("every slot given is unique") +} From d9fa765b8bfe2dde5159f6048ff50d3b756b80e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 23:21:15 +0000 Subject: [PATCH 2/4] crossword attention probe: dom/wdeg falsifier compares the search path only Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .../examples/crossword_attention_probe.rs | 73 ++++++++++++------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/crates/cognitive-shader-driver/examples/crossword_attention_probe.rs b/crates/cognitive-shader-driver/examples/crossword_attention_probe.rs index 3050f459c..6604ac7c2 100644 --- a/crates/cognitive-shader-driver/examples/crossword_attention_probe.rs +++ b/crates/cognitive-shader-driver/examples/crossword_attention_probe.rs @@ -572,41 +572,58 @@ mod tests { } } - /// DomWdeg's weights only grow from contradictions, and a search with - /// none behaves exactly like popcount. + /// The search itself, without the popcount-only disagreement counters. + fn path(s: &Stats) -> (u64, u64, u64, Option>, Vec) { + ( + s.nodes, + s.contradictions, + s.fills, + s.first.clone(), + s.fail_trace.clone(), + ) + } + + /// DomWdeg's weights only grow from contradictions: a search with none + /// is exactly the popcount search, and one with contradictions goes + /// somewhere else. #[test] fn dom_wdeg_is_popcount_until_something_fails() { let b = bench_small(); - let c = &b.created[0]; - // From the full givens propagation finishes the puzzle: no choice, - // no contradiction, and the two policies are identical. - let pc = search( - &b.hot, - &c.puz, - &b.prior, - &c.givens, - Policy::Popcount, - BUDGET, - 2, - ); - let dw = search( - &b.hot, - &c.puz, - &b.prior, - &c.givens, - Policy::DomWdeg, - BUDGET, - 2, - ); - if pc.contradictions == 0 { - assert_eq!(pc, dw); + let mut silent = 0; + for c in &b.created { + let pc = search( + &b.hot, + &c.puz, + &b.prior, + &c.givens, + Policy::Popcount, + BUDGET, + 2, + ); + let dw = search( + &b.hot, + &c.puz, + &b.prior, + &c.givens, + Policy::DomWdeg, + BUDGET, + 2, + ); + if pc.contradictions == 0 { + assert_eq!(path(&pc), path(&dw)); + silent += 1; + } } - // Can fire: on empty grids contradictions happen and the weights - // move the search somewhere else. + assert!( + silent > 0, + "the silent half needs a search without contradictions" + ); let (_, a) = run_work(b, Work::Fill, Policy::Popcount, BUDGET); let (_, d) = run_work(b, Work::Fill, Policy::DomWdeg, BUDGET); assert!(a.iter().map(|s| s.contradictions).sum::() > 0); - assert_ne!(a, d); + let pa: Vec<_> = a.iter().map(path).collect(); + let pd: Vec<_> = d.iter().map(path).collect(); + assert_ne!(pa, pd); } /// Same puzzles, same seed, same policy: the same search. From fbe7c56ecc6b2f2fa4d9afb8e47dfa02c15cdce4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 23:33:24 +0000 Subject: [PATCH 3/4] Board: D-PUZZLE-ATTN-0 entry and status row Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .claude/board/STATUS_BOARD.md | 1 + ...zle-attn-0-entropy-attention-crosswords.md | 59 +++++++++++++++++++ .claude/board/entries/README.md | 3 +- 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-07-puzzle-attn-0-entropy-attention-crosswords.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 1912ce467..c5bd1d300 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -48,6 +48,7 @@ Plan: `.claude/plans/2026-10-06-global-sudoku-replayable-orchestration-v1.md`. O | **D-EPI-CANON-0** | Canonical joint `EpistemicState5` reading of bits 59..63 in the contract | Shipped (#1381) | `contract::epistemic_state5`: Cartesian `Topology2 × Certification3`, `raw5 = t \| c << 2`, 24 meaningful / 8 reserved; per-factor facts; declarations + `project_state5`; `CausalEdge64::{epistemic_raw5, with_epistemic_raw5}`; non-declared-reading writers deprecated | | **D-EPI-POP-0** | Facts → 32-state population: the missing operator between one edge's coordinate (u5) and a semantic condition over all coordinates (u32) | Shipped (#1381) | `contract::epistemic_state5::{Population, facts_population, EpistemicState5::bit}`: `const fn`, no DSL / query object / planner. Tests pin exact sets (`CAUSES` = {20..23}, `IND_UNKNOWN \| RELATED` = {10,14,18,22}, `IND_KNOWN \| SUPPORTS` = {13,17,21}, `RELATED` = 8..23), operator == state enumeration over all 2048 requirements with no reserved code, conjunction == intersection; 2 disable runs red (any-of semantics; reserved read as fact-less) | | **D-EPI-LEGACY-DEPROJECT-0** | Deproject the remaining `ReasoningBand` / CE64-`TrustTexture` consumers: each goes to an exact coordinate, a `facts_population(..)`, MUL, or another subsystem (e.g. counterfactual context) — never a 1:1 replacement enum | Shipped (#1381) | `entries/2026-10-07-epi-legacy-deprojection.md`: 6 ReasoningBand caller sites (1 certification question, 1 non-interference witness → both exact coordinates; 2 context/obsolete → compat; 2 refusal witnesses), 0 production CE64 `TrustTexture` consumers, 4 `TrustTexture` types with no bridge; `EdgeRole::band` removed; hinge + V3 lift migrated; 4 code loops → `facts_population` | +| **D-PUZZLE-ATTN-0** | Entropy as the focus of attention on crosswords (`cognitive-shader-driver/examples/crossword_attention_probe.rs`, shared `crossword_core.rs`) | In PR | popcount (uniform entropy) needs 14.5× fewer nodes than fixed order to prove uniqueness; frequency-weighted Shannon costs 5.9× more than popcount; dom/wdeg reaches first fills sooner (41k vs 92k on open puzzles); contradiction rate flat within a search (`entries/2026-10-07-puzzle-attn-0-entropy-attention-crosswords.md`) | | **D-CE64-CYCLE-0** | Cross-cycle executor experiments on the surprisal scheduler: forget arms, FIFO baseline, 350 ms windows, 1k..1M folds, attractors | In PR | the next cycle needs evidence table + register (Epi5) + attention state; H and bits 40..42 are derived/echo; pooled 3 seeds: no wake advantage over round-robin, 11% fewer folds; FIFO fits 1M folds in a 350 ms window at ~220 ns/fold, the heap scheduler ~440k at ~800 ns (`entries/2026-10-07-ce64-cycle-0-cross-cycle-executor.md`) | | **D-CE64-STAUNEN-0** | Surprisal scheduler over many CE64 basins across windows (`lance-graph-planner/examples/staunen_scheduler_probe.rs`) | Shipped (#1395); wake claim corrected by D-CE64-CYCLE-0 (pooled: no wake advantage) | naive raw surprise is captured by noise (resolved 0.649); habituation + learning progress fix it (0.952, wake 4.2 windows vs round-robin 5.1); round-robin still resolves slightly more (0.964); bits 50..52 left as plasticity (`learn` gates on them); 18 tests, 13 disable runs red (`entries/2026-10-07-ce64-staunen-0-surprisal-scheduler.md`) | | **D-CE64-LOOP-0** | Bounded self-orchestrating fold loop over one CE64 register (`lance-graph-planner/examples/self_orchestration_probe.rs`) + bits 40..63 census | Shipped (#1395) | combined policy reaches `IndirectKnown × Causes` with H ≤ 0.5 bit in 3 folds (round-robin 10, random median 14); ~830 ns/fold release; 15 tests in CI, 11 disable runs red; H and activation stay out of the register (`entries/2026-10-07-ce64-loop-0-self-orchestration.md`) | diff --git a/.claude/board/entries/2026-10-07-puzzle-attn-0-entropy-attention-crosswords.md b/.claude/board/entries/2026-10-07-puzzle-attn-0-entropy-attention-crosswords.md new file mode 100644 index 000000000..251df0e05 --- /dev/null +++ b/.claude/board/entries/2026-10-07-puzzle-attn-0-entropy-attention-crosswords.md @@ -0,0 +1,59 @@ +# 2026-10-07 — D-PUZZLE-ATTN-0: on crosswords, uniform entropy focuses search; frequency-weighted Shannon misdirects it + +**Status:** TEST-PINNED (`crates/cognitive-shader-driver/examples/crossword_attention_probe.rs`, `test = true`, 9 tests run by the shader-driver `cargo test` step). MEASURED (`cargo run --release -p cognitive-shader-driver --example crossword_attention_probe`; 20 created puzzles and 20 empty NYT grids, sides 5 and 7; 200,000 nodes per run). + +## DECISION + +- **SCOPE:** crossword solving is a fixed point (#1387), so deduction needs no attention. This probe changes only the search's choice of the next slot once deduction stalls. Everything else is held fixed: + - the puzzles; + - the propagation, moved unchanged into `examples/shared/crossword_core.rs` (#1387's 20 tests pass on it); + - the value order, most frequent word first. +- **Policies:** + - fixed order; + - seeded random; + - widest first (negative control); + - **popcount**: most-constrained first, i.e. entropy under a uniform prior, `log2(count)`; + - **Shannon**: `H = -Σ p log2 p` with `p ∝` COCA-All frequency; + - **dom/wdeg** (Boussemart et al. 2004): count divided by accumulated contradictions on the slot that ran dry, so earlier failures steer later choices. +- **Contradiction:** a slot with no candidate has no entropy under either reading and is never settled (pinned). + +## Measured + +| policy | prove: solved | prove: nodes | open: solved | open: nodes / to first | fill: solved | +|---|---|---|---|---|---| +| fixed | 16/20 | 852,535 | 7/20 | 2,858,517 / 2,488,660 | 5/20 | +| random | 15/20 | 1,049,801 | 5/20 | 3,151,741 / 2,572,628 | 0/20 | +| widest | 16/20 | 916,437 | 5/20 | 3,186,550 / 2,653,695 | 10/20 | +| **popcount** | **20/20** | **58,729** | **20/20** | 570,450 / 91,930 | 10/20 | +| Shannon | 19/20 | 348,578 | 16/20 | 1,156,793 / 550,562 | 10/20 | +| **dom/wdeg** | **20/20** | 53,892 | **20/20** | 539,520 / **40,938** | 10/20 | + +(prove = count fills to 2 on created puzzles; open = half the givens, count to 50; fill = an empty grid.) + +- **Entropy as focus of attention works, read as uniform entropy.** Popcount needs 14.5× fewer nodes than fixed order to prove uniqueness, and 5× fewer on the open puzzles, where fixed order finishes only 7 of 20. +- **Frequency-weighted Shannon is worse than popcount**: 5.9× the nodes on prove; on open, 2× the nodes and 4 puzzles unfinished. It picks a different slot on 36–50% of decisions. + - The frequency prior measures how *predictable* a slot is; a search pays for how many candidates it must *try*. + - So for attention in search, `log2(count)` is the right reading, not a proxy for something better. + - The `ms` column overstates popcount: its runs also compute Shannon on every decision to count disagreements. +- **Earlier contradictions do direct later choices.** dom/wdeg uses about the same total nodes as popcount but reaches the first fill sooner: 33,017 vs 41,029 on prove, 40,938 vs 91,930 on open. +- **The contradiction rate does not improve within a search.** Per tenth of the search it stays flat at 0.73–0.85 for both popcount and dom/wdeg. The gain shows up as an earlier solution, not a lower failure rate. +- **Empty grids:** 10 of 20 stay unfilled within 200,000 nodes under every good policy. This is likely the vocabulary rather than attention; not measured. + +## Gates + +7 disable runs, each red on its named falsifier: +- popcount picks the first slot +- Shannon reads popcount +- an empty mask has H = 0 +- a flat prior +- no failure weights +- no propagation +- a wall-clock seed + +One falsifier was vacuous on first write: the dom/wdeg "differs from popcount" check compared whole stats, which differ anyway through popcount-only counters. It now compares the search path, and its disable goes red. + +## OPEN + +- Shannon may belong in *value* ordering (which word to try first) rather than slot choice. Untested: the value order here is fixed by frequency. +- Coverage is narrow: one frequency prior (COCA academic), one vocabulary, sides 5 and 7 only. Side 9 takes minutes per created puzzle. +- No CE64 register in this loop. The slot state is the mask (the evidence); entropy is derived from it. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 569079672..dc364117d 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,12 +25,13 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -260 entries, 2026-08-06 .. 2026-10-07. +261 entries, 2026-08-06 .. 2026-10-07. | date | entry id | finding | file | |---|---|---|---| | 2026-10-07 | `tinker-janus-fold-harvest` | TinkerPop bulk = K (GroupReduce), ONE_BULK = S, GValue pinning = bundle invalidation; JanusGraph slice = OrderedLaneWitness→Range (30–38×); no new V4 op | [2026-10-07-tinker-janus-fold-harvest.md](2026-10-07-tinker-janus-fold-harvest.md) | | 2026-10-07 | `selector-seq-aware-coverage` | | [2026-10-07-selector-seq-aware-coverage.md](2026-10-07-selector-seq-aware-coverage.md) | +| 2026-10-07 | `D-PUZZLE-ATTN-0` | | [2026-10-07-puzzle-attn-0-entropy-attention-crosswords.md](2026-10-07-puzzle-attn-0-entropy-attention-crosswords.md) | | 2026-10-07 | `D-PUZZLE-0` | | [2026-10-07-puzzle-0-sudoku-population-fold.md](2026-10-07-puzzle-0-sudoku-population-fold.md) | | 2026-10-07 | `D-PUZZLE-0-XWORD` | | [2026-10-07-puzzle-0-crossword-same-algebra.md](2026-10-07-puzzle-0-crossword-same-algebra.md) | | 2026-10-07 | `D-PUZZLE-0-XWORD-REAL` | | [2026-10-07-puzzle-0-crossword-real-words.md](2026-10-07-puzzle-0-crossword-real-words.md) | From ce63247ca696dfc751e1134badc935942679d897 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 23:36:27 +0000 Subject: [PATCH 4/4] Board: #1397 merge hygiene (PR arc, latest state, D-CE64-CYCLE-0 shipped) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .claude/board/LATEST_STATE.md | 5 +++++ .claude/board/PR_ARC_INVENTORY.md | 7 +++++++ .claude/board/STATUS_BOARD.md | 2 +- .claude/board/entries/README.md | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 4e141305f..ea926f080 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,8 @@ +## 2026-10-07 — lance-graph #1397 MERGED (`4a0a72c`); D-PUZZLE-ATTN-0 on branch + +- #1397: D-CE64-CYCLE-0, probe-only (no contract change). +- D-PUZZLE-ATTN-0 (branch `claude/epistemicstate5-ce64-bits-3oe0dq`, unmerged): the crossword core moves to `cognitive-shader-driver/examples/shared/crossword_core.rs`; `crossword_attention_probe.rs` compares slot policies. No contract change. + ## 2026-10-07 — lance-graph #1395 MERGED (`5658c10`); D-CE64-CYCLE-0 on branch - #1395: D-CE64-LOOP-0 and D-CE64-STAUNEN-0, probes in `lance-graph-planner/examples` (no contract change). diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 922d983f0..26445cac3 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -1,3 +1,10 @@ +## 2026-10-07 — lance-graph PR #1397 (merged `4a0a72c`, branch `claude/epistemicstate5-ce64-bits-3oe0dq`, head `ea16f16`, 3 commits) — D-CE64-CYCLE-0: what a CE64 register must carry across cycles + +- **Added:** `staunen_scheduler_probe.rs` gains `run_with(opts)`, a lazy-stamp heap scheduler, a FIFO worklist baseline, wall-clock windows with continuation, `Forget` arms, the #1390 Simpson + trial population, attractor and quadrant readings, dense 1k..1M windows; comparative tests pool 3 seeds. +- **Locked (test-pinned):** the next cycle needs the evidence table, the register (Epi5) and the attention state; H and bits 40..42 are derived / echo; no wake advantage over round-robin when pooled (corrects #1395). 23 tests, 19 disable runs red. +- **Deferred / open:** the heap scheduler costs ~3.5× FIFO per fold; Moore/EWA/Sudoku folds not in the loop; where cross-cycle state lives in the register. +- **Confidence:** medium — pooled over 3 seeds, one world model. + ## 2026-10-07 — lance-graph PR #1395 (merged `5658c10`, branch `claude/epistemicstate5-ce64-bits-3oe0dq`, head `086189a`, 5 commits) — D-CE64-LOOP-0 + D-CE64-STAUNEN-0: self-orchestrating fold loop, surprisal scheduler, bits 40..63 census - **Added:** `lance-graph-planner/examples/self_orchestration_probe.rs` (one register, five policies, 15 tests) and `staunen_scheduler_probe.rs` (many basins across windows, surprisal distinct from Shannon H, 18 tests), both `test = true`; the bits 40..63 reader/writer census (board entry). diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index c5bd1d300..df0edeba3 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -49,7 +49,7 @@ Plan: `.claude/plans/2026-10-06-global-sudoku-replayable-orchestration-v1.md`. O | **D-EPI-POP-0** | Facts → 32-state population: the missing operator between one edge's coordinate (u5) and a semantic condition over all coordinates (u32) | Shipped (#1381) | `contract::epistemic_state5::{Population, facts_population, EpistemicState5::bit}`: `const fn`, no DSL / query object / planner. Tests pin exact sets (`CAUSES` = {20..23}, `IND_UNKNOWN \| RELATED` = {10,14,18,22}, `IND_KNOWN \| SUPPORTS` = {13,17,21}, `RELATED` = 8..23), operator == state enumeration over all 2048 requirements with no reserved code, conjunction == intersection; 2 disable runs red (any-of semantics; reserved read as fact-less) | | **D-EPI-LEGACY-DEPROJECT-0** | Deproject the remaining `ReasoningBand` / CE64-`TrustTexture` consumers: each goes to an exact coordinate, a `facts_population(..)`, MUL, or another subsystem (e.g. counterfactual context) — never a 1:1 replacement enum | Shipped (#1381) | `entries/2026-10-07-epi-legacy-deprojection.md`: 6 ReasoningBand caller sites (1 certification question, 1 non-interference witness → both exact coordinates; 2 context/obsolete → compat; 2 refusal witnesses), 0 production CE64 `TrustTexture` consumers, 4 `TrustTexture` types with no bridge; `EdgeRole::band` removed; hinge + V3 lift migrated; 4 code loops → `facts_population` | | **D-PUZZLE-ATTN-0** | Entropy as the focus of attention on crosswords (`cognitive-shader-driver/examples/crossword_attention_probe.rs`, shared `crossword_core.rs`) | In PR | popcount (uniform entropy) needs 14.5× fewer nodes than fixed order to prove uniqueness; frequency-weighted Shannon costs 5.9× more than popcount; dom/wdeg reaches first fills sooner (41k vs 92k on open puzzles); contradiction rate flat within a search (`entries/2026-10-07-puzzle-attn-0-entropy-attention-crosswords.md`) | -| **D-CE64-CYCLE-0** | Cross-cycle executor experiments on the surprisal scheduler: forget arms, FIFO baseline, 350 ms windows, 1k..1M folds, attractors | In PR | the next cycle needs evidence table + register (Epi5) + attention state; H and bits 40..42 are derived/echo; pooled 3 seeds: no wake advantage over round-robin, 11% fewer folds; FIFO fits 1M folds in a 350 ms window at ~220 ns/fold, the heap scheduler ~440k at ~800 ns (`entries/2026-10-07-ce64-cycle-0-cross-cycle-executor.md`) | +| **D-CE64-CYCLE-0** | Cross-cycle executor experiments on the surprisal scheduler: forget arms, FIFO baseline, 350 ms windows, 1k..1M folds, attractors | Shipped (#1397) | the next cycle needs evidence table + register (Epi5) + attention state; H and bits 40..42 are derived/echo; pooled 3 seeds: no wake advantage over round-robin, 11% fewer folds; FIFO fits 1M folds in a 350 ms window at ~220 ns/fold, the heap scheduler ~440k at ~800 ns (`entries/2026-10-07-ce64-cycle-0-cross-cycle-executor.md`) | | **D-CE64-STAUNEN-0** | Surprisal scheduler over many CE64 basins across windows (`lance-graph-planner/examples/staunen_scheduler_probe.rs`) | Shipped (#1395); wake claim corrected by D-CE64-CYCLE-0 (pooled: no wake advantage) | naive raw surprise is captured by noise (resolved 0.649); habituation + learning progress fix it (0.952, wake 4.2 windows vs round-robin 5.1); round-robin still resolves slightly more (0.964); bits 50..52 left as plasticity (`learn` gates on them); 18 tests, 13 disable runs red (`entries/2026-10-07-ce64-staunen-0-surprisal-scheduler.md`) | | **D-CE64-LOOP-0** | Bounded self-orchestrating fold loop over one CE64 register (`lance-graph-planner/examples/self_orchestration_probe.rs`) + bits 40..63 census | Shipped (#1395) | combined policy reaches `IndirectKnown × Causes` with H ≤ 0.5 bit in 3 folds (round-robin 10, random median 14); ~830 ns/fold release; 15 tests in CI, 11 disable runs red; H and activation stay out of the register (`entries/2026-10-07-ce64-loop-0-self-orchestration.md`) | | **D-PEARL-PROD-0** | Pearl's ladder in production: `lance_graph_planner::pearl::{reason, revise, hydrate}` over `lance_graph_contract::certification` | Shipped (#1391) | the P7a folds become `contract::certification::{CertificationModel, ModelBuilder}`; the dispatch, write-back and topology gate become `planner::pearl`, taking the caller's declared reading. The 14 falsifier tests run in the planner suite (CI); 12 disable runs red against the production module. No runtime caller yet: nothing in production produces sealed populations or chains for it (`entries/2026-10-07-pearl-prod-0-production-wiring.md`) | diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index dc364117d..683dc2a2c 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -261 entries, 2026-08-06 .. 2026-10-07. +262 entries, 2026-08-06 .. 2026-10-07. | date | entry id | finding | file | |---|---|---|---|