From 4ed5e4ea3f9d490363c168ca42690bf0c4c59a1e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:12:58 +0000 Subject: [PATCH 1/9] D-PUZZLE-0: move the domain-free fold into shared/population_fold.rs The propagation reading, the five questions, the three folds, the group fold and the timing move out of the Sudoku probe so a second domain can use them unchanged. A fence test fails if domain vocabulary appears in the shared half. The Sudoku probe's counts are identical before and after (711,990 / 263,700 / 227,189 / 221,101 / 288,076 on 1,000,066 edges). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .../examples/shared/population_fold.rs | 321 ++++++++++++++ .../examples/sudoku_population_fold_probe.rs | 392 ++++-------------- 2 files changed, 391 insertions(+), 322 deletions(-) create mode 100644 crates/cognitive-shader-driver/examples/shared/population_fold.rs diff --git a/crates/cognitive-shader-driver/examples/shared/population_fold.rs b/crates/cognitive-shader-driver/examples/shared/population_fold.rs new file mode 100644 index 000000000..b8ad78536 --- /dev/null +++ b/crates/cognitive-shader-driver/examples/shared/population_fold.rs @@ -0,0 +1,321 @@ +//! D-PUZZLE-0: the domain-free half of every puzzle probe. +//! +//! One edge carries a coordinate (`raw5`, CE64 bits 59..63). One question is +//! a population, a `u32` over the 32 codes (`facts_population`). Many edges +//! are answered by a fold over the edge words. The class declaration is +//! checked once per lane, never per edge. +//! +//! What a domain supplies: its claims, its law (which claims survive, which are +//! forced), and its own counters. What lives here: the propagation reading, the +//! questions, the folds and the oracle. The fence test at the bottom fails if +//! domain vocabulary appears above it: a domain needing code here would be +//! evidence against "same algebra". +//! +//! # The propagation reading (probe-local, not canon) +//! +//! | a claim at the snapshot | `Topology2 × Certification3` | raw5 | +//! |---|---|---| +//! | given | `Direct × Causes` | 20 | +//! | forced by propagation | `IndirectKnown × Causes` | 21 | +//! | true, but not yet forced | `IndirectUnknown × Causes` | 22 | +//! | any other surviving candidate | `Direct × Associated` | 4 | +//! +//! "True, not yet forced" needs a unique solution, which each domain checks. +//! Refuted claims are absent: the 5-bit state has no "refuted" coordinate. + +#![allow(dead_code)] + +use std::hint::black_box; +use std::time::Instant; + +use causal_edge::edge::CausalEdge64; +use causal_edge::layout::EPISTEMIC_SHIFT; +use lance_graph_contract::band_reading::EdgeProvenance; +use lance_graph_contract::class_view::ClassId; +use lance_graph_contract::epistemic_state5::fact::{ + ASSOCIATED, CAUSES, DIRECT, IND_KNOWN, IND_UNKNOWN, RELATED, +}; +use lance_graph_contract::epistemic_state5::{ + facts_population, Certification3, Epi5Declarations, Epi5Gen, Epi5Reading, EpistemicState5, + Population, Topology2, +}; +use lance_graph_contract::rail_geometry::RailAxis; + +pub const RAIL: RailAxis = RailAxis::Taxonomy; + +const fn state(t: Topology2, c: Certification3) -> EpistemicState5 { + EpistemicState5::new(Epi5Gen::V1, t, c) +} +pub const GIVEN: EpistemicState5 = state(Topology2::Direct, Certification3::Causes); +pub const FORCED: EpistemicState5 = state(Topology2::IndirectKnown, Certification3::Causes); +pub const ENTAILED: EpistemicState5 = state(Topology2::IndirectUnknown, Certification3::Causes); +pub const CANDIDATE: EpistemicState5 = state(Topology2::Direct, Certification3::Associated); +/// Meaningful in general, never emitted by this reading. +pub const UNKNOWN_CAUSES: EpistemicState5 = state(Topology2::Unknown, Certification3::Causes); + +/// The four states in emission order; `Lane::expected` is indexed alike. +pub const STATES: [EpistemicState5; 4] = [GIVEN, FORCED, ENTAILED, CANDIDATE]; + +/// A question: its population (fast path) and the same question asked of one +/// decoded state (oracle). Both are written from facts, never from codes. +pub struct Query { + pub name: &'static str, + pub population: Population, + pub asks: fn(EpistemicState5) -> bool, +} + +pub fn queries() -> [Query; 5] { + [ + Query { + name: "every asserted claim (any topology, Causes)", + population: facts_population(CAUSES), + asks: |s| s.asserts(CAUSES), + }, + Query { + name: "given (Direct x Causes)", + population: facts_population(DIRECT | CAUSES), + asks: |s| s.asserts(DIRECT | CAUSES), + }, + Query { + name: "forced, chain known (IndirectKnown x Causes)", + population: facts_population(IND_KNOWN | CAUSES), + asks: |s| s.asserts(IND_KNOWN | CAUSES), + }, + Query { + name: "entailed, not yet forced (IndirectUnknown x Causes)", + population: facts_population(IND_UNKNOWN | CAUSES), + asks: |s| s.asserts(IND_UNKNOWN | CAUSES), + }, + Query { + name: "live candidates (Direct, Associated, not Related)", + population: facts_population(DIRECT | ASSOCIATED) & !facts_population(RELATED), + asks: |s| s.asserts(DIRECT | ASSOCIATED) && !s.asserts(RELATED), + }, + ] +} + +/// Edges, the group each belongs to, and the domain's own counts per state +/// (`expected[i]` counts `STATES[i]`), kept apart from the edges so the folds +/// have something independent to agree with. +#[derive(Default)] +pub struct Lane { + pub edges: Vec, + pub group_of: Vec, + pub groups: u32, + pub expected: [usize; 4], +} + +impl Lane { + pub fn push(&mut self, s: EpistemicState5) { + self.edges + .push(CausalEdge64::ZERO.with_epistemic_raw5(s.raw())); + self.group_of.push(self.groups); + } + + pub fn close_group(&mut self) { + self.groups += 1; + } + + /// The domain's count for question `q` of `queries()`: question 0 is the + /// union of the three asserted states, 1..=4 are the states themselves. + pub fn expected_for(&self, q: usize) -> usize { + match q { + 0 => self.expected[0] + self.expected[1] + self.expected[2], + q => self.expected[q - 1], + } + } +} + +/// The declaration gate, checked once per lane: a lane whose class does not +/// declare the canonical reading is refused before any edge is read. +pub fn admit(decl: &Epi5Declarations, class: ClassId) -> Option { + decl.get(class, RAIL) +} + +pub fn declarations(class: ClassId) -> Epi5Declarations { + let mut d = Epi5Declarations::new(); + d.declare(class, RAIL, Epi5Reading::default()); + d +} + +pub fn raw5(e: CausalEdge64) -> u32 { + (e.0 >> EPISTEMIC_SHIFT) as u32 +} + +/// Fast path, one question: a shift and a mask per edge. +pub fn count_in(edges: &[CausalEdge64], population: Population) -> usize { + edges + .iter() + .filter(|&&e| population >> raw5(e) & 1 == 1) + .count() +} + +/// Fast path, every question at once: one pass builds a 32-bin histogram; a +/// population's count is the sum of its bins. +pub fn histogram(edges: &[CausalEdge64]) -> [usize; 32] { + let mut h = [0usize; 32]; + for &e in edges { + h[raw5(e) as usize] += 1; + } + h +} + +pub fn count_from(h: &[usize; 32], population: Population) -> usize { + (0..32) + .filter(|&r| population >> r & 1 == 1) + .map(|r| h[r]) + .sum() +} + +/// Oracle: decode every edge through the declaration, then ask the state. +pub fn count_decoded( + edges: &[CausalEdge64], + decl: &Epi5Declarations, + class: ClassId, + asks: fn(EpistemicState5) -> bool, +) -> usize { + edges + .iter() + .filter(|e| { + let s = decl + .project_state5( + class, + RAIL, + Epi5Gen::V1, + e.epistemic_raw5(), + EdgeProvenance::V2Stamped, + ) + .expect("every edge in a declared lane decodes"); + asks(s) + }) + .count() +} + +/// Group fold: one question per group. +pub fn per_group(lane: &Lane, population: Population) -> Vec { + let mut out = vec![0u32; lane.groups as usize]; + for (&e, &g) in lane.edges.iter().zip(&lane.group_of) { + out[g as usize] += population >> raw5(e) & 1; + } + out +} + +/// Every question three ways (filter, histogram, decode) against the domain's +/// counts. Panics on the first disagreement; returns the counts. +pub fn check_three_ways(lane: &Lane, decl: &Epi5Declarations, class: ClassId) -> [usize; 5] { + let h = histogram(&lane.edges); + let mut out = [0usize; 5]; + for (i, q) in queries().iter().enumerate() { + let want = lane.expected_for(i); + assert_eq!( + count_in(&lane.edges, q.population), + want, + "filter: {}", + q.name + ); + assert_eq!(count_from(&h, q.population), want, "histogram: {}", q.name); + assert_eq!( + count_decoded(&lane.edges, decl, class, q.asks), + want, + "decode: {}", + q.name + ); + out[i] = want; + } + out +} + +pub fn median_ns_per_edge(edges: usize, mut f: impl FnMut() -> usize) -> f64 { + let mut times: Vec = (0..7) + .map(|_| { + let t = Instant::now(); + black_box(f()); + t.elapsed().as_nanos() as f64 / edges as f64 + }) + .collect(); + times.sort_by(f64::total_cmp); + times[times.len() / 2] +} + +/// Print the counts and the three timed paths for one lane. +pub fn report(lane: &Lane, decl: &Epi5Declarations, class: ClassId, counts: &[usize; 5]) { + for (q, n) in queries().iter().zip(counts) { + println!(" {:<56} pop {:#010x} {:>8}", q.name, q.population, n); + } + println!( + " build: debug_assertions={} avx2={} avx512f={}", + cfg!(debug_assertions), + cfg!(target_feature = "avx2"), + cfg!(target_feature = "avx512f") + ); + let n = lane.edges.len(); + let pop = facts_population(IND_UNKNOWN | CAUSES); + let filter = median_ns_per_edge(n, || count_in(black_box(&lane.edges), pop)); + let hist = median_ns_per_edge(n, || histogram(black_box(&lane.edges))[22]); + let decoded = median_ns_per_edge(n, || { + count_decoded(black_box(&lane.edges), decl, class, |s| { + s.asserts(IND_UNKNOWN | CAUSES) + }) + }); + println!(" median of 7, ns per edge:"); + println!(" one population, filter fold {filter:.3}"); + println!(" 32-bin histogram (every question) {hist:.3}"); + println!(" per-edge decode + asserts (oracle) {decoded:.3}"); +} + +/// SplitMix64: deterministic, seedable. +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + pub fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + pub fn shuffle(&mut self, xs: &mut [T]) { + for i in (1..xs.len()).rev() { + xs.swap(i, self.below(i as u64 + 1) as usize); + } + } +} + +#[cfg(test)] +pub mod fence { + /// The shared half carries no domain vocabulary. Checked on the source + /// above this module, so the list below does not match itself. + #[test] + fn the_shared_fold_names_no_domain() { + let src = include_str!("population_fold.rs"); + let shared = &src[..src.find("#[cfg(test)]\npub mod fence").unwrap()]; + let lower = shared.to_lowercase(); + for banned in [ + "sudoku", + "crossword", + "chess", + "digit", + "slot", + "cell", + "letter", + "grid", + "board", + "move", + ] { + assert!(!lower.contains(banned), "shared fold mentions `{banned}`"); + } + } + + /// The four states are the stated codes, and the five questions are + /// exactly the stated code sets. + #[test] + fn the_reading_and_the_questions_are_the_stated_codes() { + use super::*; + let raws: Vec = STATES.iter().map(|s| s.raw()).collect(); + assert_eq!(raws, [20, 21, 22, 4]); + let pops: Vec = queries().iter().map(|q| q.population).collect(); + assert_eq!(pops, [0xF0_0000, 1 << 20, 1 << 21, 1 << 22, 1 << 4]); + } +} diff --git a/crates/cognitive-shader-driver/examples/sudoku_population_fold_probe.rs b/crates/cognitive-shader-driver/examples/sudoku_population_fold_probe.rs index a2fb3b4c7..be4999678 100644 --- a/crates/cognitive-shader-driver/examples/sudoku_population_fold_probe.rs +++ b/crates/cognitive-shader-driver/examples/sudoku_population_fold_probe.rs @@ -1,12 +1,10 @@ //! D-PUZZLE-0, step 1: Sudoku at scale on the `EpistemicState5` population //! algebra. //! -//! One edge carries a coordinate (`raw5`, bits 59..63). One semantic question -//! is a population: a `u32` over all 32 codes -//! (`epistemic_state5::facts_population`). Many edges are answered by a fold -//! over the edge words — the class declaration is checked ONCE per lane, never -//! per edge. This probe measures that shape on about a million edges and -//! checks every count three ways. +//! The domain-free half (the propagation reading, the five questions, the +//! folds, the oracle) is `shared/population_fold.rs`, shared with the +//! crossword probe. This file holds only Sudoku: its claims, its law (naked +//! singles) and its own counters. //! //! # The corpus //! @@ -19,23 +17,13 @@ //! copy's solve independently of the propagation order. Each copy is //! snapshotted after `k` naked-single placements, `k` varying across copies. //! -//! # The reading (a probe-local declaration, not canon) +//! # The claims //! -//! Claim = "cell c holds digit d". One edge per claim not yet eliminated: -//! -//! | claim at the snapshot | `Topology2 × Certification3` | raw5 | -//! |---|---|---| -//! | a given | `Direct × Causes` | 20 | -//! | placed by a naked single | `IndirectKnown × Causes` | 21 | -//! | the true digit of a cell not yet placed | `IndirectUnknown × Causes` | 22 | -//! | any other surviving candidate | `Direct × Associated` | 4 | -//! -//! The third row is the case the Cartesian layout was built to keep: the value -//! is entailed (the puzzle has one solution, checked) but the chain that -//! derives it has not been produced yet. Eliminated claims are not edges: the -//! 5-bit state has no "refuted" coordinate, and inventing one would be a -//! semantic choice this probe does not make. The policy pins (which state each -//! row gets) belong to this probe's class only. +//! Claim = "cell c holds digit d", one edge per claim not yet eliminated, in +//! the shared propagation reading: a given clue is `GIVEN`, a naked-single +//! placement `FORCED`, the true digit of a cell not yet placed `ENTAILED` +//! (the puzzle has one solution, checked), every other surviving candidate +//! `CANDIDATE`. Class `0x0906` is this probe's own. //! //! # Not decided here //! @@ -47,25 +35,19 @@ //! Run: `cargo run --release -p cognitive-shader-driver --example sudoku_population_fold_probe` //! Tests: `cargo test -p cognitive-shader-driver --example sudoku_population_fold_probe` -use std::hint::black_box; use std::time::Instant; -use causal_edge::edge::CausalEdge64; -use causal_edge::layout::EPISTEMIC_SHIFT; -use lance_graph_contract::band_reading::EdgeProvenance; use lance_graph_contract::class_view::ClassId; -use lance_graph_contract::epistemic_state5::fact::{ - ASSOCIATED, CAUSES, DIRECT, IND_KNOWN, IND_UNKNOWN, RELATED, -}; -use lance_graph_contract::epistemic_state5::{ - facts_population, Certification3, Epi5Declarations, Epi5Gen, Epi5Reading, EpistemicState5, - Population, Topology2, + +#[path = "shared/population_fold.rs"] +mod population_fold; +use population_fold::{ + admit, check_three_ways, declarations, per_group, report, Lane, Rng, CANDIDATE, ENTAILED, + FORCED, GIVEN, }; -use lance_graph_contract::rail_geometry::RailAxis; /// The probe's own class (no production class declares a Sudoku reading). const SUDOKU_CLASS: ClassId = 0x0906; -const RAIL: RailAxis = RailAxis::Taxonomy; type Grid = [u8; 81]; @@ -82,59 +64,6 @@ const PUZZLE: Grid = [ 0, 0, 0, 0, 8, 0, 0, 7, 9, // ]; -// ── the four claim states ──────────────────────────────────────────────── - -const fn state(t: Topology2, c: Certification3) -> EpistemicState5 { - EpistemicState5::new(Epi5Gen::V1, t, c) -} -const GIVEN: EpistemicState5 = state(Topology2::Direct, Certification3::Causes); -const DERIVED: EpistemicState5 = state(Topology2::IndirectKnown, Certification3::Causes); -const ENTAILED: EpistemicState5 = state(Topology2::IndirectUnknown, Certification3::Causes); -const CANDIDATE: EpistemicState5 = state(Topology2::Direct, Certification3::Associated); - -// ── the questions, each one population ─────────────────────────────────── - -/// A question: its population (the fast path) and the same question asked of -/// one decoded state (the slow path). Both are written from the facts, never -/// from code numbers. -struct Query { - name: &'static str, - population: Population, - asks: fn(EpistemicState5) -> bool, -} - -fn queries() -> [Query; 5] { - [ - Query { - name: "every placed digit (any topology, Causes)", - population: facts_population(CAUSES), - asks: |s| s.asserts(CAUSES), - }, - Query { - name: "givens (Direct x Causes)", - population: facts_population(DIRECT | CAUSES), - asks: |s| s.asserts(DIRECT | CAUSES), - }, - Query { - name: "derived, chain known (IndirectKnown x Causes)", - population: facts_population(IND_KNOWN | CAUSES), - asks: |s| s.asserts(IND_KNOWN | CAUSES), - }, - Query { - name: "entailed, chain not yet derived (IndirectUnknown x Causes)", - population: facts_population(IND_UNKNOWN | CAUSES), - asks: |s| s.asserts(IND_UNKNOWN | CAUSES), - }, - Query { - name: "live candidates (Direct, Associated, not Related)", - population: facts_population(DIRECT | ASSOCIATED) & !facts_population(RELATED), - asks: |s| s.asserts(DIRECT | ASSOCIATED) && !s.asserts(RELATED), - }, - ] -} - -// ── Sudoku: peers, naked singles, symmetries ───────────────────────────── - fn peers() -> Vec<[u8; 20]> { (0..81) .map(|i| { @@ -208,23 +137,6 @@ fn is_solution(g: &Grid) -> bool { }) } -/// SplitMix64: deterministic, seedable. -struct Rng(u64); -impl Rng { - fn next(&mut self) -> u64 { - self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z ^ (z >> 31) - } - fn shuffle(&mut self, xs: &mut [T]) { - for i in (1..xs.len()).rev() { - xs.swap(i, (self.next() % (i as u64 + 1)) as usize); - } - } -} - /// One Sudoku symmetry: digit relabelling, then rows and columns through /// band/stack and within-band/stack permutations, then optional transpose. #[derive(Clone, Copy)] @@ -278,17 +190,7 @@ impl Symmetry { } } -// ── the lane ───────────────────────────────────────────────────────────── - -/// About a million claim edges, with the solver's own counts kept apart from -/// the edges so the folds have something independent to agree with. -struct Lane { - edges: Vec, - puzzle_of: Vec, - puzzles: u32, - expected: [usize; 5], -} - +/// About `min_edges` claim edges; each copy of the puzzle is one group. fn build_lane(min_edges: usize, seed: u64) -> Lane { let peers = peers(); let base = propagate(&PUZZLE, usize::MAX, &peers); @@ -299,244 +201,98 @@ fn build_lane(min_edges: usize, seed: u64) -> Lane { let base_givens = PUZZLE.iter().filter(|&&d| d != 0).count(); let mut rng = Rng(seed); - let (mut edges, mut puzzle_of) = (Vec::new(), Vec::new()); - let mut expected = [0usize; 5]; - let mut puzzles = 0u32; - while edges.len() < min_edges { + let mut lane = Lane::default(); + while lane.edges.len() < min_edges { let sym = Symmetry::draw(&mut rng); let puzzle = sym.apply(&PUZZLE); let solution = sym.apply(&base.grid); // The copy solves to the transformed base solution, whatever order // propagation takes. let full = propagate(&puzzle, usize::MAX, &peers); - assert_eq!(full.grid, solution, "copy {puzzles} solves differently"); - let depth = (rng.next() % (81 - base_givens as u64 + 1)) as usize; + assert_eq!( + full.grid, solution, + "copy {} solves differently", + lane.groups + ); + let depth = rng.below(81 - base_givens as u64 + 1) as usize; let snap = propagate(&puzzle, depth, &peers); - let mut push = |s: EpistemicState5| { - edges.push(CausalEdge64::ZERO.with_epistemic_raw5(s.raw())); - puzzle_of.push(puzzles); - }; for i in 0..81 { if puzzle[i] != 0 { - push(GIVEN); - expected[1] += 1; + lane.push(GIVEN); + lane.expected[0] += 1; } else if snap.grid[i] != 0 { - push(DERIVED); - expected[2] += 1; + lane.push(FORCED); + lane.expected[1] += 1; } else { // Naked singles never eliminate the true digit. assert_ne!(snap.candidates[i] & 1 << solution[i], 0); - push(ENTAILED); - expected[3] += 1; + lane.push(ENTAILED); + lane.expected[2] += 1; let others = snap.candidates[i] & !(1 << solution[i]); for _ in 0..others.count_ones() { - push(CANDIDATE); - expected[4] += 1; + lane.push(CANDIDATE); + lane.expected[3] += 1; } } } - assert_eq!(snap.derived.len(), expected_derived_in(&puzzle, &snap.grid)); - expected[0] += 81; - puzzles += 1; - } - Lane { - edges, - puzzle_of, - puzzles, - expected, - } -} - -fn expected_derived_in(puzzle: &Grid, grid: &Grid) -> usize { - (0..81).filter(|&i| puzzle[i] == 0 && grid[i] != 0).count() -} - -/// The declaration gate, checked once per lane: a lane whose class does not -/// declare the canonical reading is refused before any edge is read. -fn admit(decl: &Epi5Declarations, class: ClassId) -> Option { - decl.get(class, RAIL) -} - -fn declarations() -> Epi5Declarations { - let mut d = Epi5Declarations::new(); - d.declare(SUDOKU_CLASS, RAIL, Epi5Reading::default()); - d -} - -// ── the folds ──────────────────────────────────────────────────────────── - -fn raw5(e: CausalEdge64) -> u32 { - (e.0 >> EPISTEMIC_SHIFT) as u32 -} - -/// Fast path, one question: a shift and a mask per edge. -fn count_in(edges: &[CausalEdge64], population: Population) -> usize { - edges - .iter() - .filter(|&&e| population >> raw5(e) & 1 == 1) - .count() -} - -/// Fast path, every question at once: one pass builds a 32-bin histogram; a -/// population's count is then the sum of its bins. -fn histogram(edges: &[CausalEdge64]) -> [usize; 32] { - let mut h = [0usize; 32]; - for &e in edges { - h[raw5(e) as usize] += 1; - } - h -} - -fn count_from(h: &[usize; 32], population: Population) -> usize { - (0..32) - .filter(|&r| population >> r & 1 == 1) - .map(|r| h[r]) - .sum() -} - -/// Slow path, the oracle: decode every edge through the declaration, then -/// ask the decoded state. -fn count_decoded( - edges: &[CausalEdge64], - decl: &Epi5Declarations, - asks: fn(EpistemicState5) -> bool, -) -> usize { - edges - .iter() - .filter(|e| { - let s = decl - .project_state5( - SUDOKU_CLASS, - RAIL, - Epi5Gen::V1, - e.epistemic_raw5(), - EdgeProvenance::V2Stamped, - ) - .expect("every edge in a declared lane decodes"); - asks(s) - }) - .count() -} - -/// Group fold: one question per puzzle. -fn per_puzzle(lane: &Lane, population: Population) -> Vec { - let mut out = vec![0u32; lane.puzzles as usize]; - for (&e, &p) in lane.edges.iter().zip(&lane.puzzle_of) { - out[p as usize] += population >> raw5(e) & 1; + assert_eq!( + snap.derived.len(), + (0..81) + .filter(|&i| puzzle[i] == 0 && snap.grid[i] != 0) + .count() + ); + lane.close_group(); } - out -} - -fn median_ns_per_edge(edges: usize, mut f: impl FnMut() -> usize) -> f64 { - let mut times: Vec = (0..7) - .map(|_| { - let t = Instant::now(); - black_box(f()); - t.elapsed().as_nanos() as f64 / edges as f64 - }) - .collect(); - times.sort_by(f64::total_cmp); - times[times.len() / 2] + lane } fn main() { - let decl = declarations(); + let decl = declarations(SUDOKU_CLASS); admit(&decl, SUDOKU_CLASS).expect("the Sudoku class declares the canonical reading"); let t = Instant::now(); let lane = build_lane(1_000_000, 0x5D0C_u64); - let built = t.elapsed(); println!("D-PUZZLE-0 / Sudoku at scale"); println!( " lane: {} edges from {} puzzle copies, built in {:.2?}", lane.edges.len(), - lane.puzzles, - built + lane.groups, + t.elapsed() ); - println!( - " build: debug_assertions={} avx2={} avx512f={}", - cfg!(debug_assertions), - cfg!(target_feature = "avx2"), - cfg!(target_feature = "avx512f") + let counts = check_three_ways(&lane, &decl, SUDOKU_CLASS); + assert!( + per_group(&lane, GIVEN.bit() | FORCED.bit() | ENTAILED.bit()) + .iter() + .all(|&n| n == 81) ); - - let h = histogram(&lane.edges); - for (q, want) in queries().iter().zip(lane.expected) { - let fast = count_in(&lane.edges, q.population); - let hist = count_from(&h, q.population); - let slow = count_decoded(&lane.edges, &decl, q.asks); - assert!(fast == want && hist == want && slow == want, "{}", q.name); - println!( - " {:<60} pop {:#010x} {:>8} edges", - q.name, q.population, want - ); - } - assert!(per_puzzle(&lane, facts_population(CAUSES)) - .iter() - .all(|&n| n == 81)); - println!(" every copy: exactly 81 placed-digit claims (group fold)"); - - let n = lane.edges.len(); - let pop = facts_population(IND_UNKNOWN | CAUSES); - let filter = median_ns_per_edge(n, || count_in(black_box(&lane.edges), pop)); - let hist = median_ns_per_edge(n, || histogram(black_box(&lane.edges))[22]); - let decoded = median_ns_per_edge(n, || { - count_decoded(black_box(&lane.edges), &decl, |s| { - s.asserts(IND_UNKNOWN | CAUSES) - }) - }); - println!(" median of 7, ns per edge:"); - println!(" one population, filter fold {filter:.3}"); - println!(" 32-bin histogram (all questions) {hist:.3}"); - println!(" per-edge decode + asserts (oracle) {decoded:.3}"); + println!(" every copy: exactly 81 asserted claims (group fold)"); + report(&lane, &decl, SUDOKU_CLASS, &counts); } #[cfg(test)] mod tests { + use super::population_fold::{count_in, queries, UNKNOWN_CAUSES}; use super::*; + use lance_graph_contract::epistemic_state5::fact::{CAUSES, DIRECT, IND_KNOWN}; + use lance_graph_contract::epistemic_state5::facts_population; fn small_lane() -> Lane { build_lane(60_000, 0x7E57) } - /// The four claim states are the coordinates the table in the module doc - /// names, and each query's population is exactly the expected code set. - #[test] - fn the_reading_and_the_populations_are_the_stated_codes() { - assert_eq!( - [GIVEN.raw(), DERIVED.raw(), ENTAILED.raw(), CANDIDATE.raw()], - [20, 21, 22, 4] - ); - let pops: Vec = queries().iter().map(|q| q.population).collect(); - assert_eq!(pops, [0xF0_0000, 1 << 20, 1 << 21, 1 << 22, 1 << 4]); - } - - /// Every count three ways: the population filter, the histogram, and the - /// per-edge decode through the declaration — all equal to the solver's own - /// independent counters. + /// Every count three ways (filter, histogram, decode through the + /// declaration), each equal to the solver's own counters, and every + /// question actually exercised. #[test] fn every_question_counts_the_same_three_ways() { - let (lane, decl) = (small_lane(), declarations()); - let h = histogram(&lane.edges); - for (q, want) in queries().iter().zip(lane.expected) { - assert!( - want > 0, - "{}: the corpus never exercised this question", - q.name - ); - assert_eq!(count_in(&lane.edges, q.population), want, "{}", q.name); - assert_eq!(count_from(&h, q.population), want, "{}", q.name); - assert_eq!( - count_decoded(&lane.edges, &decl, q.asks), - want, - "{}", - q.name - ); - } + let lane = small_lane(); + let counts = check_three_ways(&lane, &declarations(SUDOKU_CLASS), SUDOKU_CLASS); + assert!(counts.iter().all(|&n| n > 0), "a question went unexercised"); } - /// The four claim states partition the lane: their populations are - /// disjoint and together cover every edge. + /// The four claim states partition the lane; as populations the three + /// asserted states miss exactly `Unknown × Causes`, which this reading never + /// emits, so the lane holds none of it. #[test] fn the_four_states_partition_the_lane() { let lane = small_lane(); @@ -549,18 +305,9 @@ mod tests { } let union = parts.iter().fold(0, |u, q| u | q.population); assert_eq!(count_in(&lane.edges, union), lane.edges.len()); - // Every placed digit is a given, a derivation or an entailment. As - // POPULATIONS the three miss exactly `Unknown × Causes` (23), which is - // meaningful in general but which this reading never emits, so the - // lane holds none of it and the counts agree. let placed = parts[..3].iter().fold(0, |u, q| u | q.population); - let unknown_causes = state(Topology2::Unknown, Certification3::Causes).bit(); - assert_eq!(qs[0].population & !placed, unknown_causes); - assert_eq!(count_in(&lane.edges, unknown_causes), 0); - assert_eq!( - count_in(&lane.edges, placed), - count_in(&lane.edges, qs[0].population) - ); + assert_eq!(qs[0].population & !placed, UNKNOWN_CAUSES.bit()); + assert_eq!(count_in(&lane.edges, UNKNOWN_CAUSES.bit()), 0); } /// Every copy keeps exactly 81 placed-digit claims and 30 givens, whatever @@ -568,13 +315,13 @@ mod tests { #[test] fn the_group_fold_sees_every_copy_whole() { let lane = small_lane(); - assert!(per_puzzle(&lane, facts_population(CAUSES)) + assert!(per_group(&lane, facts_population(CAUSES)) .iter() .all(|&n| n == 81)); - assert!(per_puzzle(&lane, facts_population(DIRECT | CAUSES)) + assert!(per_group(&lane, facts_population(DIRECT | CAUSES)) .iter() .all(|&n| n == 30)); - let derived = per_puzzle(&lane, facts_population(IND_KNOWN | CAUSES)); + let derived = per_group(&lane, facts_population(IND_KNOWN | CAUSES)); let (lo, hi) = (derived.iter().min().unwrap(), derived.iter().max().unwrap()); assert!(*lo < 5 && *hi > 45, "depths do not vary: {lo}..{hi}"); } @@ -583,6 +330,7 @@ mod tests { /// over the declared facts, not only the five questions. #[test] fn the_histogram_answers_every_requirement() { + use super::population_fold::{count_from, histogram}; use lance_graph_contract::epistemic_state5::fact::{CERTIFICATION_MASK, TOPOLOGY_MASK}; let lane = small_lane(); let h = histogram(&lane.edges); @@ -602,7 +350,7 @@ mod tests { /// before any edge is read. #[test] fn an_undeclared_lane_is_refused_before_any_edge() { - let decl = declarations(); + let decl = declarations(SUDOKU_CLASS); assert!(admit(&decl, SUDOKU_CLASS).is_some()); assert!(admit(&decl, 0x0904).is_none()); } From 0ac5d6b94a9acff34c9031cbdf00f9414b582e68 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:14:49 +0000 Subject: [PATCH 2/9] D-PUZZLE-0 step 2: crosswords on the same population algebra Ten-slot 5x5 crosswords with generated fills, distractor words, two given slots and a forced-single law; every kept instance has exactly one solution (backtracking). The probe holds only the domain; questions, folds and the oracle come unchanged from shared/population_fold.rs. 1,000,052 edges, every count equal three ways and to the instances' own counters. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- crates/cognitive-shader-driver/Cargo.toml | 6 + .../crossword_population_fold_probe.rs | 386 ++++++++++++++++++ 2 files changed, 392 insertions(+) create mode 100644 crates/cognitive-shader-driver/examples/crossword_population_fold_probe.rs diff --git a/crates/cognitive-shader-driver/Cargo.toml b/crates/cognitive-shader-driver/Cargo.toml index 5c442260f..132a0ddba 100644 --- a/crates/cognitive-shader-driver/Cargo.toml +++ b/crates/cognitive-shader-driver/Cargo.toml @@ -153,6 +153,12 @@ test = true name = "sudoku_population_fold_probe" test = true +# D-PUZZLE-0 step 2: crosswords on the same population algebra (shared fold +# with the Sudoku probe); tests run under `cargo test`. +[[example]] +name = "crossword_population_fold_probe" +test = true + [dependencies] lance-graph-contract = { path = "../lance-graph-contract" } # Read-only ontology registry (Phase 7, v4 plan). default-features = false so the diff --git a/crates/cognitive-shader-driver/examples/crossword_population_fold_probe.rs b/crates/cognitive-shader-driver/examples/crossword_population_fold_probe.rs new file mode 100644 index 000000000..581a2fbf8 --- /dev/null +++ b/crates/cognitive-shader-driver/examples/crossword_population_fold_probe.rs @@ -0,0 +1,386 @@ +//! D-PUZZLE-0, step 2: crosswords on the same population algebra as Sudoku. +//! +//! Everything domain-free (the propagation reading, the five questions, the +//! folds, the oracle) comes unchanged from `shared/population_fold.rs`, the +//! module the Sudoku probe uses. This file holds only the crossword: its +//! claims, its law, its own counters. If a crossword question needed code in +//! the shared half, "same algebra" would be false; the shared module's fence +//! test is what would catch that. +//! +//! # The corpus +//! +//! A 5×5 grid with two black squares and ten crossing slots: +//! +//! ```text +//! . . . . # +//! . . . . . +//! . . . . . +//! . . . . . +//! # . . . . +//! ``` +//! +//! Each instance fills the 23 white squares with random letters from a small +//! alphabet; the slot words of that fill are the solution. The dictionary is +//! those words plus random distractors of each length. A few slots are +//! given. An instance is kept only if the dictionary admits exactly one fill +//! consistent with the givens (checked by backtracking), so "the true word" is +//! well defined. Each instance is snapshotted after `k` forced placements. +//! +//! # The law (the only domain logic) +//! +//! A slot's candidates are the dictionary words of its length that agree with +//! every letter already fixed by a placed word. A slot with exactly one +//! candidate is forced (lowest slot first) and fixes its letters. This is the +//! crossword's analogue of a naked single. +//! +//! # The claims +//! +//! Claim = "slot s holds word w", one edge per claim still consistent, in the +//! shared propagation reading: a given slot is `GIVEN`, a forced placement +//! `FORCED`, the true word of a slot not yet placed `ENTAILED`, every other +//! surviving candidate `CANDIDATE`. Class `0x0907` is this probe's own. +//! +//! Run: `cargo run --release -p cognitive-shader-driver --example crossword_population_fold_probe` +//! Tests: `cargo test -p cognitive-shader-driver --example crossword_population_fold_probe` + +use std::time::Instant; + +use lance_graph_contract::class_view::ClassId; + +#[path = "shared/population_fold.rs"] +mod population_fold; +use population_fold::{ + admit, check_three_ways, declarations, per_group, report, Lane, Rng, CANDIDATE, ENTAILED, + FORCED, GIVEN, +}; + +/// The probe's own class (no production class declares a crossword reading). +const CROSSWORD_CLASS: ClassId = 0x0907; + +const SIDE: usize = 5; +const TEMPLATE: [&str; SIDE] = ["....#", ".....", ".....", ".....", "#...."]; +const ALPHABET: u8 = 3; +const DISTRACTORS_PER_LENGTH: usize = 10; +const GIVEN_SLOTS: usize = 2; + +type Word = Vec; + +/// A slot is the ordered list of the squares it covers. +fn slots() -> Vec> { + let white = |r: usize, c: usize| TEMPLATE[r].as_bytes()[c] == b'.'; + let mut out = Vec::new(); + for across in [true, false] { + for line in 0..SIDE { + let mut run = Vec::new(); + for k in 0..=SIDE { + let (r, c) = if across { (line, k) } else { (k, line) }; + if k < SIDE && white(r, c) { + run.push(r * SIDE + c); + } else { + if run.len() >= 2 { + out.push(run.clone()); + } + run.clear(); + } + } + } + } + out +} + +/// One instance: the dictionary (deduplicated), the given slots, and the +/// solution word of every slot. +struct Instance { + dictionary: Vec, + given: Vec, + solution: Vec, +} + +/// Letters fixed so far (`None` = open). +type Fill = [Option; SIDE * SIDE]; + +fn fits(word: &[u8], slot: &[usize], fill: &Fill) -> bool { + slot.iter() + .zip(word) + .all(|(&sq, &l)| fill[sq].is_none_or(|f| f == l)) +} + +fn place(word: &[u8], slot: &[usize], fill: &mut Fill) { + for (&sq, &l) in slot.iter().zip(word) { + fill[sq] = Some(l); + } +} + +fn candidates<'a>(inst: &'a Instance, slot: &[usize], fill: &Fill) -> Vec<&'a Word> { + inst.dictionary + .iter() + .filter(|w| w.len() == slot.len() && fits(w, slot, fill)) + .collect() +} + +/// Number of complete fills consistent with the givens, counted up to `cap`. +fn count_solutions(inst: &Instance, slots: &[Vec], cap: usize) -> usize { + fn go( + inst: &Instance, + slots: &[Vec], + i: usize, + fill: &mut Fill, + n: &mut usize, + cap: usize, + ) { + if *n >= cap { + return; + } + if i == slots.len() { + *n += 1; + return; + } + for w in candidates(inst, &slots[i], fill) { + let saved = *fill; + place(w, &slots[i], fill); + go(inst, slots, i + 1, fill, n, cap); + *fill = saved; + } + } + let mut fill: Fill = [None; SIDE * SIDE]; + for (s, slot) in slots.iter().enumerate() { + if inst.given[s] { + place(&inst.solution[s], slot, &mut fill); + } + } + let mut n = 0; + go(inst, slots, 0, &mut fill, &mut n, cap); + n +} + +/// Draw instances until one has exactly one solution. +fn draw_instance(rng: &mut Rng, slots: &[Vec]) -> Instance { + loop { + let squares: Vec = (0..SIDE * SIDE) + .map(|_| rng.below(u64::from(ALPHABET)) as u8) + .collect(); + let solution: Vec = slots + .iter() + .map(|s| s.iter().map(|&sq| squares[sq]).collect()) + .collect(); + let mut dictionary = solution.clone(); + let mut lengths: Vec = slots.iter().map(Vec::len).collect(); + lengths.sort_unstable(); + lengths.dedup(); + for len in lengths { + for _ in 0..DISTRACTORS_PER_LENGTH { + dictionary.push( + (0..len) + .map(|_| rng.below(u64::from(ALPHABET)) as u8) + .collect(), + ); + } + } + dictionary.sort(); + dictionary.dedup(); + let mut order: Vec = (0..slots.len()).collect(); + rng.shuffle(&mut order); + let mut given = vec![false; slots.len()]; + for &s in &order[..GIVEN_SLOTS] { + given[s] = true; + } + let inst = Instance { + dictionary, + given, + solution, + }; + if count_solutions(&inst, slots, 2) == 1 { + return inst; + } + } +} + +/// The givens, then at most `depth` forced placements. Returns which slots are +/// placed (`Some(true)` given, `Some(false)` forced) and the fill. +fn propagate(inst: &Instance, slots: &[Vec], depth: usize) -> (Vec>, Fill) { + let mut placed: Vec> = vec![None; slots.len()]; + let mut fill: Fill = [None; SIDE * SIDE]; + for (s, slot) in slots.iter().enumerate() { + if inst.given[s] { + place(&inst.solution[s], slot, &mut fill); + placed[s] = Some(true); + } + } + let mut forced = 0; + while forced < depth { + let next = (0..slots.len()).find_map(|s| { + if placed[s].is_some() { + return None; + } + let c = candidates(inst, &slots[s], &fill); + (c.len() == 1).then(|| (s, c[0].clone())) + }); + let Some((s, w)) = next else { break }; + place(&w, &slots[s], &mut fill); + placed[s] = Some(false); + forced += 1; + } + (placed, fill) +} + +/// About `min_edges` claim edges; each instance is one group. +fn build_lane(min_edges: usize, seed: u64) -> Lane { + let slots = slots(); + assert_eq!(slots.len(), 10); + let mut rng = Rng(seed); + let mut lane = Lane::default(); + while lane.edges.len() < min_edges { + let inst = draw_instance(&mut rng, &slots); + let depth = rng.below(slots.len() as u64 - GIVEN_SLOTS as u64 + 1) as usize; + let (placed, fill) = propagate(&inst, &slots, depth); + for (s, slot) in slots.iter().enumerate() { + match placed[s] { + Some(true) => { + lane.push(GIVEN); + lane.expected[0] += 1; + } + Some(false) => { + // A forced word is the true word: the instance is unique. + assert_eq!( + slot.iter().map(|&sq| fill[sq].unwrap()).collect::(), + inst.solution[s] + ); + lane.push(FORCED); + lane.expected[1] += 1; + } + None => { + let c = candidates(&inst, slot, &fill); + // The law never rules out the true word. + assert!(c.contains(&&inst.solution[s])); + lane.push(ENTAILED); + lane.expected[2] += 1; + for _ in 1..c.len() { + lane.push(CANDIDATE); + lane.expected[3] += 1; + } + } + } + } + lane.close_group(); + } + lane +} + +fn main() { + let decl = declarations(CROSSWORD_CLASS); + admit(&decl, CROSSWORD_CLASS).expect("the crossword class declares the canonical reading"); + let t = Instant::now(); + let lane = build_lane(1_000_000, 0xC0_55_u64); + println!("D-PUZZLE-0 / crosswords on the same algebra"); + println!( + " lane: {} edges from {} unique-solution instances, built in {:.2?}", + lane.edges.len(), + lane.groups, + t.elapsed() + ); + let counts = check_three_ways(&lane, &decl, CROSSWORD_CLASS); + assert!( + per_group(&lane, GIVEN.bit() | FORCED.bit() | ENTAILED.bit()) + .iter() + .all(|&n| n == 10) + ); + println!(" every instance: exactly 10 asserted claims, one per slot (group fold)"); + report(&lane, &decl, CROSSWORD_CLASS, &counts); +} + +#[cfg(test)] +mod tests { + use super::population_fold::{count_in, queries, UNKNOWN_CAUSES}; + use super::*; + use lance_graph_contract::epistemic_state5::fact::{CAUSES, DIRECT, IND_KNOWN}; + use lance_graph_contract::epistemic_state5::facts_population; + + fn small_lane() -> Lane { + build_lane(20_000, 0x7E57) + } + + /// The template has the ten slots drawn in the module doc, and every white + /// square sits in exactly two of them. + #[test] + fn the_template_has_ten_crossing_slots() { + let slots = slots(); + let lens: Vec = slots.iter().map(Vec::len).collect(); + assert_eq!(lens, [4, 5, 5, 5, 4, 4, 5, 5, 5, 4]); + let mut cover = [0u8; SIDE * SIDE]; + for s in &slots { + for &sq in s { + cover[sq] += 1; + } + } + let white = cover.iter().filter(|&&n| n > 0).count(); + assert_eq!(white, 23); + assert!(cover.iter().all(|&n| n == 0 || n == 2)); + } + + /// Every count three ways, each equal to the instance's own counters, and + /// every question actually exercised. + #[test] + fn every_question_counts_the_same_three_ways() { + let lane = small_lane(); + let counts = check_three_ways(&lane, &declarations(CROSSWORD_CLASS), CROSSWORD_CLASS); + assert!(counts.iter().all(|&n| n > 0), "a question went unexercised"); + } + + /// Same partition as Sudoku: disjoint states covering every edge, and no + /// `Unknown × Causes`. + #[test] + fn the_four_states_partition_the_lane() { + let lane = small_lane(); + let union = queries()[1..].iter().fold(0, |u, q| u | q.population); + assert_eq!(count_in(&lane.edges, union), lane.edges.len()); + assert_eq!(count_in(&lane.edges, UNKNOWN_CAUSES.bit()), 0); + } + + /// Every instance keeps exactly one asserted claim per slot and its given + /// slots, whatever its depth; the depths actually vary. + #[test] + fn the_group_fold_sees_every_instance_whole() { + let lane = small_lane(); + assert!(per_group(&lane, facts_population(CAUSES)) + .iter() + .all(|&n| n == 10)); + assert!(per_group(&lane, facts_population(DIRECT | CAUSES)) + .iter() + .all(|&n| n as usize == GIVEN_SLOTS)); + let forced = per_group(&lane, facts_population(IND_KNOWN | CAUSES)); + assert_eq!(*forced.iter().min().unwrap(), 0); + assert!(*forced.iter().max().unwrap() >= 5, "forcing never got deep"); + } + + /// Every kept instance has exactly one solution, and the generator does + /// reject ambiguous ones (otherwise the check would be decoration). + #[test] + fn instances_are_unique_and_ambiguous_ones_are_rejected() { + let slots = slots(); + let mut rng = Rng(9); + for _ in 0..50 { + assert_eq!( + count_solutions(&draw_instance(&mut rng, &slots), &slots, 3), + 1 + ); + } + // With no givens and a wide dictionary, ambiguity is common. + let mut rejected = 0; + for _ in 0..50 { + let mut inst = draw_instance(&mut rng, &slots); + inst.given = vec![false; slots.len()]; + if count_solutions(&inst, &slots, 2) > 1 { + rejected += 1; + } + } + assert!(rejected > 0, "uniqueness never bit"); + } + + /// The gate is the declaration, for this domain's class as for Sudoku's. + #[test] + fn an_undeclared_lane_is_refused_before_any_edge() { + let decl = declarations(CROSSWORD_CLASS); + assert!(admit(&decl, CROSSWORD_CLASS).is_some()); + assert!(admit(&decl, 0x0906).is_none()); + } +} From 3e05c7c7540e254d4f1b635f6c236134511768e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:16:03 +0000 Subject: [PATCH 3/9] D-PUZZLE-0 step 2 board: crossword entry, status row, README Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .claude/board/STATUS_BOARD.md | 2 +- ...6-10-07-puzzle-0-crossword-same-algebra.md | 74 +++++++++++++++++++ .claude/board/entries/README.md | 3 +- .../examples/README.md | 13 ++++ 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 .claude/board/entries/2026-10-07-puzzle-0-crossword-same-algebra.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 79d0df653..fa8632089 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -48,7 +48,7 @@ Plan: `.claude/plans/2026-10-06-global-sudoku-replayable-orchestration-v1.md`. O | **D-EPI-CANON-0** | Canonical joint `EpistemicState5` reading of bits 59..63 in the contract | Shipped (#1381) | `contract::epistemic_state5`: Cartesian `Topology2 × Certification3`, `raw5 = t \| c << 2`, 24 meaningful / 8 reserved; per-factor facts; declarations + `project_state5`; `CausalEdge64::{epistemic_raw5, with_epistemic_raw5}`; non-declared-reading writers deprecated | | **D-EPI-POP-0** | Facts → 32-state population: the missing operator between one edge's coordinate (u5) and a semantic condition over all coordinates (u32) | Shipped (#1381) | `contract::epistemic_state5::{Population, facts_population, EpistemicState5::bit}`: `const fn`, no DSL / query object / planner. Tests pin exact sets (`CAUSES` = {20..23}, `IND_UNKNOWN \| RELATED` = {10,14,18,22}, `IND_KNOWN \| SUPPORTS` = {13,17,21}, `RELATED` = 8..23), operator == state enumeration over all 2048 requirements with no reserved code, conjunction == intersection; 2 disable runs red (any-of semantics; reserved read as fact-less) | | **D-EPI-LEGACY-DEPROJECT-0** | Deproject the remaining `ReasoningBand` / CE64-`TrustTexture` consumers: each goes to an exact coordinate, a `facts_population(..)`, MUL, or another subsystem (e.g. counterfactual context) — never a 1:1 replacement enum | Shipped (#1381) | `entries/2026-10-07-epi-legacy-deprojection.md`: 6 ReasoningBand caller sites (1 certification question, 1 non-interference witness → both exact coordinates; 2 context/obsolete → compat; 2 refusal witnesses), 0 production CE64 `TrustTexture` consumers, 4 `TrustTexture` types with no bridge; `EdgeRole::band` removed; hinge + V3 lift migrated; 4 code loops → `facts_population` | -| **D-PUZZLE-0** | Sudoku / crossword / chess on the same population algebra under domain laws | In progress | step 1 Sudoku at scale (`cognitive-shader-driver/examples/sudoku_population_fold_probe.rs`, `entries/2026-10-07-puzzle-0-sudoku-population-fold.md`): 1,000,066 claim edges, counts equal 3 ways + solver counters; filter fold 0.35–0.39 ns/edge, 32-bin histogram 0.93–1.07, decode oracle 3.8–4.0 (one machine). Next: crossword (lance-graph), chess (`stockfish-rs`, GPL) | +| **D-PUZZLE-0** | Sudoku / crossword / chess on the same population algebra under domain laws | In progress | step 1 Sudoku shipped (#1382). Step 2 crossword (`crossword_population_fold_probe.rs`, `entries/2026-10-07-puzzle-0-crossword-same-algebra.md`): domain-free half moved to `examples/shared/population_fold.rs` (fence test, Sudoku counts unchanged); 1,000,052 crossword edges, counts equal 3 ways; filter 0.30–0.43 ns/edge in both domains. Next: chess in `stockfish-rs` (GPL) — how it reaches the shared half is open | | **D-EPI-MIG-0** | Migrate readers/writers to the canonical reading; #1378 → conformance | Shipped (#1381) | `entries/2026-10-07-epi-mig-0.md`: affordance law (per-factor compile) + P7a (identity packing) + 6 probes + dismech + V3 lift; conformance census 24/0/8 (was 3/5/16); 3 historical-band probes cannot migrate without invention | | **D-EPI-CONFLICT-0** | (superseded by D-EPI-MIG-0: probe converted to conformance) Pin the disagreement between the D-ACR-7-declared P7a band and the affordance law on CE64 bits 59..63 | Superseded | **Current:** the same probe is the conformance test, census 24 agree / 0 disagree / 8 reserved under the Cartesian V1 (D-EPI-MIG-0). **HISTORICAL — the pre-Cartesian pins below no longer describe the code:** `cognitive-shader-driver/examples/epistemic_reading_conflict_probe.rs`: contract-projected `Causes` reads as affordance code 20 `Related` (topology 0) or refuses (1..3), `COUNTERFACTUAL_PROBE` never eligible; `CAUSES` codes 1 and 17 sit on P7a `Open` / `CausalCandidate`; codes 25, 30 on P7a reserved bands; census over 24 cells = 3 agree, 5 disagree, 16 refuse; the affordance law measures where the contract refuses (undeclared / band `Absent`). Both readings run from shared code (`shared/affordance_law.rs`, and P7a's reading moved unchanged into `shared/certification_reading.rs`, used by `relational_certification_probe` too), never copied. Pins of current state, to become agreement tests when the issue closes (`ISS-EPISTEMIC-READINGS-DISAGREE-ON-BITS-61-63`); 5 tests, 6 disable runs red (3 on the affordance codebook, 3 on P7a's: `Contributes` entailment, a reserved code admitted, `Causes` recoded) | | **D-MBX-CUTOVER-0** | ShaderDriver on MailboxSoA without BindSpace | In PR | `entries/2026-10-06-shaderdriver-bindspace-cutover.md`: every ShaderDriver → BindSpace edge classified (production state / shared resource / fallback / test oracle / fossil / dead). In `run()` the only mailbox-mode BindSpace read was the ontology; outside it, metrics and the builder read the singleton and `backing()` silently fell back for any mailbox not under id 0. Cutover: optional BindSpace, explicit mailbox selection with no fallback, ontology on the driver, metrics from the selected substrate; no temporal change. `tests/mailbox_cutover.rs` 8 tests, 8 disable runs red | diff --git a/.claude/board/entries/2026-10-07-puzzle-0-crossword-same-algebra.md b/.claude/board/entries/2026-10-07-puzzle-0-crossword-same-algebra.md new file mode 100644 index 000000000..f3018b7b7 --- /dev/null +++ b/.claude/board/entries/2026-10-07-puzzle-0-crossword-same-algebra.md @@ -0,0 +1,74 @@ +# 2026-10-07 — D-PUZZLE-0 step 2: crosswords on the same algebra as Sudoku + +**Probes:** `cognitive-shader-driver/examples/crossword_population_fold_probe.rs`, +`sudoku_population_fold_probe.rs`, both over +`examples/shared/population_fold.rs`. +**Status:** MEASURED (this machine), TEST-PINNED (fence + 6 domain tests, 3 +disable runs red). + +## What "same algebra" means here, and how it is checked + +The domain-free half moved out of the Sudoku probe into +`shared/population_fold.rs`: the propagation reading (given / forced / +entailed-not-forced / candidate), the five questions as `facts_population` +masks, the filter / histogram / decode folds, the group fold, the declaration +gate. Each domain file supplies only its claims, its law and its counters. + +- **Fence:** a test fails if `sudoku`, `crossword`, `chess`, `digit`, `slot`, + `cell`, `letter`, `grid`, `board` or `move` appears in the shared half. + Disable run: one planted word → red. +- **Refactor checked by output:** the Sudoku probe's five counts are + identical before and after the move (1,000,066 edges). + +## The crossword domain + +A 5×5 grid, two black squares, ten slots, every white square in two slots. +Fills are random over a 3-letter alphabet; the dictionary is the solution +words plus 10 distractors per length; two slots are given. An instance is kept +only with exactly one solution (backtracking). The law is the forced single: a +slot whose consistent dictionary words narrow to one is placed. + +## Measured (1,000,052 edges, 22,455 instances) + +| question | edges | +|---|---| +| every asserted claim | 224,550 | +| given | 44,910 | +| forced, chain known | 13,745 | +| entailed, not yet forced | 165,895 | +| live candidates | 775,502 | + +Every count is equal three ways and to the instances' own counters. Every +instance holds exactly 10 asserted claims. The instance build takes 7.3 s +(rejection sampling with backtracking). + +Timing, release, `avx2=true avx512f=false`, same 4-core shared Xeon, median of +7, three runs: + +| path | crossword ns/edge | Sudoku ns/edge | +|---|---|---| +| one population, filter | 0.30 – 0.43 | 0.35 – 0.39 | +| 32-bin histogram | 1.67 – 1.75 | 0.93 – 1.07 | +| per-edge decode (oracle) | 3.9 – 4.1 | 3.8 – 4.0 | + +The filter and the oracle cost the same in both domains: they read five bits +and do not care what the claim is about. The histogram costs more on the +crossword lane, where 77 % of edges share one code. Repeated increments of +one bin are the likely cause; this is a hypothesis, not measured. + +## What differs, and it is the law, not the algebra + +Forcing is rare in crosswords: 13,745 of 177,640 non-given slots, against +Sudoku's naked singles, which solve the base puzzle completely. A slot rarely +narrows to one word without stronger propagation (arc consistency over +crossings), so most true words stay `IndirectUnknown × Causes`. That is a +statement about the law. The questions and folds did not change. + +## Next + +Chess in `stockfish-rs` (GPL; shakmaty supplies legality), using the same +shared half. The open point is how it gets there. The module is an +example-local file in lance-graph and is not part of `lance-graph-contract`. +Copying it into stockfish-rs would make a second copy (a fence violation in +spirit); promoting it into the contract is a contract change. The decision is +open. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 99c214dec..35d835f66 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,13 +25,14 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -247 entries, 2026-08-06 .. 2026-10-07. +248 entries, 2026-08-06 .. 2026-10-07. | date | entry id | finding | file | |---|---|---|---| | 2026-10-07 | `tinker-janus-fold-harvest` | TinkerPop bulk = K (GroupReduce), ONE_BULK = S, GValue pinning = bundle invalidation; JanusGraph slice = OrderedLaneWitness→Range (30–38×); no new V4 op | [2026-10-07-tinker-janus-fold-harvest.md](2026-10-07-tinker-janus-fold-harvest.md) | | 2026-10-07 | `selector-seq-aware-coverage` | | [2026-10-07-selector-seq-aware-coverage.md](2026-10-07-selector-seq-aware-coverage.md) | | 2026-10-07 | `D-PUZZLE-0` | | [2026-10-07-puzzle-0-sudoku-population-fold.md](2026-10-07-puzzle-0-sudoku-population-fold.md) | +| 2026-10-07 | `D-PUZZLE-0` | | [2026-10-07-puzzle-0-crossword-same-algebra.md](2026-10-07-puzzle-0-crossword-same-algebra.md) | | 2026-10-07 | `gated-gather-bit-schedule` | Production gather is 3.8× slow from a per-row branch; bit-gated visitation wins or ties at every density; ordinal vectors never win (R1 holds) | [2026-10-07-gated-gather-bit-schedule.md](2026-10-07-gated-gather-bit-schedule.md) | | 2026-10-07 | `D-EPI-MIG-INVENTORY` | | [2026-10-07-epi-mig-inventory.md](2026-10-07-epi-mig-inventory.md) | | 2026-10-07 | `D-EPI-CANON-0` | | [2026-10-07-epi-mig-0.md](2026-10-07-epi-mig-0.md) | diff --git a/crates/cognitive-shader-driver/examples/README.md b/crates/cognitive-shader-driver/examples/README.md index 6e33a9809..89fe34abb 100644 --- a/crates/cognitive-shader-driver/examples/README.md +++ b/crates/cognitive-shader-driver/examples/README.md @@ -413,3 +413,16 @@ checked once per lane. Every count agrees three ways — population filter, ```bash cargo run --release -p cognitive-shader-driver --example sudoku_population_fold_probe ``` + +## crossword_population_fold_probe.rs + +D-PUZZLE-0, step 2. Crosswords on the same population algebra as the Sudoku +probe: ten-slot 5×5 grids with generated fills, distractor words, two given +slots and a forced-single law, each instance checked to have exactly one +solution. The probe holds only the domain; the reading, questions, folds and +oracle come unchanged from `shared/population_fold.rs`, whose fence test +fails if domain vocabulary enters the shared half. + +```bash +cargo run --release -p cognitive-shader-driver --example crossword_population_fold_probe +``` From cc887e32f154966ab94bf1452bb4f0c5a7cb0b7c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:22:52 +0000 Subject: [PATCH 4/9] D-PUZZLE-0 step 2b: crosswords filled with real COCA (and optional DeReKo) words The step-2 crossword drew its words from a synthetic three-letter alphabet. This probe fills the same 5x5 template from a real dictionary, the top 1000 words per slot length: English COCA forms from the committed word_forms.csv (the file DeepNSM-v2's lexical layer reads), and German DeReKo-2014 only when DEREKO_PATH names the file at run time. DeReKo is CC BY-NC 3.0; nothing derived from it is committed. Instances are random fills, with givens added in random order until the dictionary admits exactly one fill. Law, claims, questions, fold and oracle are unchanged from step 2 and shared/population_fold.rs. Release run: EN 1,002,858 edges / 1338 instances in 4.2 s; DE 1,000,847 / 1483 in 4.7 s. Every count agrees three ways in both. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- crates/cognitive-shader-driver/Cargo.toml | 4 + .../examples/crossword_real_words_probe.rs | 625 ++++++++++++++++++ 2 files changed, 629 insertions(+) create mode 100644 crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs diff --git a/crates/cognitive-shader-driver/Cargo.toml b/crates/cognitive-shader-driver/Cargo.toml index 132a0ddba..f7c682232 100644 --- a/crates/cognitive-shader-driver/Cargo.toml +++ b/crates/cognitive-shader-driver/Cargo.toml @@ -159,6 +159,10 @@ test = true name = "crossword_population_fold_probe" test = true +[[example]] +name = "crossword_real_words_probe" +test = true + [dependencies] lance-graph-contract = { path = "../lance-graph-contract" } # Read-only ontology registry (Phase 7, v4 plan). default-features = false so the diff --git a/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs b/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs new file mode 100644 index 000000000..d012ccaff --- /dev/null +++ b/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs @@ -0,0 +1,625 @@ +//! D-PUZZLE-0, step 2b: crosswords filled with real words. +//! +//! Step 2 (`crossword_population_fold_probe`) used a synthetic three-letter +//! alphabet. This probe fills the same 5×5 template from a real word list: +//! +//! - **English** (default, always available): COCA word forms from the +//! committed `crates/deepnsm/word_frequency/word_forms.csv`, the file +//! DeepNSM-v2's lexical layer reads. Alphabetic ASCII surfaces, ranked by +//! their largest `wordFreq`. +//! - **German** (only with `DEREKO_PATH` set): the DeReKo-2014 STTS frequency +//! list (`form, lemma, tag, freq`, tab-separated), proper nouns (`NE`) +//! excluded, frequencies summed per lowercase form. DeReKo is CC BY-NC 3.0 +//! (IDS Mannheim); it is read from the given path at run time and nothing +//! derived from it is committed. Without the variable the German run is +//! skipped, never failed. +//! +//! The dictionary is the `TOP` most frequent words of each slot length. An +//! instance is a random fill of the template from that dictionary; given +//! slots are added in random order until the dictionary admits exactly one +//! fill consistent with them, so "the true word" is well defined. The law, +//! the claims and every question are the step-2 ones; the fold, the reading +//! and the oracle come unchanged from `shared/population_fold.rs`. +//! +//! Run: `cargo run --release -p cognitive-shader-driver --example crossword_real_words_probe` +//! German too: `DEREKO_PATH=/path/DeReKo-2014-II-MainArchive-STT.100000.freq cargo run ...` +//! Tests: `cargo test -p cognitive-shader-driver --example crossword_real_words_probe` + +use std::collections::HashMap; +use std::time::Instant; + +use lance_graph_contract::class_view::ClassId; + +#[path = "shared/population_fold.rs"] +mod population_fold; +use population_fold::{ + admit, check_three_ways, declarations, per_group, report, Lane, Rng, CANDIDATE, ENTAILED, + FORCED, GIVEN, +}; + +/// Same class as step 2: the same domain, a different dictionary. +const CROSSWORD_CLASS: ClassId = 0x0907; + +const SIDE: usize = 5; +const TEMPLATE: [&str; SIDE] = ["....#", ".....", ".....", ".....", "#...."]; +/// Words per slot length kept in the dictionary. +const TOP: usize = 1000; +/// Search nodes allowed for one random fill before a fresh start. +const FILL_BUDGET: usize = 20_000; + +const COCA: &str = include_str!("../../deepnsm/word_frequency/word_forms.csv"); + +type Word = Vec; +type Fill = [Option; SIDE * SIDE]; + +fn slots() -> Vec> { + let white = |r: usize, c: usize| TEMPLATE[r].as_bytes()[c] == b'.'; + let mut out = Vec::new(); + for across in [true, false] { + for line in 0..SIDE { + let mut run = Vec::new(); + for k in 0..=SIDE { + let (r, c) = if across { (line, k) } else { (k, line) }; + if k < SIDE && white(r, c) { + run.push(r * SIDE + c); + } else { + if run.len() >= 2 { + out.push(run.clone()); + } + run.clear(); + } + } + } + } + out +} + +/// COCA surfaces: alphabetic ASCII, lowercase, each scored by its largest +/// `wordFreq` row. +fn coca_frequencies() -> HashMap { + let mut out: HashMap = HashMap::new(); + for line in COCA.lines().skip(1) { + let f: Vec<&str> = line.trim_end_matches('\r').split(',').collect(); + let [_, _, _, _, freq, word] = f.as_slice() else { + continue; + }; + let w = word.trim().to_lowercase(); + if w.is_empty() || !w.chars().all(|c| c.is_ascii_alphabetic()) { + continue; + } + let n: f64 = freq.trim().parse().unwrap_or(0.0); + let e = out.entry(w).or_insert(0.0); + *e = e.max(n); + } + out +} + +/// DeReKo forms: alphabetic (umlauts and ß included), lowercase, proper nouns +/// excluded, frequencies summed over lemma/tag rows. +fn dereko_frequencies(text: &str) -> HashMap { + let mut out: HashMap = HashMap::new(); + for line in text.lines() { + let f: Vec<&str> = line.split('\t').collect(); + let [form, _, tag, freq] = f.as_slice() else { + continue; + }; + if *tag == "NE" { + continue; + } + let w = form.to_lowercase(); + if w.is_empty() || !w.chars().all(char::is_alphabetic) { + continue; + } + *out.entry(w).or_insert(0.0) += freq.trim().parse::().unwrap_or(0.0); + } + out +} + +/// The `top` most frequent words of one length, ties broken alphabetically +/// so the list is deterministic. +fn top_words(freq: &HashMap, len: usize, top: usize) -> Vec { + let mut ws: Vec<(&String, f64)> = freq + .iter() + .filter(|(w, _)| w.chars().count() == len) + .map(|(w, &f)| (w, f)) + .collect(); + ws.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(b.0))); + ws.into_iter() + .take(top) + .map(|(w, _)| w.chars().collect()) + .collect() +} + +/// One length's words, indexed by (position, letter) as a bitset over words. +struct Bucket { + words: Vec, + index: Vec>>, +} + +impl Bucket { + fn new(words: Vec, len: usize) -> Self { + let blocks = words.len().div_ceil(64); + let mut index = vec![HashMap::new(); len]; + for (i, w) in words.iter().enumerate() { + for (p, &ch) in w.iter().enumerate() { + let bits = index[p].entry(ch).or_insert_with(|| vec![0u64; blocks]); + bits[i / 64] |= 1 << (i % 64); + } + } + Self { words, index } + } + + /// Indices of the words agreeing with every letter fixed in `slot`. + fn candidates(&self, slot: &[usize], fill: &Fill) -> Vec { + let blocks = self.words.len().div_ceil(64); + let mut acc = vec![u64::MAX; blocks]; + if !self.words.len().is_multiple_of(64) { + acc[blocks - 1] = (1u64 << (self.words.len() % 64)) - 1; + } + for (p, &sq) in slot.iter().enumerate() { + if let Some(ch) = fill[sq] { + match self.index[p].get(&ch) { + Some(bits) => acc.iter_mut().zip(bits).for_each(|(a, b)| *a &= b), + None => return Vec::new(), + } + } + } + let mut out = Vec::new(); + for (b, &word) in acc.iter().enumerate() { + let mut w = word; + while w != 0 { + out.push(b * 64 + w.trailing_zeros() as usize); + w &= w - 1; + } + } + out + } +} + +struct Dictionary { + by_len: HashMap, +} + +impl Dictionary { + fn new(freq: &HashMap, slots: &[Vec], top: usize) -> Self { + let mut lens: Vec = slots.iter().map(Vec::len).collect(); + lens.sort_unstable(); + lens.dedup(); + let by_len = lens + .into_iter() + .map(|l| (l, Bucket::new(top_words(freq, l, top), l))) + .collect(); + Self { by_len } + } + + fn bucket(&self, slot: &[usize]) -> &Bucket { + &self.by_len[&slot.len()] + } + + fn candidates(&self, slot: &[usize], fill: &Fill) -> Vec<&Word> { + let b = self.bucket(slot); + b.candidates(slot, fill) + .into_iter() + .map(|i| &b.words[i]) + .collect() + } +} + +fn place(word: &[char], slot: &[usize], fill: &mut Fill) { + for (&sq, &l) in slot.iter().zip(word) { + fill[sq] = Some(l); + } +} + +/// The open slot with the fewest candidates (most-constrained first). +fn tightest(dict: &Dictionary, slots: &[Vec], done: &[bool], fill: &Fill) -> Option { + (0..slots.len()) + .filter(|&s| !done[s]) + .min_by_key(|&s| dict.bucket(&slots[s]).candidates(&slots[s], fill).len()) +} + +/// Complete fills consistent with `fill`, counted up to `cap`. +fn count_fills( + dict: &Dictionary, + slots: &[Vec], + done: &mut [bool], + fill: &mut Fill, + cap: usize, +) -> usize { + let Some(s) = tightest(dict, slots, done, fill) else { + return 1; + }; + let mut n = 0; + done[s] = true; + for w in dict.candidates(&slots[s], fill) { + let saved = *fill; + place(w, &slots[s], fill); + n += count_fills(dict, slots, done, fill, cap - n); + *fill = saved; + if n >= cap { + break; + } + } + done[s] = false; + n +} + +/// One random complete fill, or `None` when the node budget runs out. +fn random_fill(dict: &Dictionary, slots: &[Vec], rng: &mut Rng) -> Option> { + fn go( + dict: &Dictionary, + slots: &[Vec], + done: &mut [bool], + fill: &mut Fill, + rng: &mut Rng, + budget: &mut usize, + ) -> bool { + let Some(s) = tightest(dict, slots, done, fill) else { + return true; + }; + let mut cs = dict.candidates(&slots[s], fill); + rng.shuffle(&mut cs); + done[s] = true; + for w in cs { + if *budget == 0 { + break; + } + *budget -= 1; + let saved = *fill; + place(w, &slots[s], fill); + if go(dict, slots, done, fill, rng, budget) { + return true; + } + *fill = saved; + } + done[s] = false; + false + } + let mut done = vec![false; slots.len()]; + let mut fill: Fill = [None; SIDE * SIDE]; + let mut budget = FILL_BUDGET; + go(dict, slots, &mut done, &mut fill, rng, &mut budget).then(|| { + slots + .iter() + .map(|s| s.iter().map(|&sq| fill[sq].unwrap()).collect()) + .collect() + }) +} + +struct Instance { + given: Vec, + solution: Vec, +} + +/// Number of fills consistent with the instance's givens, up to `cap`. +fn count_solutions(dict: &Dictionary, slots: &[Vec], inst: &Instance, cap: usize) -> usize { + let mut fill: Fill = [None; SIDE * SIDE]; + let mut done = inst.given.clone(); + for (s, slot) in slots.iter().enumerate() { + if inst.given[s] { + place(&inst.solution[s], slot, &mut fill); + } + } + count_fills(dict, slots, &mut done, &mut fill, cap) +} + +/// A random fill, then given slots added in random order until it is the only +/// fill the dictionary admits. Every placed word must be a dictionary word, so +/// the solution is checked against the dictionary too. +fn draw_instance(dict: &Dictionary, slots: &[Vec], rng: &mut Rng) -> Instance { + loop { + let Some(solution) = random_fill(dict, slots, rng) else { + continue; + }; + let mut order: Vec = (0..slots.len()).collect(); + rng.shuffle(&mut order); + let mut inst = Instance { + given: vec![false; slots.len()], + solution, + }; + for &s in &order { + inst.given[s] = true; + if count_solutions(dict, slots, &inst, 2) == 1 { + return inst; + } + } + unreachable!("with every slot given the fill is unique"); + } +} + +fn propagate( + dict: &Dictionary, + slots: &[Vec], + inst: &Instance, + depth: usize, +) -> (Vec>, Fill) { + let mut placed: Vec> = vec![None; slots.len()]; + let mut fill: Fill = [None; SIDE * SIDE]; + for (s, slot) in slots.iter().enumerate() { + if inst.given[s] { + place(&inst.solution[s], slot, &mut fill); + placed[s] = Some(true); + } + } + let mut forced = 0; + while forced < depth { + let next = (0..slots.len()).find_map(|s| { + if placed[s].is_some() { + return None; + } + let c = dict.candidates(&slots[s], &fill); + (c.len() == 1).then(|| (s, c[0].clone())) + }); + let Some((s, w)) = next else { break }; + place(&w, &slots[s], &mut fill); + placed[s] = Some(false); + forced += 1; + } + (placed, fill) +} + +/// Per-lane bookkeeping the shared `Lane` does not keep. +#[derive(Default)] +struct Stats { + givens: Vec, + forced: usize, +} + +fn build_lane(dict: &Dictionary, min_edges: usize, seed: u64) -> (Lane, Stats) { + let slots = slots(); + let mut rng = Rng(seed); + let mut lane = Lane::default(); + let mut stats = Stats::default(); + while lane.edges.len() < min_edges { + let inst = draw_instance(dict, &slots, &mut rng); + let g = inst.given.iter().filter(|&&b| b).count(); + stats.givens.push(g); + let depth = rng.below((slots.len() - g) as u64 + 1) as usize; + let (placed, fill) = propagate(dict, &slots, &inst, depth); + for (s, slot) in slots.iter().enumerate() { + match placed[s] { + Some(true) => { + lane.push(GIVEN); + lane.expected[0] += 1; + } + Some(false) => { + assert_eq!( + slot.iter().map(|&sq| fill[sq].unwrap()).collect::(), + inst.solution[s] + ); + lane.push(FORCED); + lane.expected[1] += 1; + stats.forced += 1; + } + None => { + let c = dict.candidates(slot, &fill); + assert!(c.contains(&&inst.solution[s])); + lane.push(ENTAILED); + lane.expected[2] += 1; + for _ in 1..c.len() { + lane.push(CANDIDATE); + lane.expected[3] += 1; + } + } + } + } + lane.close_group(); + } + (lane, stats) +} + +fn run(name: &str, dict: &Dictionary) { + let decl = declarations(CROSSWORD_CLASS); + admit(&decl, CROSSWORD_CLASS).expect("the crossword class declares the canonical reading"); + let t = Instant::now(); + let (lane, stats) = build_lane(dict, 1_000_000, 0xC0_CA_u64); + let mut hist = [0usize; 11]; + for &g in &stats.givens { + hist[g] += 1; + } + println!("D-PUZZLE-0 / real-word crosswords: {name}"); + for (len, b) in { + let mut v: Vec<_> = dict.by_len.iter().collect(); + v.sort_by_key(|(l, _)| **l); + v + } { + println!(" dictionary: {} words of length {len}", b.words.len()); + } + println!( + " lane: {} edges from {} unique-solution instances, built in {:.2?}", + lane.edges.len(), + lane.groups, + t.elapsed() + ); + let spread: Vec = hist + .iter() + .enumerate() + .filter(|(_, &n)| n > 0) + .map(|(g, n)| format!("{g}:{n}")) + .collect(); + println!( + " givens needed for uniqueness (givens:instances): {}", + spread.join(" ") + ); + let counts = check_three_ways(&lane, &decl, CROSSWORD_CLASS); + assert!( + per_group(&lane, GIVEN.bit() | FORCED.bit() | ENTAILED.bit()) + .iter() + .all(|&n| n == 10) + ); + println!(" every instance: exactly 10 asserted claims, one per slot (group fold)"); + report(&lane, &decl, CROSSWORD_CLASS, &counts); +} + +fn main() { + let slots = slots(); + run( + "English COCA (committed word_forms.csv)", + &Dictionary::new(&coca_frequencies(), &slots, TOP), + ); + match std::env::var("DEREKO_PATH") { + Ok(path) => { + let text = std::fs::read_to_string(&path).expect("DEREKO_PATH is readable"); + run( + "German DeReKo-2014 (CC BY-NC 3.0, read at run time)", + &Dictionary::new(&dereko_frequencies(&text), &slots, TOP), + ); + } + Err(_) => println!("German run skipped: set DEREKO_PATH to the DeReKo-2014 .freq file"), + } +} + +#[cfg(test)] +mod tests { + use super::population_fold::{count_in, queries, UNKNOWN_CAUSES}; + use super::*; + use lance_graph_contract::epistemic_state5::fact::{CAUSES, DIRECT, IND_KNOWN}; + use lance_graph_contract::epistemic_state5::facts_population; + + fn coca() -> Dictionary { + Dictionary::new(&coca_frequencies(), &slots(), TOP) + } + + /// The English dictionary is real COCA words, in frequency order. + #[test] + fn the_coca_dictionary_is_real_words() { + let d = coca(); + // COCA's word-form list holds only 902 alphabetic four-letter forms, + // so that length is the whole list rather than the top `TOP`. + for (len, n) in [(4, 902), (5, TOP)] { + let b = &d.by_len[&len]; + assert_eq!(b.words.len(), n); + assert!(b.words.iter().all(|w| w.len() == len)); + } + let four: Vec = d.by_len[&4] + .words + .iter() + .map(|w| w.iter().collect()) + .collect(); + assert_eq!(four[0], "that"); + for w in ["have", "with", "this", "from"] { + assert!(four.iter().any(|x| x == w), "{w} missing"); + } + } + + /// The bitset index returns exactly the words a direct scan returns. + #[test] + fn the_index_agrees_with_a_scan() { + let d = coca(); + let slots = slots(); + let mut rng = Rng(3); + for _ in 0..200 { + let mut fill: Fill = [None; SIDE * SIDE]; + for sq in fill.iter_mut() { + if rng.below(4) == 0 { + *sq = Some((b'a' + rng.below(26) as u8) as char); + } + } + for slot in &slots { + let b = d.bucket(slot); + let scan: Vec = (0..b.words.len()) + .filter(|&i| { + slot.iter() + .zip(&b.words[i]) + .all(|(&sq, &l)| fill[sq].is_none_or(|f| f == l)) + }) + .collect(); + assert_eq!(b.candidates(slot, &fill), scan); + } + } + } + + fn small_lane() -> (Lane, Stats) { + build_lane(&coca(), 30_000, 0x7E57) + } + + #[test] + fn every_question_counts_the_same_three_ways() { + let (lane, _) = small_lane(); + let counts = check_three_ways(&lane, &declarations(CROSSWORD_CLASS), CROSSWORD_CLASS); + assert!(counts.iter().all(|&n| n > 0), "a question went unexercised"); + } + + #[test] + fn the_four_states_partition_the_lane() { + let (lane, _) = small_lane(); + let union = queries()[1..].iter().fold(0, |u, q| u | q.population); + assert_eq!(count_in(&lane.edges, union), lane.edges.len()); + assert_eq!(count_in(&lane.edges, UNKNOWN_CAUSES.bit()), 0); + } + + /// One asserted claim per slot; the given count per instance is what the + /// generator recorded; forcing happens on real words. + #[test] + fn the_group_fold_sees_every_instance_whole() { + let (lane, stats) = small_lane(); + assert!(per_group(&lane, facts_population(CAUSES)) + .iter() + .all(|&n| n == 10)); + let given: Vec = per_group(&lane, facts_population(DIRECT | CAUSES)) + .into_iter() + .map(|n| n as usize) + .collect(); + assert_eq!(given, stats.givens); + let forced = per_group(&lane, facts_population(IND_KNOWN | CAUSES)); + assert!( + forced.iter().sum::() > 0, + "no real-word slot was ever forced" + ); + } + + /// Kept instances are unique, every solution word is a dictionary word, + /// and dropping the last given makes the instance ambiguous again (the + /// givens are minimal along the drawn order, so uniqueness binds). + #[test] + fn instances_are_unique_and_the_last_given_is_needed() { + let d = coca(); + let slots = slots(); + let mut rng = Rng(11); + let mut ambiguous_without = 0; + for _ in 0..20 { + let inst = draw_instance(&d, &slots, &mut rng); + assert_eq!(count_solutions(&d, &slots, &inst, 3), 1); + for (s, w) in inst.solution.iter().enumerate() { + assert!(d.bucket(&slots[s]).words.contains(w)); + } + let g = inst.given.iter().filter(|&&b| b).count(); + if g > 0 { + // Drop every given: with no givens a real-word grid of this + // size always has another fill (its own transpose at least is + // checked by the count, not assumed). + let open = Instance { + given: vec![false; slots.len()], + solution: inst.solution.clone(), + }; + if count_solutions(&d, &slots, &open, 2) > 1 { + ambiguous_without += 1; + } + } + } + assert!(ambiguous_without > 0, "uniqueness never needed a given"); + } + + /// German parsing: umlauts and ß kept, proper nouns dropped, rows summed. + #[test] + fn dereko_rows_are_parsed_by_tag_and_summed() { + let text = "Haus\tHaus\tNN\t10\nhaus\thausen\tVVIMP\t2\nBerlin\tBerlin\tNE\t99\n\ + Füße\tFuß\tNN\t5\n,\t,\t$,\t1000\n"; + let f = dereko_frequencies(text); + assert_eq!(f.get("haus"), Some(&12.0)); + assert_eq!(f.get("füße"), Some(&5.0)); + assert!(!f.contains_key("berlin")); + assert!(!f.contains_key(",")); + assert_eq!( + top_words(&f, 4, 10), + vec!["haus".chars().collect::(), "füße".chars().collect()] + ); + } + + #[test] + fn an_undeclared_lane_is_refused_before_any_edge() { + let decl = declarations(CROSSWORD_CLASS); + assert!(admit(&decl, CROSSWORD_CLASS).is_some()); + assert!(admit(&decl, 0x0906).is_none()); + } +} From 30b15427b5c0b575000fac6f6c50cfd1c216520f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:24:17 +0000 Subject: [PATCH 5/9] D-PUZZLE-0 step 2b board: real-word crossword entry, status row, README Also renames the uniqueness test to what it checks: with every given removed, the dictionary admits more than one fill. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .claude/board/STATUS_BOARD.md | 2 +- ...026-10-07-puzzle-0-crossword-real-words.md | 71 +++++++++++++++++++ .claude/board/entries/README.md | 3 +- .../examples/README.md | 15 ++++ .../examples/crossword_real_words_probe.rs | 24 +++---- 5 files changed, 98 insertions(+), 17 deletions(-) create mode 100644 .claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index fa8632089..86a139d8f 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -48,7 +48,7 @@ Plan: `.claude/plans/2026-10-06-global-sudoku-replayable-orchestration-v1.md`. O | **D-EPI-CANON-0** | Canonical joint `EpistemicState5` reading of bits 59..63 in the contract | Shipped (#1381) | `contract::epistemic_state5`: Cartesian `Topology2 × Certification3`, `raw5 = t \| c << 2`, 24 meaningful / 8 reserved; per-factor facts; declarations + `project_state5`; `CausalEdge64::{epistemic_raw5, with_epistemic_raw5}`; non-declared-reading writers deprecated | | **D-EPI-POP-0** | Facts → 32-state population: the missing operator between one edge's coordinate (u5) and a semantic condition over all coordinates (u32) | Shipped (#1381) | `contract::epistemic_state5::{Population, facts_population, EpistemicState5::bit}`: `const fn`, no DSL / query object / planner. Tests pin exact sets (`CAUSES` = {20..23}, `IND_UNKNOWN \| RELATED` = {10,14,18,22}, `IND_KNOWN \| SUPPORTS` = {13,17,21}, `RELATED` = 8..23), operator == state enumeration over all 2048 requirements with no reserved code, conjunction == intersection; 2 disable runs red (any-of semantics; reserved read as fact-less) | | **D-EPI-LEGACY-DEPROJECT-0** | Deproject the remaining `ReasoningBand` / CE64-`TrustTexture` consumers: each goes to an exact coordinate, a `facts_population(..)`, MUL, or another subsystem (e.g. counterfactual context) — never a 1:1 replacement enum | Shipped (#1381) | `entries/2026-10-07-epi-legacy-deprojection.md`: 6 ReasoningBand caller sites (1 certification question, 1 non-interference witness → both exact coordinates; 2 context/obsolete → compat; 2 refusal witnesses), 0 production CE64 `TrustTexture` consumers, 4 `TrustTexture` types with no bridge; `EdgeRole::band` removed; hinge + V3 lift migrated; 4 code loops → `facts_population` | -| **D-PUZZLE-0** | Sudoku / crossword / chess on the same population algebra under domain laws | In progress | step 1 Sudoku shipped (#1382). Step 2 crossword (`crossword_population_fold_probe.rs`, `entries/2026-10-07-puzzle-0-crossword-same-algebra.md`): domain-free half moved to `examples/shared/population_fold.rs` (fence test, Sudoku counts unchanged); 1,000,052 crossword edges, counts equal 3 ways; filter 0.30–0.43 ns/edge in both domains. Next: chess in `stockfish-rs` (GPL) — how it reaches the shared half is open | +| **D-PUZZLE-0** | Sudoku / crossword / chess on the same population algebra under domain laws | In progress | step 1 Sudoku shipped (#1382). Step 2 crossword (`crossword_population_fold_probe.rs`, `entries/2026-10-07-puzzle-0-crossword-same-algebra.md`): domain-free half moved to `examples/shared/population_fold.rs` (fence test, Sudoku counts unchanged); 1,000,052 crossword edges, counts equal 3 ways; filter 0.30–0.43 ns/edge in both domains. Step 2b real words (`crossword_real_words_probe.rs`, `entries/2026-10-07-puzzle-0-crossword-real-words.md`): COCA from the committed CSV, DeReKo only from `DEREKO_PATH` (CC BY-NC, nothing committed); ~1.0M edges each, counts equal 3 ways. Next: chess in `stockfish-rs` (GPL) — how it reaches the shared half is open | | **D-EPI-MIG-0** | Migrate readers/writers to the canonical reading; #1378 → conformance | Shipped (#1381) | `entries/2026-10-07-epi-mig-0.md`: affordance law (per-factor compile) + P7a (identity packing) + 6 probes + dismech + V3 lift; conformance census 24/0/8 (was 3/5/16); 3 historical-band probes cannot migrate without invention | | **D-EPI-CONFLICT-0** | (superseded by D-EPI-MIG-0: probe converted to conformance) Pin the disagreement between the D-ACR-7-declared P7a band and the affordance law on CE64 bits 59..63 | Superseded | **Current:** the same probe is the conformance test, census 24 agree / 0 disagree / 8 reserved under the Cartesian V1 (D-EPI-MIG-0). **HISTORICAL — the pre-Cartesian pins below no longer describe the code:** `cognitive-shader-driver/examples/epistemic_reading_conflict_probe.rs`: contract-projected `Causes` reads as affordance code 20 `Related` (topology 0) or refuses (1..3), `COUNTERFACTUAL_PROBE` never eligible; `CAUSES` codes 1 and 17 sit on P7a `Open` / `CausalCandidate`; codes 25, 30 on P7a reserved bands; census over 24 cells = 3 agree, 5 disagree, 16 refuse; the affordance law measures where the contract refuses (undeclared / band `Absent`). Both readings run from shared code (`shared/affordance_law.rs`, and P7a's reading moved unchanged into `shared/certification_reading.rs`, used by `relational_certification_probe` too), never copied. Pins of current state, to become agreement tests when the issue closes (`ISS-EPISTEMIC-READINGS-DISAGREE-ON-BITS-61-63`); 5 tests, 6 disable runs red (3 on the affordance codebook, 3 on P7a's: `Contributes` entailment, a reserved code admitted, `Causes` recoded) | | **D-MBX-CUTOVER-0** | ShaderDriver on MailboxSoA without BindSpace | In PR | `entries/2026-10-06-shaderdriver-bindspace-cutover.md`: every ShaderDriver → BindSpace edge classified (production state / shared resource / fallback / test oracle / fossil / dead). In `run()` the only mailbox-mode BindSpace read was the ontology; outside it, metrics and the builder read the singleton and `backing()` silently fell back for any mailbox not under id 0. Cutover: optional BindSpace, explicit mailbox selection with no fallback, ontology on the driver, metrics from the selected substrate; no temporal change. `tests/mailbox_cutover.rs` 8 tests, 8 disable runs red | diff --git a/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md b/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md new file mode 100644 index 000000000..862c4b480 --- /dev/null +++ b/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md @@ -0,0 +1,71 @@ +# 2026-10-07 — D-PUZZLE-0 step 2b: real-word crosswords (COCA, optional DeReKo) + +**Probe:** `cognitive-shader-driver/examples/crossword_real_words_probe.rs`, +over `examples/shared/population_fold.rs` (unchanged). +**Status:** MEASURED (this machine), TEST-PINNED (10 tests, 3 disable runs red). + +## What changed from step 2 + +The step-2 crossword drew its words from a synthetic 3-letter alphabet. This +probe fills the same 5×5 template from a real dictionary: the 1000 most +frequent words per slot length. + +- **English:** COCA word forms from the committed + `crates/deepnsm/word_frequency/word_forms.csv`, the file DeepNSM-v2's + lexical layer reads. Alphabetic ASCII surfaces, scored by their largest + `wordFreq`. COCA holds only 902 such four-letter forms, so that length is the + whole list. +- **German:** the DeReKo-2014 STTS frequency list, only when `DEREKO_PATH` + names the file at run time; skipped otherwise. Proper nouns (`NE`) are + excluded and frequencies summed per lowercase form. DeReKo is © IDS + Mannheim, CC BY-NC 3.0. No German word, list or derived artifact is + committed; this entry carries counts only. + +The probe reads the COCA CSV itself. DeepNSM-v2's `load_word_forms_csv` routes +each surface through a `PaletteVocab` (the KJV-trained vocabulary), which would +drop most of the words a crossword needs. + +An instance is a random fill of the template from the dictionary. Given slots +are then added in random order until the dictionary admits exactly one fill +(backtracking). Law, claims, questions, fold and oracle are the step-2 ones. + +## Measured (release, `avx2=true avx512f=false`, median of 7) + +| | English COCA | German DeReKo | +|---|---|---| +| edges | 1,002,858 | 1,000,847 | +| instances | 1,338 | 1,483 | +| build time | 4.2 s | 4.7 s | +| given | 5,673 | 6,545 | +| forced | 1,574 | 1,713 | +| entailed, not yet forced | 6,133 | 6,572 | +| live candidates | 989,478 | 986,017 | +| filter fold, ns/edge | 0.34 | 0.36 | +| histogram, ns/edge | 2.08 | 2.02 | +| decode oracle, ns/edge | 3.87 | 3.83 | + +Every count is equal three ways (filter, histogram, decode) and to the +instances' own counters. Every instance holds exactly 10 asserted claims. + +Givens needed for uniqueness (givens: instances), EN: +`1:102 2:249 3:214 4:202 5:172 6:169 7:122 8:77 9:31`. DE: +`1:94 2:240 3:246 4:221 5:212 6:189 7:140 8:94 9:47`. These are minimal +along a random order, not globally minimal; a smarter order would need fewer. + +The filter and the oracle cost the same as for Sudoku and the synthetic +crossword. The histogram is slower again (2.0 ns against 1.7 synthetic and +1.0 Sudoku). That fits the step-2 hypothesis: here 98.6 % of edges share one +code, against 77 % there. It fits, but nothing here measures the cause. + +## Falsifiers + +- Uniqueness check returning 1 always → `instances_are_unique_and_givens_are_needed` + red (the givens must be needed for at least one instance). +- Index ignoring fixed letters → four tests red, including + `the_index_agrees_with_a_scan` (bitset index against a direct scan). +- `NE` rows kept → `dereko_rows_are_parsed_by_tag_and_summed` red. + +## Open + +- German runs only where the DeReKo file is present; CI covers English only. +- Chess in `stockfish-rs` and how it reaches the shared fold: still open. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 35d835f66..5e51b6bc5 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -248 entries, 2026-08-06 .. 2026-10-07. +249 entries, 2026-08-06 .. 2026-10-07. | date | entry id | finding | file | |---|---|---|---| @@ -33,6 +33,7 @@ exactly one of them, never both. | 2026-10-07 | `selector-seq-aware-coverage` | | [2026-10-07-selector-seq-aware-coverage.md](2026-10-07-selector-seq-aware-coverage.md) | | 2026-10-07 | `D-PUZZLE-0` | | [2026-10-07-puzzle-0-sudoku-population-fold.md](2026-10-07-puzzle-0-sudoku-population-fold.md) | | 2026-10-07 | `D-PUZZLE-0` | | [2026-10-07-puzzle-0-crossword-same-algebra.md](2026-10-07-puzzle-0-crossword-same-algebra.md) | +| 2026-10-07 | `D-PUZZLE-0` | | [2026-10-07-puzzle-0-crossword-real-words.md](2026-10-07-puzzle-0-crossword-real-words.md) | | 2026-10-07 | `gated-gather-bit-schedule` | Production gather is 3.8× slow from a per-row branch; bit-gated visitation wins or ties at every density; ordinal vectors never win (R1 holds) | [2026-10-07-gated-gather-bit-schedule.md](2026-10-07-gated-gather-bit-schedule.md) | | 2026-10-07 | `D-EPI-MIG-INVENTORY` | | [2026-10-07-epi-mig-inventory.md](2026-10-07-epi-mig-inventory.md) | | 2026-10-07 | `D-EPI-CANON-0` | | [2026-10-07-epi-mig-0.md](2026-10-07-epi-mig-0.md) | diff --git a/crates/cognitive-shader-driver/examples/README.md b/crates/cognitive-shader-driver/examples/README.md index 89fe34abb..d97873471 100644 --- a/crates/cognitive-shader-driver/examples/README.md +++ b/crates/cognitive-shader-driver/examples/README.md @@ -426,3 +426,18 @@ fails if domain vocabulary enters the shared half. ```bash cargo run --release -p cognitive-shader-driver --example crossword_population_fold_probe ``` + +## crossword_real_words_probe.rs + +D-PUZZLE-0 step 2b. The step-2 crossword with a real dictionary: the 1000 most +frequent words per slot length, English from the committed COCA +`crates/deepnsm/word_frequency/word_forms.csv`, German from DeReKo-2014 only +when `DEREKO_PATH` names the file (CC BY-NC 3.0; nothing derived is committed). +Instances are random fills, givens added until the fill is unique. About 1.0M +edges per language; every count agrees three ways. Board: +`.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md`. + +``` +cargo run --release -p cognitive-shader-driver --example crossword_real_words_probe +DEREKO_PATH=/path/DeReKo-2014-II-MainArchive-STT.100000.freq cargo run --release -p cognitive-shader-driver --example crossword_real_words_probe +``` diff --git a/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs b/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs index d012ccaff..5bf2a8e88 100644 --- a/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs +++ b/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs @@ -569,10 +569,10 @@ mod tests { } /// Kept instances are unique, every solution word is a dictionary word, - /// and dropping the last given makes the instance ambiguous again (the - /// givens are minimal along the drawn order, so uniqueness binds). + /// and the givens are doing work: with all of them removed, the + /// dictionary admits more than one fill of the same grid. #[test] - fn instances_are_unique_and_the_last_given_is_needed() { + fn instances_are_unique_and_givens_are_needed() { let d = coca(); let slots = slots(); let mut rng = Rng(11); @@ -583,18 +583,12 @@ mod tests { for (s, w) in inst.solution.iter().enumerate() { assert!(d.bucket(&slots[s]).words.contains(w)); } - let g = inst.given.iter().filter(|&&b| b).count(); - if g > 0 { - // Drop every given: with no givens a real-word grid of this - // size always has another fill (its own transpose at least is - // checked by the count, not assumed). - let open = Instance { - given: vec![false; slots.len()], - solution: inst.solution.clone(), - }; - if count_solutions(&d, &slots, &open, 2) > 1 { - ambiguous_without += 1; - } + let open = Instance { + given: vec![false; slots.len()], + solution: inst.solution.clone(), + }; + if count_solutions(&d, &slots, &open, 2) > 1 { + ambiguous_without += 1; } } assert!(ambiguous_without > 0, "uniqueness never needed a given"); From d182301de231f7f2ec3667a2ddff523035c7e8f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:27:05 +0000 Subject: [PATCH 6/9] D-PUZZLE-0 step 2b: dictionaries are DeepNSM's vocabularies English is now exactly what DeepNSM reads: v1's 4096-word table (word_rank_lookup.csv, ranks 1..=4096) united with v2's academic_20k.csv; word_forms.csv is no longer a source. German is the 20,000 most frequent DeReKo forms (NE and non-alphabetic excluded). The dictionary is every word of a slot's length in that vocabulary; the top-1000 cut is gone. Dictionaries: EN 1250 four-letter / 1697 five-letter, DE 802 / 1466. Release run: EN 1,005,146 edges / 1209 instances, DE 1,000,074 / 1390; counts equal three ways in both. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .../examples/crossword_real_words_probe.rs | 146 ++++++++++++------ 1 file changed, 99 insertions(+), 47 deletions(-) diff --git a/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs b/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs index 5bf2a8e88..22be8431e 100644 --- a/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs +++ b/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs @@ -3,18 +3,19 @@ //! Step 2 (`crossword_population_fold_probe`) used a synthetic three-letter //! alphabet. This probe fills the same 5×5 template from a real word list: //! -//! - **English** (default, always available): COCA word forms from the -//! committed `crates/deepnsm/word_frequency/word_forms.csv`, the file -//! DeepNSM-v2's lexical layer reads. Alphabetic ASCII surfaces, ranked by -//! their largest `wordFreq`. -//! - **German** (only with `DEREKO_PATH` set): the DeReKo-2014 STTS frequency -//! list (`form, lemma, tag, freq`, tab-separated), proper nouns (`NE`) -//! excluded, frequencies summed per lowercase form. DeReKo is CC BY-NC 3.0 -//! (IDS Mannheim); it is read from the given path at run time and nothing -//! derived from it is committed. Without the variable the German run is -//! skipped, never failed. +//! - **English** (default, always available): the two vocabularies DeepNSM +//! reads. v1's 4096-word table (`word_rank_lookup.csv`, ranks 1..=4096) and +//! v2's academic list (`academic_20k.csv`, 18,559 distinct words), both +//! committed under `crates/deepnsm/word_frequency/`. Their union, alphabetic +//! ASCII only, scored by the larger COCA frequency. No other English source. +//! - **German** (only with `DEREKO_PATH` set): the 20,000 most frequent forms +//! of the DeReKo-2014 STTS list (`form, lemma, tag, freq`, tab-separated), +//! after excluding proper nouns (`NE`) and non-alphabetic forms and summing +//! frequencies per lowercase form. DeReKo is CC BY-NC 3.0 (IDS Mannheim); it +//! is read from the given path at run time and nothing derived from it is +//! committed. Without the variable the German run is skipped, never failed. //! -//! The dictionary is the `TOP` most frequent words of each slot length. An +//! The dictionary is every word of a slot's length in that vocabulary. An //! instance is a random fill of the template from that dictionary; given //! slots are added in random order until the dictionary admits exactly one //! fill consistent with them, so "the true word" is well defined. The law, @@ -42,12 +43,15 @@ const CROSSWORD_CLASS: ClassId = 0x0907; const SIDE: usize = 5; const TEMPLATE: [&str; SIDE] = ["....#", ".....", ".....", ".....", "#...."]; -/// Words per slot length kept in the dictionary. -const TOP: usize = 1000; +/// v1's vocabulary size (`deepnsm::vocabulary::VOCAB_SIZE`). +const V1_VOCAB: u32 = 4096; +/// Forms kept from the German frequency list. +const GERMAN_VOCAB: usize = 20_000; /// Search nodes allowed for one random fill before a fresh start. const FILL_BUDGET: usize = 20_000; -const COCA: &str = include_str!("../../deepnsm/word_frequency/word_forms.csv"); +const RANK_LOOKUP: &str = include_str!("../../deepnsm/word_frequency/word_rank_lookup.csv"); +const ACADEMIC: &str = include_str!("../../deepnsm/word_frequency/academic_20k.csv"); type Word = Vec; type Fill = [Option; SIDE * SIDE]; @@ -74,22 +78,36 @@ fn slots() -> Vec> { out } -/// COCA surfaces: alphabetic ASCII, lowercase, each scored by its largest -/// `wordFreq` row. -fn coca_frequencies() -> HashMap { - let mut out: HashMap = HashMap::new(); - for line in COCA.lines().skip(1) { +/// Insert `word` with `freq`, keeping the larger frequency, if it is +/// alphabetic ASCII. +fn keep_max(out: &mut HashMap, word: &str, freq: &str) { + let w = word.trim().to_lowercase(); + if w.is_empty() || !w.chars().all(|c| c.is_ascii_alphabetic()) { + return; + } + let n: f64 = freq.trim().parse().unwrap_or(0.0); + let e = out.entry(w).or_insert(0.0); + *e = e.max(n); +} + +/// DeepNSM's English vocabulary: v1's 4096 (`rank,word,pos,freq`, rank cut) +/// united with v2's academic list (`ID,band,status,word,Pos,COCA-All,...`). +fn english_vocabulary() -> HashMap { + let mut out = HashMap::new(); + for line in RANK_LOOKUP.lines().skip(1) { let f: Vec<&str> = line.trim_end_matches('\r').split(',').collect(); - let [_, _, _, _, freq, word] = f.as_slice() else { + let [rank, word, _, freq] = f.as_slice() else { continue; }; - let w = word.trim().to_lowercase(); - if w.is_empty() || !w.chars().all(|c| c.is_ascii_alphabetic()) { - continue; + if rank.trim().parse::().is_ok_and(|r| r <= V1_VOCAB) { + keep_max(&mut out, word, freq); + } + } + for line in ACADEMIC.lines().skip(1) { + let f: Vec<&str> = line.trim_end_matches('\r').split(',').collect(); + if let (Some(word), Some(freq)) = (f.get(3), f.get(5)) { + keep_max(&mut out, word, freq); } - let n: f64 = freq.trim().parse().unwrap_or(0.0); - let e = out.entry(w).or_insert(0.0); - *e = e.max(n); } out } @@ -115,6 +133,15 @@ fn dereko_frequencies(text: &str) -> HashMap { out } +/// The `GERMAN_VOCAB` most frequent DeReKo forms (ties alphabetical). +fn german_vocabulary(text: &str) -> HashMap { + let f = dereko_frequencies(text); + let mut ranked: Vec<(String, f64)> = f.into_iter().collect(); + ranked.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + ranked.truncate(GERMAN_VOCAB); + ranked.into_iter().collect() +} + /// The `top` most frequent words of one length, ties broken alphabetically /// so the list is deterministic. fn top_words(freq: &HashMap, len: usize, top: usize) -> Vec { @@ -181,13 +208,13 @@ struct Dictionary { } impl Dictionary { - fn new(freq: &HashMap, slots: &[Vec], top: usize) -> Self { + fn new(freq: &HashMap, slots: &[Vec]) -> Self { let mut lens: Vec = slots.iter().map(Vec::len).collect(); lens.sort_unstable(); lens.dedup(); let by_len = lens .into_iter() - .map(|l| (l, Bucket::new(top_words(freq, l, top), l))) + .map(|l| (l, Bucket::new(top_words(freq, l, usize::MAX), l))) .collect(); Self { by_len } } @@ -454,15 +481,15 @@ fn run(name: &str, dict: &Dictionary) { fn main() { let slots = slots(); run( - "English COCA (committed word_forms.csv)", - &Dictionary::new(&coca_frequencies(), &slots, TOP), + "English, DeepNSM v1 4096 + academic 20k (committed)", + &Dictionary::new(&english_vocabulary(), &slots), ); match std::env::var("DEREKO_PATH") { Ok(path) => { let text = std::fs::read_to_string(&path).expect("DEREKO_PATH is readable"); run( - "German DeReKo-2014 (CC BY-NC 3.0, read at run time)", - &Dictionary::new(&dereko_frequencies(&text), &slots, TOP), + "German, DeReKo-2014 top 20k (CC BY-NC 3.0, read at run time)", + &Dictionary::new(&german_vocabulary(&text), &slots), ); } Err(_) => println!("German run skipped: set DEREKO_PATH to the DeReKo-2014 .freq file"), @@ -477,29 +504,36 @@ mod tests { use lance_graph_contract::epistemic_state5::facts_population; fn coca() -> Dictionary { - Dictionary::new(&coca_frequencies(), &slots(), TOP) + Dictionary::new(&english_vocabulary(), &slots()) } - /// The English dictionary is real COCA words, in frequency order. + /// The English dictionary is exactly DeepNSM's two vocabularies. Each + /// source boundary has a witness: v1 core words, academic-only words, + /// words ranked past 4096 in v1's table and absent from the academic list + /// (the rank cut), and an inflected form only `word_forms.csv` carries. #[test] - fn the_coca_dictionary_is_real_words() { + fn the_english_dictionary_is_deepnsm_v1_plus_academic() { let d = coca(); - // COCA's word-form list holds only 902 alphabetic four-letter forms, - // so that length is the whole list rather than the top `TOP`. - for (len, n) in [(4, 902), (5, TOP)] { + for (len, n) in [(4, 1250), (5, 1697)] { let b = &d.by_len[&len]; assert_eq!(b.words.len(), n); assert!(b.words.iter().all(|w| w.len() == len)); } - let four: Vec = d.by_len[&4] - .words - .iter() - .map(|w| w.iter().collect()) - .collect(); - assert_eq!(four[0], "that"); - for w in ["have", "with", "this", "from"] { - assert!(four.iter().any(|x| x == w), "{w} missing"); - } + let has = |w: &str| { + let w: Word = w.chars().collect(); + d.by_len[&w.len()].words.contains(&w) + }; + assert_eq!(d.by_len[&4].words[0], "have".chars().collect::()); + assert!(has("that") && has("with"), "v1 core words"); + assert!(has("abyss") && has("acorn"), "academic-only words"); + assert!( + !has("gosh") && !has("kinda"), + "v1 rank cut at 4096 not applied" + ); + assert!( + !has("says"), + "an inflected form from word_forms.csv leaked in" + ); } /// The bitset index returns exactly the words a direct scan returns. @@ -610,6 +644,24 @@ mod tests { ); } + /// German keeps the 20,000 most frequent forms and drops the rest. + #[test] + fn german_keeps_the_top_20k_forms() { + let mut text = String::new(); + for i in 0..GERMAN_VOCAB + 5 { + // Distinct alphabetic forms, frequency falling with i. + let w: String = format!("{i:05}") + .bytes() + .map(|b| (b'a' + (b - b'0')) as char) + .collect(); + text.push_str(&format!("{w}\t{w}\tNN\t{}\n", 1_000_000 - i)); + } + let v = german_vocabulary(&text); + assert_eq!(v.len(), GERMAN_VOCAB); + assert!(v.contains_key("aaaaa"), "most frequent form dropped"); + assert!(!v.contains_key("caaae"), "form ranked past 20k kept"); // i = 20004 + } + #[test] fn an_undeclared_lane_is_refused_before_any_edge() { let decl = declarations(CROSSWORD_CLASS); From 3e236e2047ae7a0e3c21cca1c7292f392dc1b6b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:28:02 +0000 Subject: [PATCH 7/9] D-PUZZLE-0 step 2b board: DeepNSM vocabularies, new counts, disable runs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .claude/board/STATUS_BOARD.md | 2 +- ...026-10-07-puzzle-0-crossword-real-words.md | 87 ++++++++++--------- .../examples/README.md | 9 +- 3 files changed, 52 insertions(+), 46 deletions(-) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 86a139d8f..261664c94 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -48,7 +48,7 @@ Plan: `.claude/plans/2026-10-06-global-sudoku-replayable-orchestration-v1.md`. O | **D-EPI-CANON-0** | Canonical joint `EpistemicState5` reading of bits 59..63 in the contract | Shipped (#1381) | `contract::epistemic_state5`: Cartesian `Topology2 × Certification3`, `raw5 = t \| c << 2`, 24 meaningful / 8 reserved; per-factor facts; declarations + `project_state5`; `CausalEdge64::{epistemic_raw5, with_epistemic_raw5}`; non-declared-reading writers deprecated | | **D-EPI-POP-0** | Facts → 32-state population: the missing operator between one edge's coordinate (u5) and a semantic condition over all coordinates (u32) | Shipped (#1381) | `contract::epistemic_state5::{Population, facts_population, EpistemicState5::bit}`: `const fn`, no DSL / query object / planner. Tests pin exact sets (`CAUSES` = {20..23}, `IND_UNKNOWN \| RELATED` = {10,14,18,22}, `IND_KNOWN \| SUPPORTS` = {13,17,21}, `RELATED` = 8..23), operator == state enumeration over all 2048 requirements with no reserved code, conjunction == intersection; 2 disable runs red (any-of semantics; reserved read as fact-less) | | **D-EPI-LEGACY-DEPROJECT-0** | Deproject the remaining `ReasoningBand` / CE64-`TrustTexture` consumers: each goes to an exact coordinate, a `facts_population(..)`, MUL, or another subsystem (e.g. counterfactual context) — never a 1:1 replacement enum | Shipped (#1381) | `entries/2026-10-07-epi-legacy-deprojection.md`: 6 ReasoningBand caller sites (1 certification question, 1 non-interference witness → both exact coordinates; 2 context/obsolete → compat; 2 refusal witnesses), 0 production CE64 `TrustTexture` consumers, 4 `TrustTexture` types with no bridge; `EdgeRole::band` removed; hinge + V3 lift migrated; 4 code loops → `facts_population` | -| **D-PUZZLE-0** | Sudoku / crossword / chess on the same population algebra under domain laws | In progress | step 1 Sudoku shipped (#1382). Step 2 crossword (`crossword_population_fold_probe.rs`, `entries/2026-10-07-puzzle-0-crossword-same-algebra.md`): domain-free half moved to `examples/shared/population_fold.rs` (fence test, Sudoku counts unchanged); 1,000,052 crossword edges, counts equal 3 ways; filter 0.30–0.43 ns/edge in both domains. Step 2b real words (`crossword_real_words_probe.rs`, `entries/2026-10-07-puzzle-0-crossword-real-words.md`): COCA from the committed CSV, DeReKo only from `DEREKO_PATH` (CC BY-NC, nothing committed); ~1.0M edges each, counts equal 3 ways. Next: chess in `stockfish-rs` (GPL) — how it reaches the shared half is open | +| **D-PUZZLE-0** | Sudoku / crossword / chess on the same population algebra under domain laws | In progress | step 1 Sudoku shipped (#1382). Step 2 crossword (`crossword_population_fold_probe.rs`, `entries/2026-10-07-puzzle-0-crossword-same-algebra.md`): domain-free half moved to `examples/shared/population_fold.rs` (fence test, Sudoku counts unchanged); 1,000,052 crossword edges, counts equal 3 ways; filter 0.30–0.43 ns/edge in both domains. Step 2b real words (`crossword_real_words_probe.rs`, `entries/2026-10-07-puzzle-0-crossword-real-words.md`): English = DeepNSM v1 4096 + academic_20k (committed), German = DeReKo top 20k only from `DEREKO_PATH` (CC BY-NC, nothing committed); ~1.0M edges each, counts equal 3 ways. Next: chess in `stockfish-rs` (GPL) — how it reaches the shared half is open | | **D-EPI-MIG-0** | Migrate readers/writers to the canonical reading; #1378 → conformance | Shipped (#1381) | `entries/2026-10-07-epi-mig-0.md`: affordance law (per-factor compile) + P7a (identity packing) + 6 probes + dismech + V3 lift; conformance census 24/0/8 (was 3/5/16); 3 historical-band probes cannot migrate without invention | | **D-EPI-CONFLICT-0** | (superseded by D-EPI-MIG-0: probe converted to conformance) Pin the disagreement between the D-ACR-7-declared P7a band and the affordance law on CE64 bits 59..63 | Superseded | **Current:** the same probe is the conformance test, census 24 agree / 0 disagree / 8 reserved under the Cartesian V1 (D-EPI-MIG-0). **HISTORICAL — the pre-Cartesian pins below no longer describe the code:** `cognitive-shader-driver/examples/epistemic_reading_conflict_probe.rs`: contract-projected `Causes` reads as affordance code 20 `Related` (topology 0) or refuses (1..3), `COUNTERFACTUAL_PROBE` never eligible; `CAUSES` codes 1 and 17 sit on P7a `Open` / `CausalCandidate`; codes 25, 30 on P7a reserved bands; census over 24 cells = 3 agree, 5 disagree, 16 refuse; the affordance law measures where the contract refuses (undeclared / band `Absent`). Both readings run from shared code (`shared/affordance_law.rs`, and P7a's reading moved unchanged into `shared/certification_reading.rs`, used by `relational_certification_probe` too), never copied. Pins of current state, to become agreement tests when the issue closes (`ISS-EPISTEMIC-READINGS-DISAGREE-ON-BITS-61-63`); 5 tests, 6 disable runs red (3 on the affordance codebook, 3 on P7a's: `Contributes` entailment, a reserved code admitted, `Causes` recoded) | | **D-MBX-CUTOVER-0** | ShaderDriver on MailboxSoA without BindSpace | In PR | `entries/2026-10-06-shaderdriver-bindspace-cutover.md`: every ShaderDriver → BindSpace edge classified (production state / shared resource / fallback / test oracle / fossil / dead). In `run()` the only mailbox-mode BindSpace read was the ontology; outside it, metrics and the builder read the singleton and `backing()` silently fell back for any mailbox not under id 0. Cutover: optional BindSpace, explicit mailbox selection with no fallback, ontology on the driver, metrics from the selected substrate; no temporal change. `tests/mailbox_cutover.rs` 8 tests, 8 disable runs red | diff --git a/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md b/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md index 862c4b480..da58cb955 100644 --- a/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md +++ b/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md @@ -1,69 +1,74 @@ -# 2026-10-07 — D-PUZZLE-0 step 2b: real-word crosswords (COCA, optional DeReKo) +# 2026-10-07 — D-PUZZLE-0 step 2b: real-word crosswords (DeepNSM vocabularies, optional DeReKo) **Probe:** `cognitive-shader-driver/examples/crossword_real_words_probe.rs`, over `examples/shared/population_fold.rs` (unchanged). -**Status:** MEASURED (this machine), TEST-PINNED (10 tests, 3 disable runs red). +**Status:** MEASURED (this machine), TEST-PINNED (11 tests, 6 disable runs red). ## What changed from step 2 The step-2 crossword drew its words from a synthetic 3-letter alphabet. This -probe fills the same 5×5 template from a real dictionary: the 1000 most -frequent words per slot length. +probe fills the same 5×5 template from a real vocabulary. The dictionary is +every word of a slot's length in that vocabulary. -- **English:** COCA word forms from the committed - `crates/deepnsm/word_frequency/word_forms.csv`, the file DeepNSM-v2's - lexical layer reads. Alphabetic ASCII surfaces, scored by their largest - `wordFreq`. COCA holds only 902 such four-letter forms, so that length is the - whole list. -- **German:** the DeReKo-2014 STTS frequency list, only when `DEREKO_PATH` - names the file at run time; skipped otherwise. Proper nouns (`NE`) are - excluded and frequencies summed per lowercase form. DeReKo is © IDS - Mannheim, CC BY-NC 3.0. No German word, list or derived artifact is - committed; this entry carries counts only. +- **English:** exactly the two vocabularies DeepNSM reads, both committed + under `crates/deepnsm/word_frequency/`. These are v1's 4096-word table + (`word_rank_lookup.csv`, ranks 1..=4096) and v2's academic list + (`academic_20k.csv`), taken as their union. Alphabetic ASCII only, scored by + the larger COCA frequency. `word_forms.csv` is not a source. Dictionary: + 1,250 four-letter, 1,697 five-letter words. +- **German:** the 20,000 most frequent DeReKo-2014 forms, after excluding + proper nouns (`NE`) and non-alphabetic forms and summing frequencies per + lowercase form. Read only when `DEREKO_PATH` names the file at run time and + skipped otherwise. DeReKo is © IDS Mannheim, CC BY-NC 3.0. No German word, + list or derived artifact is committed; this entry carries counts only. + Dictionary: 802 four-letter, 1,466 five-letter words. -The probe reads the COCA CSV itself. DeepNSM-v2's `load_word_forms_csv` routes -each surface through a `PaletteVocab` (the KJV-trained vocabulary), which would -drop most of the words a crossword needs. +The first version of this probe (`cc887e3`) used the top 1000 `word_forms.csv` +surfaces per length. It was replaced on review because that list is not a +vocabulary DeepNSM uses. An instance is a random fill of the template from the dictionary. Given slots are then added in random order until the dictionary admits exactly one fill (backtracking). Law, claims, questions, fold and oracle are the step-2 ones. -## Measured (release, `avx2=true avx512f=false`, median of 7) +## Measured (release, `avx2=true avx512f=false`, median of 7, one run) -| | English COCA | German DeReKo | +| | English (v1 4096 + academic) | German (DeReKo top 20k) | |---|---|---| -| edges | 1,002,858 | 1,000,847 | -| instances | 1,338 | 1,483 | -| build time | 4.2 s | 4.7 s | -| given | 5,673 | 6,545 | -| forced | 1,574 | 1,713 | -| entailed, not yet forced | 6,133 | 6,572 | -| live candidates | 989,478 | 986,017 | -| filter fold, ns/edge | 0.34 | 0.36 | -| histogram, ns/edge | 2.08 | 2.02 | -| decode oracle, ns/edge | 3.87 | 3.83 | +| edges | 1,005,146 | 1,000,074 | +| instances | 1,209 | 1,390 | +| build time | 7.4 s | 6.7 s | +| given | 5,370 | 6,118 | +| forced | 1,278 | 1,400 | +| entailed, not yet forced | 5,442 | 6,382 | +| live candidates | 993,056 | 986,174 | +| filter fold, ns/edge | 0.35 | 0.36 | +| histogram, ns/edge | 2.23 | 2.10 | +| decode oracle, ns/edge | 4.01 | 3.79 | Every count is equal three ways (filter, histogram, decode) and to the instances' own counters. Every instance holds exactly 10 asserted claims. Givens needed for uniqueness (givens: instances), EN: -`1:102 2:249 3:214 4:202 5:172 6:169 7:122 8:77 9:31`. DE: -`1:94 2:240 3:246 4:221 5:212 6:189 7:140 8:94 9:47`. These are minimal -along a random order, not globally minimal; a smarter order would need fewer. +`1:23 2:246 3:208 4:204 5:161 6:129 7:109 8:88 9:41`. DE: +`1:53 2:250 3:263 4:208 5:200 6:152 7:132 8:80 9:52`. These are minimal +along a random order, not globally minimal. The filter and the oracle cost the same as for Sudoku and the synthetic -crossword. The histogram is slower again (2.0 ns against 1.7 synthetic and -1.0 Sudoku). That fits the step-2 hypothesis: here 98.6 % of edges share one -code, against 77 % there. It fits, but nothing here measures the cause. +crossword. The histogram is slower again (2.1–2.2 ns, against 1.7 synthetic +and 1.0 Sudoku). That fits the step-2 hypothesis: here 98.6–98.8 % of edges +share one code, against 77 % there. It fits, but nothing here measures the +cause. -## Falsifiers +## Falsifiers (disable-verified red) -- Uniqueness check returning 1 always → `instances_are_unique_and_givens_are_needed` - red (the givens must be needed for at least one instance). -- Index ignoring fixed letters → four tests red, including - `the_index_agrees_with_a_scan` (bitset index against a direct scan). -- `NE` rows kept → `dereko_rows_are_parsed_by_tag_and_summed` red. +- Uniqueness check returning 1 always → `instances_are_unique_and_givens_are_needed`. +- Index ignoring fixed letters → 4 tests, including `the_index_agrees_with_a_scan`. +- `NE` rows kept → `dereko_rows_are_parsed_by_tag_and_summed`. +- 4096 rank cut removed → `the_english_dictionary_is_deepnsm_v1_plus_academic` + (words ranked past 4096 and absent from the academic list leak in). +- Academic list dropped → the same test (academic-only words missing). +- German 20k truncation removed → `german_keeps_the_top_20k_forms`. ## Open diff --git a/crates/cognitive-shader-driver/examples/README.md b/crates/cognitive-shader-driver/examples/README.md index d97873471..8caa26ec8 100644 --- a/crates/cognitive-shader-driver/examples/README.md +++ b/crates/cognitive-shader-driver/examples/README.md @@ -429,10 +429,11 @@ cargo run --release -p cognitive-shader-driver --example crossword_population_fo ## crossword_real_words_probe.rs -D-PUZZLE-0 step 2b. The step-2 crossword with a real dictionary: the 1000 most -frequent words per slot length, English from the committed COCA -`crates/deepnsm/word_frequency/word_forms.csv`, German from DeReKo-2014 only -when `DEREKO_PATH` names the file (CC BY-NC 3.0; nothing derived is committed). +D-PUZZLE-0 step 2b. The step-2 crossword with a real dictionary: every 4- and +5-letter word of DeepNSM's English vocabularies (v1's 4096-word +`word_rank_lookup.csv` plus v2's `academic_20k.csv`, both committed), and of +the 20,000 most frequent DeReKo-2014 German forms, read only when +`DEREKO_PATH` names the file (CC BY-NC 3.0; nothing derived is committed). Instances are random fills, givens added until the fill is unique. About 1.0M edges per language; every count agrees three ways. Board: `.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md`. From 0b1b2354dd5d5e32976ab9d8ec7fcf2d99655de6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:29:33 +0000 Subject: [PATCH 8/9] Board: distinct entry ids for the two crossword steps entries_index.py --check refuses three rows with id D-PUZZLE-0 (the merged Sudoku entry plus both crossword entries). The crossword steps become D-PUZZLE-0-XWORD and D-PUZZLE-0-XWORD-REAL, in the headings and the index. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .../board/entries/2026-10-07-puzzle-0-crossword-real-words.md | 2 +- .../entries/2026-10-07-puzzle-0-crossword-same-algebra.md | 2 +- .claude/board/entries/README.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md b/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md index da58cb955..5c43cc6e2 100644 --- a/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md +++ b/.claude/board/entries/2026-10-07-puzzle-0-crossword-real-words.md @@ -1,4 +1,4 @@ -# 2026-10-07 — D-PUZZLE-0 step 2b: real-word crosswords (DeepNSM vocabularies, optional DeReKo) +# 2026-10-07 — D-PUZZLE-0-XWORD-REAL (D-PUZZLE-0 step 2b): real-word crosswords (DeepNSM vocabularies, optional DeReKo) **Probe:** `cognitive-shader-driver/examples/crossword_real_words_probe.rs`, over `examples/shared/population_fold.rs` (unchanged). diff --git a/.claude/board/entries/2026-10-07-puzzle-0-crossword-same-algebra.md b/.claude/board/entries/2026-10-07-puzzle-0-crossword-same-algebra.md index f3018b7b7..a77f8437a 100644 --- a/.claude/board/entries/2026-10-07-puzzle-0-crossword-same-algebra.md +++ b/.claude/board/entries/2026-10-07-puzzle-0-crossword-same-algebra.md @@ -1,4 +1,4 @@ -# 2026-10-07 — D-PUZZLE-0 step 2: crosswords on the same algebra as Sudoku +# 2026-10-07 — D-PUZZLE-0-XWORD (D-PUZZLE-0 step 2): crosswords on the same algebra as Sudoku **Probes:** `cognitive-shader-driver/examples/crossword_population_fold_probe.rs`, `sudoku_population_fold_probe.rs`, both over diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 5e51b6bc5..704e3e0f6 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -32,8 +32,8 @@ exactly one of them, never both. | 2026-10-07 | `tinker-janus-fold-harvest` | TinkerPop bulk = K (GroupReduce), ONE_BULK = S, GValue pinning = bundle invalidation; JanusGraph slice = OrderedLaneWitness→Range (30–38×); no new V4 op | [2026-10-07-tinker-janus-fold-harvest.md](2026-10-07-tinker-janus-fold-harvest.md) | | 2026-10-07 | `selector-seq-aware-coverage` | | [2026-10-07-selector-seq-aware-coverage.md](2026-10-07-selector-seq-aware-coverage.md) | | 2026-10-07 | `D-PUZZLE-0` | | [2026-10-07-puzzle-0-sudoku-population-fold.md](2026-10-07-puzzle-0-sudoku-population-fold.md) | -| 2026-10-07 | `D-PUZZLE-0` | | [2026-10-07-puzzle-0-crossword-same-algebra.md](2026-10-07-puzzle-0-crossword-same-algebra.md) | -| 2026-10-07 | `D-PUZZLE-0` | | [2026-10-07-puzzle-0-crossword-real-words.md](2026-10-07-puzzle-0-crossword-real-words.md) | +| 2026-10-07 | `D-PUZZLE-0-XWORD` | | [2026-10-07-puzzle-0-crossword-same-algebra.md](2026-10-07-puzzle-0-crossword-same-algebra.md) | +| 2026-10-07 | `D-PUZZLE-0-XWORD-REAL` | | [2026-10-07-puzzle-0-crossword-real-words.md](2026-10-07-puzzle-0-crossword-real-words.md) | | 2026-10-07 | `gated-gather-bit-schedule` | Production gather is 3.8× slow from a per-row branch; bit-gated visitation wins or ties at every density; ordinal vectors never win (R1 holds) | [2026-10-07-gated-gather-bit-schedule.md](2026-10-07-gated-gather-bit-schedule.md) | | 2026-10-07 | `D-EPI-MIG-INVENTORY` | | [2026-10-07-epi-mig-inventory.md](2026-10-07-epi-mig-inventory.md) | | 2026-10-07 | `D-EPI-CANON-0` | | [2026-10-07-epi-mig-0.md](2026-10-07-epi-mig-0.md) | From 94499c3316ddb720638fd3707cb1dbf6382e4b41 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 16:39:59 +0000 Subject: [PATCH 9/9] D-PUZZLE-0 step 2b: NYT-rule grids and the crossword-answer lexicon Grids: alongside the fixed step-2 template, a fresh random grid per instance under the NYT rules (180-degree symmetry, every word at least 3 letters, one connected white region, at most a sixth black). The rules are written from their statement, not from akshayravikumar/crosswords (no licence). Each rule has its own refusing fixture. Lexicon: answers of the Down-and-Across clue/answer set (text-machine-lab/xword_benchmark), read only from XWORD_ANSWERS_DIR at run time; the repo has no licence and the answers come from NYT puzzles, so nothing is committed. Word splits are rejoined, repeats counted. The asserted-claims-per-group check now follows each grid's own slot count. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0177nSSyQaXqbNuGR1vMXXN6 --- .../examples/crossword_real_words_probe.rs | 475 ++++++++++++++---- 1 file changed, 381 insertions(+), 94 deletions(-) diff --git a/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs b/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs index 22be8431e..c79b7f2e4 100644 --- a/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs +++ b/crates/cognitive-shader-driver/examples/crossword_real_words_probe.rs @@ -41,8 +41,13 @@ use population_fold::{ /// Same class as step 2: the same domain, a different dictionary. const CROSSWORD_CLASS: ClassId = 0x0907; -const SIDE: usize = 5; -const TEMPLATE: [&str; SIDE] = ["....#", ".....", ".....", ".....", "#...."]; +/// The step-2 template: two black squares, ten crossing slots. +const TEMPLATE: [&str; 5] = ["....#", ".....", ".....", ".....", "#...."]; +/// NYT grid rules: shortest word, and the most black squares (a sixth). +const MIN_WORD: usize = 3; +const MAX_BLACK_DIVISOR: usize = 6; +/// Side of the random NYT-rule grids. +const NYT_SIDE: usize = 7; /// v1's vocabulary size (`deepnsm::vocabulary::VOCAB_SIZE`). const V1_VOCAB: u32 = 4096; /// Forms kept from the German frequency list. @@ -54,28 +59,154 @@ const RANK_LOOKUP: &str = include_str!("../../deepnsm/word_frequency/word_rank_l const ACADEMIC: &str = include_str!("../../deepnsm/word_frequency/academic_20k.csv"); type Word = Vec; -type Fill = [Option; SIDE * SIDE]; - -fn slots() -> Vec> { - let white = |r: usize, c: usize| TEMPLATE[r].as_bytes()[c] == b'.'; - let mut out = Vec::new(); - for across in [true, false] { - for line in 0..SIDE { - let mut run = Vec::new(); - for k in 0..=SIDE { - let (r, c) = if across { (line, k) } else { (k, line) }; - if k < SIDE && white(r, c) { - run.push(r * SIDE + c); - } else { - if run.len() >= 2 { - out.push(run.clone()); +/// Letters fixed so far, one per square (`None` = open or black). +type Fill = Vec>; + +/// A square grid: its side and which squares are white. +#[derive(Clone, Debug, PartialEq)] +struct Grid { + side: usize, + white: Vec, +} + +impl Grid { + fn template() -> Self { + let white = TEMPLATE + .iter() + .flat_map(|r| r.bytes().map(|b| b == b'.')) + .collect(); + Self { + side: TEMPLATE.len(), + white, + } + } + + fn cells(&self) -> usize { + self.side * self.side + } + + /// Every maximal white run, across then down. + fn runs(&self) -> Vec> { + let n = self.side; + let mut out = Vec::new(); + for across in [true, false] { + for line in 0..n { + let mut run = Vec::new(); + for k in 0..=n { + let (r, c) = if across { (line, k) } else { (k, line) }; + if k < n && self.white[r * n + c] { + run.push(r * n + c); + } else { + if !run.is_empty() { + out.push(run.clone()); + } + run.clear(); } - run.clear(); } } } + out + } + + /// The word slots: runs of two or more squares. + fn slots(&self) -> Vec> { + self.runs().into_iter().filter(|r| r.len() >= 2).collect() + } + + fn blacks(&self) -> usize { + self.white.iter().filter(|&&w| !w).count() + } + + /// 180° rotational symmetry: square `i` and square `cells - 1 - i` agree. + fn symmetric(&self) -> bool { + let n = self.cells(); + (0..n).all(|i| self.white[i] == self.white[n - 1 - i]) + } + + /// The white squares form one 4-connected region. + fn connected(&self) -> bool { + let n = self.side; + let Some(start) = self.white.iter().position(|&w| w) else { + return false; + }; + let mut seen = vec![false; self.cells()]; + let mut stack = vec![start]; + seen[start] = true; + while let Some(i) = stack.pop() { + let (r, c) = (i / n, i % n); + let up = (r > 0).then(|| i - n); + let down = (r + 1 < n).then(|| i + n); + let left = (c > 0).then(|| i - 1); + let right = (c + 1 < n).then(|| i + 1); + for j in [up, down, left, right].into_iter().flatten() { + if self.white[j] && !seen[j] { + seen[j] = true; + stack.push(j); + } + } + } + (0..self.cells()).all(|i| !self.white[i] || seen[i]) + } + + /// The NYT grid rules: rotational symmetry, every word at least + /// `MIN_WORD` letters (so every white square is checked), one connected + /// white region, at most a sixth of the squares black. + fn is_nyt_valid(&self) -> bool { + self.symmetric() + && self.runs().iter().all(|r| r.len() >= MIN_WORD) + && self.connected() + && self.blacks() * MAX_BLACK_DIVISOR <= self.cells() + } + + /// A random grid obeying the NYT rules: symmetric black-square pairs drawn + /// until the grid is valid and has at least one black square. + fn random_nyt(side: usize, rng: &mut Rng) -> Self { + let cells = side * side; + loop { + let mut white = vec![true; cells]; + for i in 0..cells.div_ceil(2) { + if rng.below(8) == 0 { + white[i] = false; + white[cells - 1 - i] = false; + } + } + let g = Self { side, white }; + if g.blacks() > 0 && g.is_nyt_valid() { + return g; + } + } + } +} + +/// Where each instance's grid comes from. +#[derive(Clone, Copy, Debug)] +enum Grids { + /// The fixed step-2 template. + Template, + /// A fresh random NYT-rule grid of this side per instance. + Nyt(usize), +} + +impl Grids { + fn draw(self, rng: &mut Rng) -> Grid { + match self { + Grids::Template => Grid::template(), + Grids::Nyt(side) => Grid::random_nyt(side, rng), + } + } + + /// Every slot length an instance can need. + fn lengths(self) -> Vec { + match self { + Grids::Template => { + let mut l: Vec = Grid::template().slots().iter().map(Vec::len).collect(); + l.sort_unstable(); + l.dedup(); + l + } + Grids::Nyt(side) => (MIN_WORD..=side).collect(), + } } - out } /// Insert `word` with `freq`, keeping the larger frequency, if it is @@ -133,6 +264,28 @@ fn dereko_frequencies(text: &str) -> HashMap { out } +/// Crossword answers from the Down-and-Across clue/answer files +/// (`*.target`, one answer per line, multi-word answers space-split by the +/// dataset's own post-processing). Spaces are removed to recover the grid +/// string; each answer is scored by how often it occurs. +fn xword_answers(targets: &[String]) -> HashMap { + let mut out: HashMap = HashMap::new(); + for text in targets { + for line in text.lines() { + let w: String = line + .chars() + .filter(|c| !c.is_whitespace()) + .collect::() + .to_lowercase(); + if w.is_empty() || !w.chars().all(|c| c.is_ascii_alphabetic()) { + continue; + } + *out.entry(w).or_insert(0.0) += 1.0; + } + } + out +} + /// The `GERMAN_VOCAB` most frequent DeReKo forms (ties alphabetical). fn german_vocabulary(text: &str) -> HashMap { let f = dereko_frequencies(text); @@ -208,12 +361,10 @@ struct Dictionary { } impl Dictionary { - fn new(freq: &HashMap, slots: &[Vec]) -> Self { - let mut lens: Vec = slots.iter().map(Vec::len).collect(); - lens.sort_unstable(); - lens.dedup(); - let by_len = lens - .into_iter() + fn new(freq: &HashMap, lengths: &[usize]) -> Self { + let by_len = lengths + .iter() + .copied() .map(|l| (l, Bucket::new(top_words(freq, l, usize::MAX), l))) .collect(); Self { by_len } @@ -259,7 +410,7 @@ fn count_fills( let mut n = 0; done[s] = true; for w in dict.candidates(&slots[s], fill) { - let saved = *fill; + let saved = fill.clone(); place(w, &slots[s], fill); n += count_fills(dict, slots, done, fill, cap - n); *fill = saved; @@ -272,7 +423,12 @@ fn count_fills( } /// One random complete fill, or `None` when the node budget runs out. -fn random_fill(dict: &Dictionary, slots: &[Vec], rng: &mut Rng) -> Option> { +fn random_fill( + dict: &Dictionary, + slots: &[Vec], + cells: usize, + rng: &mut Rng, +) -> Option> { fn go( dict: &Dictionary, slots: &[Vec], @@ -292,7 +448,7 @@ fn random_fill(dict: &Dictionary, slots: &[Vec], rng: &mut Rng) -> Option break; } *budget -= 1; - let saved = *fill; + let saved = fill.clone(); place(w, &slots[s], fill); if go(dict, slots, done, fill, rng, budget) { return true; @@ -303,7 +459,7 @@ fn random_fill(dict: &Dictionary, slots: &[Vec], rng: &mut Rng) -> Option false } let mut done = vec![false; slots.len()]; - let mut fill: Fill = [None; SIDE * SIDE]; + let mut fill: Fill = vec![None; cells]; let mut budget = FILL_BUDGET; go(dict, slots, &mut done, &mut fill, rng, &mut budget).then(|| { slots @@ -314,39 +470,55 @@ fn random_fill(dict: &Dictionary, slots: &[Vec], rng: &mut Rng) -> Option } struct Instance { + grid: Grid, + slots: Vec>, given: Vec, solution: Vec, } -/// Number of fills consistent with the instance's givens, up to `cap`. -fn count_solutions(dict: &Dictionary, slots: &[Vec], inst: &Instance, cap: usize) -> usize { - let mut fill: Fill = [None; SIDE * SIDE]; - let mut done = inst.given.clone(); - for (s, slot) in slots.iter().enumerate() { - if inst.given[s] { - place(&inst.solution[s], slot, &mut fill); +impl Instance { + /// The fill holding every given slot's true word. + fn given_fill(&self) -> Fill { + let mut fill: Fill = vec![None; self.grid.cells()]; + for (s, slot) in self.slots.iter().enumerate() { + if self.given[s] { + place(&self.solution[s], slot, &mut fill); + } } + fill } - count_fills(dict, slots, &mut done, &mut fill, cap) +} + +/// Number of fills consistent with the instance's givens, up to `cap`. +fn count_solutions(dict: &Dictionary, inst: &Instance, cap: usize) -> usize { + let mut fill = inst.given_fill(); + let mut done = inst.given.clone(); + count_fills(dict, &inst.slots, &mut done, &mut fill, cap) } /// A random fill, then given slots added in random order until it is the only /// fill the dictionary admits. Every placed word must be a dictionary word, so /// the solution is checked against the dictionary too. -fn draw_instance(dict: &Dictionary, slots: &[Vec], rng: &mut Rng) -> Instance { +/// A grid comes fresh from `grids` on every attempt, so a grid the +/// dictionary cannot fill within the budget is simply replaced. +fn draw_instance(dict: &Dictionary, grids: Grids, rng: &mut Rng) -> Instance { loop { - let Some(solution) = random_fill(dict, slots, rng) else { + let grid = grids.draw(rng); + let slots = grid.slots(); + let Some(solution) = random_fill(dict, &slots, grid.cells(), rng) else { continue; }; let mut order: Vec = (0..slots.len()).collect(); rng.shuffle(&mut order); let mut inst = Instance { given: vec![false; slots.len()], + grid, + slots, solution, }; for &s in &order { inst.given[s] = true; - if count_solutions(dict, slots, &inst, 2) == 1 { + if count_solutions(dict, &inst, 2) == 1 { return inst; } } @@ -354,20 +526,10 @@ fn draw_instance(dict: &Dictionary, slots: &[Vec], rng: &mut Rng) -> Inst } } -fn propagate( - dict: &Dictionary, - slots: &[Vec], - inst: &Instance, - depth: usize, -) -> (Vec>, Fill) { - let mut placed: Vec> = vec![None; slots.len()]; - let mut fill: Fill = [None; SIDE * SIDE]; - for (s, slot) in slots.iter().enumerate() { - if inst.given[s] { - place(&inst.solution[s], slot, &mut fill); - placed[s] = Some(true); - } - } +fn propagate(dict: &Dictionary, inst: &Instance, depth: usize) -> (Vec>, Fill) { + let slots = &inst.slots; + let mut fill = inst.given_fill(); + let mut placed: Vec> = inst.given.iter().map(|&g| g.then_some(true)).collect(); let mut forced = 0; while forced < depth { let next = (0..slots.len()).find_map(|s| { @@ -389,20 +551,24 @@ fn propagate( #[derive(Default)] struct Stats { givens: Vec, + slots: Vec, + blacks: Vec, forced: usize, } -fn build_lane(dict: &Dictionary, min_edges: usize, seed: u64) -> (Lane, Stats) { - let slots = slots(); +fn build_lane(dict: &Dictionary, grids: Grids, min_edges: usize, seed: u64) -> (Lane, Stats) { let mut rng = Rng(seed); let mut lane = Lane::default(); let mut stats = Stats::default(); while lane.edges.len() < min_edges { - let inst = draw_instance(dict, &slots, &mut rng); + let inst = draw_instance(dict, grids, &mut rng); + let slots = &inst.slots; let g = inst.given.iter().filter(|&&b| b).count(); stats.givens.push(g); + stats.slots.push(slots.len()); + stats.blacks.push(inst.grid.blacks()); let depth = rng.below((slots.len() - g) as u64 + 1) as usize; - let (placed, fill) = propagate(dict, &slots, &inst, depth); + let (placed, fill) = propagate(dict, &inst, depth); for (s, slot) in slots.iter().enumerate() { match placed[s] { Some(true) => { @@ -435,16 +601,17 @@ fn build_lane(dict: &Dictionary, min_edges: usize, seed: u64) -> (Lane, Stats) { (lane, stats) } -fn run(name: &str, dict: &Dictionary) { +fn run(name: &str, freq: &HashMap, grids: Grids) { let decl = declarations(CROSSWORD_CLASS); admit(&decl, CROSSWORD_CLASS).expect("the crossword class declares the canonical reading"); + let dict = &Dictionary::new(freq, &grids.lengths()); let t = Instant::now(); - let (lane, stats) = build_lane(dict, 1_000_000, 0xC0_CA_u64); - let mut hist = [0usize; 11]; + let (lane, stats) = build_lane(dict, grids, 1_000_000, 0xC0_CA_u64); + let mut hist = vec![0usize; stats.slots.iter().max().unwrap() + 1]; for &g in &stats.givens { hist[g] += 1; } - println!("D-PUZZLE-0 / real-word crosswords: {name}"); + println!("D-PUZZLE-0 / real-word crosswords: {name}, grids {grids:?}"); for (len, b) in { let mut v: Vec<_> = dict.by_len.iter().collect(); v.sort_by_key(|(l, _)| **l); @@ -468,28 +635,53 @@ fn run(name: &str, dict: &Dictionary) { " givens needed for uniqueness (givens:instances): {}", spread.join(" ") ); - let counts = check_three_ways(&lane, &decl, CROSSWORD_CLASS); - assert!( - per_group(&lane, GIVEN.bit() | FORCED.bit() | ENTAILED.bit()) - .iter() - .all(|&n| n == 10) + let mean = |v: &[usize]| v.iter().sum::() as f64 / v.len() as f64; + println!( + " per grid: {:.1} slots, {:.1} black squares on average", + mean(&stats.slots), + mean(&stats.blacks) ); - println!(" every instance: exactly 10 asserted claims, one per slot (group fold)"); + let counts = check_three_ways(&lane, &decl, CROSSWORD_CLASS); + let asserted: Vec = per_group(&lane, GIVEN.bit() | FORCED.bit() | ENTAILED.bit()) + .into_iter() + .map(|n| n as usize) + .collect(); + assert_eq!(asserted, stats.slots); + println!(" every instance: exactly one asserted claim per slot (group fold)"); report(&lane, &decl, CROSSWORD_CLASS, &counts); } fn main() { - let slots = slots(); - run( - "English, DeepNSM v1 4096 + academic 20k (committed)", - &Dictionary::new(&english_vocabulary(), &slots), - ); + let english = english_vocabulary(); + let name = "English, DeepNSM v1 4096 + academic 20k (committed)"; + run(name, &english, Grids::Template); + run(name, &english, Grids::Nyt(NYT_SIDE)); + match std::env::var("XWORD_ANSWERS_DIR") { + Ok(dir) => { + let targets: Vec = ["train", "val", "test"] + .iter() + .map(|s| { + std::fs::read_to_string(format!("{dir}/{s}.target")) + .expect("XWORD_ANSWERS_DIR holds train/val/test.target") + }) + .collect(); + run( + "crossword answers, Down-and-Across clue set (read at run time)", + &xword_answers(&targets), + Grids::Nyt(NYT_SIDE), + ); + } + Err(_) => println!( + "Crossword-answer run skipped: set XWORD_ANSWERS_DIR to the folder of *.target files" + ), + } match std::env::var("DEREKO_PATH") { Ok(path) => { let text = std::fs::read_to_string(&path).expect("DEREKO_PATH is readable"); run( "German, DeReKo-2014 top 20k (CC BY-NC 3.0, read at run time)", - &Dictionary::new(&german_vocabulary(&text), &slots), + &german_vocabulary(&text), + Grids::Template, ); } Err(_) => println!("German run skipped: set DEREKO_PATH to the DeReKo-2014 .freq file"), @@ -504,7 +696,7 @@ mod tests { use lance_graph_contract::epistemic_state5::facts_population; fn coca() -> Dictionary { - Dictionary::new(&english_vocabulary(), &slots()) + Dictionary::new(&english_vocabulary(), &Grids::Nyt(NYT_SIDE).lengths()) } /// The English dictionary is exactly DeepNSM's two vocabularies. Each @@ -540,10 +732,11 @@ mod tests { #[test] fn the_index_agrees_with_a_scan() { let d = coca(); - let slots = slots(); let mut rng = Rng(3); for _ in 0..200 { - let mut fill: Fill = [None; SIDE * SIDE]; + let grid = Grid::random_nyt(NYT_SIDE, &mut rng); + let slots = grid.slots(); + let mut fill: Fill = vec![None; grid.cells()]; for sq in fill.iter_mut() { if rng.below(4) == 0 { *sq = Some((b'a' + rng.below(26) as u8) as char); @@ -564,14 +757,19 @@ mod tests { } fn small_lane() -> (Lane, Stats) { - build_lane(&coca(), 30_000, 0x7E57) + build_lane(&coca(), Grids::Template, 30_000, 0x7E57) + } + + fn nyt_lane() -> (Lane, Stats) { + build_lane(&coca(), Grids::Nyt(NYT_SIDE), 30_000, 0x7E58) } #[test] fn every_question_counts_the_same_three_ways() { - let (lane, _) = small_lane(); - let counts = check_three_ways(&lane, &declarations(CROSSWORD_CLASS), CROSSWORD_CLASS); - assert!(counts.iter().all(|&n| n > 0), "a question went unexercised"); + for (lane, _) in [small_lane(), nyt_lane()] { + let counts = check_three_ways(&lane, &declarations(CROSSWORD_CLASS), CROSSWORD_CLASS); + assert!(counts.iter().all(|&n| n > 0), "a question went unexercised"); + } } #[test] @@ -590,6 +788,7 @@ mod tests { assert!(per_group(&lane, facts_population(CAUSES)) .iter() .all(|&n| n == 10)); + assert!(stats.slots.iter().all(|&n| n == 10)); let given: Vec = per_group(&lane, facts_population(DIRECT | CAUSES)) .into_iter() .map(|n| n as usize) @@ -608,21 +807,24 @@ mod tests { #[test] fn instances_are_unique_and_givens_are_needed() { let d = coca(); - let slots = slots(); let mut rng = Rng(11); let mut ambiguous_without = 0; - for _ in 0..20 { - let inst = draw_instance(&d, &slots, &mut rng); - assert_eq!(count_solutions(&d, &slots, &inst, 3), 1); - for (s, w) in inst.solution.iter().enumerate() { - assert!(d.bucket(&slots[s]).words.contains(w)); - } - let open = Instance { - given: vec![false; slots.len()], - solution: inst.solution.clone(), - }; - if count_solutions(&d, &slots, &open, 2) > 1 { - ambiguous_without += 1; + for grids in [Grids::Template, Grids::Nyt(NYT_SIDE)] { + for _ in 0..10 { + let inst = draw_instance(&d, grids, &mut rng); + assert_eq!(count_solutions(&d, &inst, 3), 1); + for (s, w) in inst.solution.iter().enumerate() { + assert!(d.bucket(&inst.slots[s]).words.contains(w)); + } + let open = Instance { + grid: inst.grid.clone(), + slots: inst.slots.clone(), + given: vec![false; inst.slots.len()], + solution: inst.solution.clone(), + }; + if count_solutions(&d, &open, 2) > 1 { + ambiguous_without += 1; + } } } assert!(ambiguous_without > 0, "uniqueness never needed a given"); @@ -662,6 +864,91 @@ mod tests { assert!(!v.contains_key("caaae"), "form ranked past 20k kept"); // i = 20004 } + fn grid(rows: &[&str]) -> Grid { + Grid { + side: rows.len(), + white: rows + .iter() + .flat_map(|r| r.bytes().map(|b| b == b'.')) + .collect(), + } + } + + /// Each NYT rule refuses on its own: the step-2 template passes, and one + /// grid per rule breaks exactly that rule. + #[test] + fn each_nyt_rule_refuses_its_own_violation() { + assert!(Grid::template().is_nyt_valid()); + let ok = grid(&[ + "...#...", ".......", ".......", "#.....#", ".......", ".......", "...#...", + ]); + assert!(ok.is_nyt_valid()); + // asymmetric: one black square without its mirror + let asym = grid(&[ + "...#...", ".......", ".......", ".......", ".......", ".......", ".......", + ]); + assert!(!asym.symmetric() && !asym.is_nyt_valid()); + // a two-letter word + let short = grid(&[ + "..#....", ".......", ".......", ".......", ".......", ".......", "....#..", + ]); + assert!(short.symmetric() && !short.is_nyt_valid()); + // two white regions: a full black column cuts the grid + let cut = grid(&[ + "...#...", "...#...", "...#...", "...#...", "...#...", "...#...", "...#...", + ]); + assert!(cut.symmetric() && !cut.connected() && !cut.is_nyt_valid()); + // too many black squares (12 of 49 > a sixth), every other rule kept + let heavy = grid(&[ + "...#...", "...#...", "###.###", ".......", "###.###", "...#...", "...#...", + ]); + assert!(heavy.symmetric() && heavy.blacks() * MAX_BLACK_DIVISOR > heavy.cells()); + assert!(!heavy.is_nyt_valid()); + } + + /// Random NYT grids are valid, carry black squares, and actually vary. + #[test] + fn random_nyt_grids_are_valid_and_vary() { + let mut rng = Rng(5); + let grids: Vec = (0..50) + .map(|_| Grid::random_nyt(NYT_SIDE, &mut rng)) + .collect(); + assert!(grids.iter().all(|g| g.is_nyt_valid() && g.blacks() > 0)); + let unique: std::collections::HashSet> = + grids.iter().map(|g| g.white.clone()).collect(); + assert!(unique.len() > 10, "only {} distinct grids", unique.len()); + } + + /// On NYT grids every instance still holds one asserted claim per slot, + /// and the slot count follows the grid. + #[test] + fn nyt_lane_holds_one_claim_per_slot_of_its_own_grid() { + let (lane, stats) = nyt_lane(); + let asserted: Vec = per_group(&lane, facts_population(CAUSES)) + .into_iter() + .map(|n| n as usize) + .collect(); + assert_eq!(asserted, stats.slots); + let lo = stats.slots.iter().min().unwrap(); + let hi = stats.slots.iter().max().unwrap(); + assert!(lo < hi, "every grid had the same number of slots"); + } + + /// Crossword answers: the dataset's word splits are rejoined, punctuation + /// is refused, repeats are counted. + #[test] + fn crossword_answers_are_rejoined_and_counted() { + let targets = vec![ + "egg white\nlimos\nen nea\n".to_string(), + "limos\nst. louis\n".to_string(), + ]; + let a = xword_answers(&targets); + assert_eq!(a.get("eggwhite"), Some(&1.0)); + assert_eq!(a.get("ennea"), Some(&1.0)); + assert_eq!(a.get("limos"), Some(&2.0)); + assert_eq!(a.len(), 3); + } + #[test] fn an_undeclared_lane_is_refused_before_any_edge() { let decl = declarations(CROSSWORD_CLASS);