Skip to content

Commit 03f6f27

Browse files
committed
hdr: harvest the rolling floor from lance-graph onto exact moments
Adds hpc::rolling_floor, the adaptive half of the HDR exposure meter, harvested from lance-graph graph/blasgraph/hdr.rs (the behavioural reference). Constants, cadence, drift rule and reset semantics carry over unchanged; the arithmetic underneath is #327's exact MomentsU32 instead of the reference's approximate integer Welford. - isqrt_u32: the reference integer Newton square root. - ReservoirU32: deterministic Algorithm-R reservoir (splitmix replacement hash keyed on the observation count), u32 empirical quantile, Pearson second skewness, kurtosis x100. Order-defined; no merge law. - RollingFloor: calibrated (mu, sigma), sigma floors mu - k sigma, empirical floors at the reference percentiles, shape evaluation every 1000 observations after the first 1000 (reservoir >= 100), normal iff |skew| < 2 and 200 < kurt < 500, drift at |dmu| > sigma/2 or |dsigma| > sigma/4, recalibration resets moments, reservoir and shape. - observe_batch folds moments_u32 between checkpoints and stops after the first shift, so any batching reproduces the scalar observe/recalibrate loop exactly. - MomentsU32::observe: scalar fold equal to merging a singleton batch. A test keeps the legacy integer Welford as an oracle: across 95 checkpoints on five streams the (mu, sigma) the drift rule sees are identical. examples/hdr_rolling_floor_bench.rs separates the hot path from the periodic shape path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019HnekoM1EidTwQLS3oFVFm
1 parent 031f5c8 commit 03f6f27

4 files changed

Lines changed: 1046 additions & 0 deletions

File tree

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
//! Cost split of the HDR rolling floor: the per-observation hot path versus
2+
//! the periodic shape path.
3+
//!
4+
//! ```sh
5+
//! cargo run --release --example hdr_rolling_floor_bench
6+
//! ```
7+
//!
8+
//! Hot path, per observation:
9+
//! 1. popcount / Hamming distance of two 2048-byte vectors
10+
//! 2. exact moments update (`MomentsU32::observe`, and `moments_u32` batch)
11+
//! 3. reservoir update
12+
//! 4. full rolling-floor update (moments + reservoir + checkpoint test)
13+
//!
14+
//! Periodic path, once per 1000 observations:
15+
//! 5. shape evaluation (sort 1000 samples, median, kurtosis)
16+
//! 6. empirical quantiles (12 lookups into the sorted reservoir)
17+
18+
use ndarray::hpc::bitwise::hamming_distance_raw;
19+
use ndarray::hpc::rolling_floor::{quantile_of_sorted, ReservoirU32, RollingFloor};
20+
use ndarray::hpc::statistics::{moments_u32, MomentsU32};
21+
use std::hint::black_box;
22+
use std::time::Instant;
23+
24+
fn xorshift(n: usize, mut s: u64) -> Vec<u64> {
25+
(0..n)
26+
.map(|_| {
27+
s ^= s << 13;
28+
s ^= s >> 7;
29+
s ^= s << 17;
30+
s
31+
})
32+
.collect()
33+
}
34+
35+
fn ns_per(label: &str, n: usize, f: impl FnOnce()) {
36+
let t = Instant::now();
37+
f();
38+
let dt = t.elapsed();
39+
println!("{label:<44} {:>9.2} ns/op ({n} ops)", dt.as_nanos() as f64 / n as f64);
40+
}
41+
42+
fn main() {
43+
println!("avx512f={} avx2={}", cfg!(target_feature = "avx512f"), cfg!(target_feature = "avx2"));
44+
45+
const VBYTES: usize = 2048; // 16384-bit vectors
46+
const N: usize = 2_000_000;
47+
let words = xorshift(VBYTES / 8 * 65, 1);
48+
let bytes: Vec<u8> = words.iter().flat_map(|w| w.to_le_bytes()).collect();
49+
let query = &bytes[..VBYTES];
50+
let db = &bytes[VBYTES..];
51+
let dists: Vec<u32> = (0..N)
52+
.map(|i| {
53+
let j = i % 64;
54+
hamming_distance_raw(query, &db[j * VBYTES..(j + 1) * VBYTES]) as u32
55+
})
56+
.collect();
57+
58+
println!("-- hot path, per observation --");
59+
ns_per("1. popcount/Hamming (2048 B)", N, || {
60+
let mut acc = 0u64;
61+
for i in 0..N {
62+
let j = i % 64;
63+
acc += hamming_distance_raw(black_box(query), &db[j * VBYTES..(j + 1) * VBYTES]);
64+
}
65+
black_box(acc);
66+
});
67+
ns_per("2a. MomentsU32::observe (scalar)", N, || {
68+
let mut m = MomentsU32::default();
69+
for &d in &dists {
70+
m.observe(black_box(d));
71+
}
72+
black_box(m);
73+
});
74+
ns_per("2b. moments_u32 (batch)", N, || {
75+
black_box(moments_u32(black_box(&dists)));
76+
});
77+
ns_per("3. ReservoirU32::observe (cap 1000)", N, || {
78+
let mut r = ReservoirU32::new(1000);
79+
for &d in &dists {
80+
r.observe(black_box(d));
81+
}
82+
black_box(r.len());
83+
});
84+
ns_per("4a. RollingFloor::observe (incl. checkpoints)", N, || {
85+
let mut f = RollingFloor::for_width(16384);
86+
for &d in &dists {
87+
if let Some(s) = f.observe(black_box(d)) {
88+
f.recalibrate(&s);
89+
}
90+
}
91+
black_box(f.mu());
92+
});
93+
ns_per("4b. RollingFloor::observe_batch (incl. checkpoints)", N, || {
94+
let mut f = RollingFloor::for_width(16384);
95+
let mut rest: &[u32] = &dists;
96+
while !rest.is_empty() {
97+
let (used, s) = f.observe_batch(rest);
98+
rest = &rest[used..];
99+
if let Some(s) = s {
100+
f.recalibrate(&s);
101+
}
102+
}
103+
black_box(f.mu());
104+
});
105+
106+
println!("-- periodic path, per checkpoint (every 1000 observations) --");
107+
let mut r = ReservoirU32::new(1000);
108+
dists[..5000].iter().for_each(|&d| r.observe(d));
109+
const K: usize = 20_000;
110+
ns_per("5. shape: sort + median + kurtosis", K, || {
111+
for _ in 0..K {
112+
let sorted = black_box(&r).sorted();
113+
black_box(quantile_of_sorted(&sorted, 0.5));
114+
black_box(r.kurtosis(8192, 64));
115+
}
116+
});
117+
let sorted = r.sorted();
118+
ns_per("6. empirical floors: 12 quantiles (sorted)", K, || {
119+
for _ in 0..K {
120+
black_box(RollingFloor::FLOOR_PERCENTILES.map(|p| quantile_of_sorted(black_box(&sorted), p)));
121+
black_box(RollingFloor::CASCADE_PERCENTILES.map(|p| quantile_of_sorted(black_box(&sorted), p)));
122+
}
123+
});
124+
}

‎src/hpc/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ pub mod blas_level2;
2525
pub mod blas_level3;
2626
pub mod reductions;
2727
pub mod statistics;
28+
pub mod rolling_floor;
2829
/// Reliability & validity statistics: Pearson r, Spearman ρ, Cronbach α, ICC.
2930
pub mod reliability;
3031
/// Entropy ladder: Staunen↔Wisdom coordinate over NARS truth + Pearl-2³ SPO.

0 commit comments

Comments
 (0)