From a1d0dcf9d088062a740d7d257ef574539809f146 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 4 Oct 2026 00:46:00 +1000 Subject: [PATCH 1/2] perf(bench): add a fixed-iteration Ir/op gate for the hot paths (LAB-7802) Wall clock cannot gate a 1% CPU regression on a shared machine, and the Criterion suite picks its iteration count from wall time, so its total under valgrind cannot be divided by a known count. benches/perf_ir.rs runs each case in its own process under cachegrind, once with 1,000 calls and once with none, after the same setup and one warm-up call: Ir/op = (Ir[1000] - Ir[0]) / 1000. Cases cover ByteStorage store/retrieve, the envelope pre-scan, AES-256-GCM encrypt/decrypt, the keyring decrypt (HKDF per call), the tenant keyring decrypt, and HKDF, at 64 B, 1 KiB and 64 KiB. `make perf-ir` compares against benches/perf_ir_baselines.json (fail at +1%, warn from +0.2%); `make perf-ir-update` ratchets budgets down, never up without --allow-increase. Two runs of one build agree exactly; between builds, heap alignment moves encrypt/1024 by up to 0.19% and every other case by at most 0.02%. The target is `bench = false`, so a plain `cargo bench` does not need valgrind, and `clippy --all-targets` still compiles it. hot_path's realistic payload generator moves to benches/common so both targets measure the same bytes. No new dependency. --- Cargo.toml | 9 + Makefile | 13 +- README.md | 25 ++ benches/common/mod.rs | 60 +++++ benches/hot_path.rs | 59 +---- benches/perf_ir.rs | 450 +++++++++++++++++++++++++++++++++ benches/perf_ir_baselines.json | 33 +++ 7 files changed, 592 insertions(+), 57 deletions(-) create mode 100644 benches/common/mod.rs create mode 100644 benches/perf_ir.rs create mode 100644 benches/perf_ir_baselines.json diff --git a/Cargo.toml b/Cargo.toml index 770c5de..458662f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,3 +170,12 @@ debug = "line-tables-only" name = "hot_path" harness = false required-features = ["encryption"] + +# Instruction-count gate (`make perf-ir`). It needs valgrind, so `bench = false` +# keeps it out of a plain `cargo bench`; `--bench perf_ir` still selects it, and +# `clippy --all-targets` still compiles it. +[[bench]] +name = "perf_ir" +harness = false +bench = false +required-features = ["encryption"] diff --git a/Makefile b/Makefile index 752080d..09b3e30 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # cachekit-core - Development Makefile -.PHONY: help check test lint clippy audit deny vet fmt fmt-check bench bench-quick fuzz-quick fuzz-deep sbom clean +.PHONY: help check test lint clippy audit deny vet fmt fmt-check bench bench-quick perf-ir perf-ir-update fuzz-quick fuzz-deep sbom clean .DEFAULT_GOAL := help # Colors for output @@ -84,6 +84,17 @@ bench-quick: ## Quick bench run for CI (1s warmup, 2s measurement, 10 samples) $(call require_binary,cargo,Install Rust: https://rustup.rs) @cargo bench -p cachekit-core --features encryption --bench hot_path -- --warm-up-time 1 --measurement-time 2 --sample-size 10 +# Per-op instruction counts (benches/perf_ir.rs) against the budgets in +# benches/perf_ir_baselines.json. Needs valgrind. Pass gate options through +# ARGS, e.g. make perf-ir ARGS="--case store/1024". See README, "Instruction counts". +perf-ir: ## Instruction-count gate: fail on >=1% Ir/op regression (needs valgrind) + $(call require_binary,cargo,Install Rust: https://rustup.rs) + @cargo bench -p cachekit-core --features encryption --bench perf_ir -- $(ARGS) + +perf-ir-update: ## Ratchet Ir/op budgets down to the measured figures (never up) + $(call require_binary,cargo,Install Rust: https://rustup.rs) + @cargo bench -p cachekit-core --features encryption --bench perf_ir -- --update $(ARGS) + fuzz-quick: ## Quick corpus-only fuzz run (2 min per target) @echo "$(BLUE)Running quick fuzzing (2 min per target)...$(RESET)" $(call require_binary,cargo-fuzz,Install: cargo install cargo-fuzz) diff --git a/README.md b/README.md index c1f61ad..d81a0dd 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,10 @@ cachekit-core/ ├── include/ │ └── cachekit.h # Generated C header │ +├── benches/ +│ ├── hot_path.rs # Criterion wall-clock suite (make bench) +│ └── perf_ir.rs # Per-op instruction counts and their gate (make perf-ir) +│ ├── fuzz/ # Fuzzing targets (16 targets) │ └── fuzz_targets/ │ @@ -329,6 +333,27 @@ Benchmarks on Apple M2 Max (64KB payload, compressible data): The figures above come from `examples/bench_throughput.rs` on highly compressible data. The Criterion suite in `benches/hot_path.rs` needs the `encryption` feature (`make bench`, or `cargo bench --features encryption`; a plain `cargo bench` skips it). It runs the ByteStorage roundtrip on three corpora side by side: `byte_storage/roundtrip` (synthetic ramp, kept for history), `byte_storage/roundtrip_msgpack` (realistic msgpack records, about 0.38 LZ4 ratio at 64 KB) and `byte_storage/roundtrip_incompressible`. The bench profile keeps symbols, so callgrind and perf attribute cost to functions. +### Instruction counts + +`make perf-ir` counts the instructions each hot path executes per call and fails when a case costs **1% or more** above its committed budget. It warns from 0.2%. Wall clock on a shared machine moves by far more than 1% between identical runs, and Criterion picks its iteration count from wall time; instruction counts at a fixed iteration count are exact. It needs valgrind (`apt install valgrind`). + +| Case | What one call does | +|:-----|:-------------------| +| `store/`, `retrieve/` | `ByteStorage::store` / `retrieve` on the realistic msgpack payload `hot_path` uses | +| `prescan/` | `check_msgpack_structure` on the envelope, the pre-scan `retrieve` runs first | +| `encrypt/`, `decrypt/` | `ZeroKnowledgeEncryptor` AES-256-GCM | +| `keyring_decrypt/` | `Keyring::decrypt_indexed`, which re-derives the tenant key (HKDF) on every call | +| `tenant_keyring_decrypt/` | `TenantKeyring::decrypt_indexed`, key derived once at `for_tenant` | +| `hkdf` | `derive_domain_key` (takes no payload, so it has no size) | + +Sizes are 64 B, 1 KiB and 64 KiB. One case at a time: `make perf-ir ARGS="--case store/1024"`. + +**Method** (`benches/perf_ir.rs`): each case runs in its own process under `valgrind --tool=cachegrind --cache-sim=no`, once with 1,000 calls and once with none, after the same setup and one warm-up call. Ir/op is `(Ir[1000] - Ir[0]) / 1000`, so process start, setup and exit cancel. The measured process gets an empty environment, so neither the shell nor a `VALGRIND_OPTS` changes what is counted. + +**Reproducibility:** two runs of the same build agree exactly on every case. Between builds, the heap moves with the size of the binary, and alignment-dependent code in AES-GCM and memcpy follows it: `encrypt/1024` moves by up to 0.19%, every other case by at most 0.02%. The counts also depend on the compiler, the locked dependencies and the CPU features valgrind passes through, so budgets are keyed by architecture and OS and record the rustc and valgrind versions they were taken with; the gate prints a note when yours differ. + +**Budgets** live in `benches/perf_ir_baselines.json`. After a change that makes a path cheaper, `make perf-ir-update` ratchets its budget down. It never raises one: an increase you mean takes `make perf-ir-update ARGS=--allow-increase`. Instruction counts ignore cache misses and branch mispredictions, so a claimed wall-clock win still needs a wall-clock A/B. + --- ## Testing diff --git a/benches/common/mod.rs b/benches/common/mod.rs new file mode 100644 index 0000000..be7f95e --- /dev/null +++ b/benches/common/mod.rs @@ -0,0 +1,60 @@ +//! Payload generators shared by the bench targets. `hot_path` (Criterion wall +//! clock) and `perf_ir` (instruction counts) must measure the same bytes, or +//! their figures cannot be read against each other. + +use serde::Serialize; + +pub fn xorshift(mut state: u64) -> u64 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state +} + +/// A typical cached row: what the SDKs hand ByteStorage after msgpack-encoding +/// a dict/object with string keys. +#[derive(Serialize)] +struct Record { + id: u64, + user: String, + email: String, + score: f64, + active: bool, + tags: Vec<&'static str>, + created_at: String, +} + +/// Realistic payload: a stream of msgpack-named records with deterministic, +/// varied values, truncated to `size`. ByteStorage treats the payload as +/// opaque bytes, so the cut tail does not affect what is measured. +pub fn msgpack_payload(size: usize) -> Vec { + const TAGS: &[&str] = &["free", "pro", "trial", "eu", "us", "beta", "admin"]; + let mut state: u64 = 0x2545f4914f6cdd1d; + let mut out = Vec::with_capacity(size + 256); + let mut id = 0u64; + while out.len() < size { + state = xorshift(state); + id += 1; + let record = Record { + id: 1_000_000 + id, + user: format!("user_{:x}", state % 0xff_ffff), + email: format!("u{}@example{}.com", state % 100_000, state % 7), + score: (state % 10_000) as f64 / 100.0, + active: state % 3 != 0, + tags: (0..(state % 4) as usize) + .map(|i| TAGS[(state as usize >> (8 * i)) % TAGS.len()]) + .collect(), + created_at: format!( + "2026-{:02}-{:02}T{:02}:{:02}:{:02}Z", + 1 + state % 12, + 1 + (state >> 8) % 28, + (state >> 16) % 24, + (state >> 24) % 60, + (state >> 32) % 60 + ), + }; + out.extend_from_slice(&rmp_serde::to_vec_named(&record).unwrap()); + } + out.truncate(size); + out +} diff --git a/benches/hot_path.rs b/benches/hot_path.rs index f15d3cf..0f24fd3 100644 --- a/benches/hot_path.rs +++ b/benches/hot_path.rs @@ -15,9 +15,11 @@ use std::cell::OnceCell; +mod common; + use cachekit_core::{ByteStorage, StorageEnvelope, ZeroKnowledgeEncryptor}; +use common::{msgpack_payload, xorshift}; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use serde::Serialize; const SIZES: &[usize] = &[64, 256, 1024, 4 * 1024, 16 * 1024, 64 * 1024]; @@ -41,61 +43,6 @@ fn incompressible(len: usize) -> Vec { out } -fn xorshift(mut state: u64) -> u64 { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - state -} - -/// A typical cached row: what the SDKs hand ByteStorage after msgpack-encoding -/// a dict/object with string keys. -#[derive(Serialize)] -struct Record { - id: u64, - user: String, - email: String, - score: f64, - active: bool, - tags: Vec<&'static str>, - created_at: String, -} - -/// Realistic payload: a stream of msgpack-named records with deterministic, -/// varied values, truncated to `size`. ByteStorage treats the payload as -/// opaque bytes, so the cut tail does not affect what is measured. -fn msgpack_payload(size: usize) -> Vec { - const TAGS: &[&str] = &["free", "pro", "trial", "eu", "us", "beta", "admin"]; - let mut state: u64 = 0x2545f4914f6cdd1d; - let mut out = Vec::with_capacity(size + 256); - let mut id = 0u64; - while out.len() < size { - state = xorshift(state); - id += 1; - let record = Record { - id: 1_000_000 + id, - user: format!("user_{:x}", state % 0xff_ffff), - email: format!("u{}@example{}.com", state % 100_000, state % 7), - score: (state % 10_000) as f64 / 100.0, - active: state % 3 != 0, - tags: (0..(state % 4) as usize) - .map(|i| TAGS[(state as usize >> (8 * i)) % TAGS.len()]) - .collect(), - created_at: format!( - "2026-{:02}-{:02}T{:02}:{:02}:{:02}Z", - 1 + state % 12, - 1 + (state >> 8) % 28, - (state >> 16) % 24, - (state >> 24) % 60, - (state >> 32) % 60 - ), - }; - out.extend_from_slice(&rmp_serde::to_vec_named(&record).unwrap()); - } - out.truncate(size); - out -} - /// Build a corpus payload and report its LZ4 compressed/raw ratio once, so /// the synthetic and realistic corpora can be compared side by side. fn corpus_fixture(storage: &ByteStorage, id: &str, payload: Vec) -> Vec { diff --git a/benches/perf_ir.rs b/benches/perf_ir.rs new file mode 100644 index 0000000..060eeae --- /dev/null +++ b/benches/perf_ir.rs @@ -0,0 +1,450 @@ +//! Per-op instruction counts (Ir/op) for cachekit-core's hot paths, and the +//! regression gate on them: `make perf-ir`. +//! +//! Wall clock cannot gate a 1% regression on a shared host, and Criterion +//! (`hot_path`) picks its iteration count from wall time, so its total under +//! valgrind cannot be divided by a known count. This target runs every +//! operation a fixed number of times instead: +//! +//! - Each case is its own process, `perf_ir run `. It builds the +//! case's inputs, runs the operation once to warm it (lazy statics, CPU +//! feature detection, the allocator's free lists), then runs it `n` times. +//! - The gate runs each case under `valgrind --tool=cachegrind --cache-sim=no` +//! at `n = N_OPS` and `n = 0`. Ir/op is `(Ir[N_OPS] - Ir[0]) / N_OPS`, so +//! process start, setup, the warm-up and exit cancel. +//! - Budgets are keyed by architecture and OS in `perf_ir_baselines.json`. +//! A case fails at `FAIL_PCT` over its budget and warns from `WARN_PCT`. +//! `--update` ratchets budgets down to the measured figures, never up +//! unless `--allow-increase` says the increase is deliberate. +//! +//! The counts depend on the compiler, the locked dependencies and the CPU +//! features valgrind passes through (ring and glibc pick their AES, GHASH +//! and memcpy code from them), so the gate notes a rustc or valgrind version +//! other than the one the budgets were recorded with. Instruction counts +//! ignore cache misses and branch mispredictions: a claimed wall-clock win +//! still needs a wall-clock A/B. + +mod common; + +use std::collections::BTreeMap; +use std::env; +use std::fs; +use std::hint::black_box; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitCode}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::OnceLock; +use std::thread; + +use cachekit_core::{ + check_msgpack_structure, derive_domain_key, ByteStorage, Keyring, ZeroKnowledgeEncryptor, +}; +use common::msgpack_payload; +use serde::{Deserialize, Serialize}; + +/// Iterations in the measured run. Counts are exact, so this sets run time, +/// not precision. +const N_OPS: u64 = 1000; +const FAIL_PCT: f64 = 1.0; +const WARN_PCT: f64 = 0.2; +const BASELINES: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/benches/perf_ir_baselines.json" +); + +/// A small value, a typical value, a large object (as `hot_path`). +const SIZES: [usize; 3] = [64, 1024, 64 * 1024]; + +/// Operations and whether they take a payload. HKDF derives from the master +/// key and tenant id alone, so it has one case with no size. +const OPS: [(&str, bool); 8] = [ + ("store", true), + ("retrieve", true), + ("prescan", true), + ("encrypt", true), + ("decrypt", true), + ("keyring_decrypt", true), + ("tenant_keyring_decrypt", true), + ("hkdf", false), +]; + +const MASTER_KEY: [u8; 32] = [0x42; 32]; +const TENANT: &str = "tenant-123"; +/// Shaped like an SDK's AAD: version byte, then length-prefixed tenant id and +/// cache key. GHASH cost depends on its length, not its bytes. +const AAD: &[u8] = b"\x03\x00\x00\x00\x0atenant-123\x00\x00\x00\x10ns:bench:func:x"; +/// ByteStorage's envelope nesting bound (`decode_envelope`). +const ENVELOPE_MAX_DEPTH: usize = 100; + +fn case_ids() -> Vec { + OPS.iter() + .flat_map(|&(op, sized)| { + if sized { + SIZES.iter().map(|size| format!("{op}/{size}")).collect() + } else { + vec![op.to_string()] + } + }) + .collect() +} + +// ── measured process ──────────────────────────────────────────────────────── + +/// Build the fixtures and return the operation `case` measures. Setup runs in +/// both the `N_OPS` and the 0 run, so only what the closure does is counted. +fn operation(case: &str) -> Box usize> { + let (op, size) = match case.split_once('/') { + Some((op, size)) => (op, size.parse().expect("case size is a number")), + None => (case, 0), + }; + let payload = msgpack_payload(size); + let storage = ByteStorage::new(None); + let envelope = storage.store(&payload, None).unwrap(); + let encryptor = ZeroKnowledgeEncryptor::new().unwrap(); + let key = derive_domain_key(&MASTER_KEY, "encryption", TENANT.as_bytes()).unwrap(); + let ciphertext = encryptor.encrypt_aes_gcm(&payload, &key, AAD).unwrap(); + match op { + "store" => Box::new(move || storage.store(black_box(&payload), None).unwrap().len()), + "retrieve" => Box::new(move || storage.retrieve(black_box(&envelope)).unwrap().0.len()), + "prescan" => Box::new(move || { + check_msgpack_structure(black_box(&envelope), ENVELOPE_MAX_DEPTH).is_ok() as usize + }), + "encrypt" => Box::new(move || { + encryptor + .encrypt_aes_gcm(black_box(&payload), &key, AAD) + .unwrap() + .len() + }), + "decrypt" => Box::new(move || { + encryptor + .decrypt_aes_gcm(black_box(&ciphertext), &key, AAD) + .unwrap() + .len() + }), + // Re-derives the tenant key (HKDF) on every call. + "keyring_decrypt" => { + let keyring = Keyring::new(&MASTER_KEY, &[]).unwrap(); + Box::new(move || { + keyring + .decrypt_indexed(&encryptor, black_box(&ciphertext), TENANT, AAD) + .unwrap() + .0 + .len() + }) + } + // Derived once at for_tenant; the steady-state decrypt surface. + "tenant_keyring_decrypt" => { + let keyring = Keyring::new(&MASTER_KEY, &[]) + .unwrap() + .for_tenant(TENANT) + .unwrap(); + Box::new(move || { + keyring + .decrypt_indexed(&encryptor, black_box(&ciphertext), AAD) + .unwrap() + .0 + .len() + }) + } + "hkdf" => Box::new(|| { + derive_domain_key(black_box(&MASTER_KEY), "encryption", TENANT.as_bytes()).unwrap()[0] + as usize + }), + other => panic!("unknown case {other}"), + } +} + +fn run(case: &str, n: u64) { + let mut op = operation(case); + black_box(op()); + for _ in 0..n { + black_box(op()); + } +} + +// ── gate ──────────────────────────────────────────────────────────────────── + +#[derive(Serialize, Deserialize, Default)] +struct Baselines { + method: String, + platforms: BTreeMap, +} + +#[derive(Serialize, Deserialize, Default)] +struct Platform { + rustc: String, + valgrind: String, + budgets: BTreeMap, +} + +fn platform_key() -> String { + format!("{}-{}", env::consts::ARCH, env::consts::OS) +} + +/// Word `word` of ` --version`, or "unknown". Only feeds the +/// recorded-with note, so a missing rustc does not stop the gate. +fn version(program: &str, word: usize) -> String { + Command::new(program) + .arg("--version") + .output() + .ok() + .and_then(|out| { + let text = String::from_utf8_lossy(&out.stdout).into_owned(); + text.split_whitespace().nth(word).map(str::to_string) + }) + .unwrap_or_else(|| "unknown".to_string()) +} + +fn which(program: &str) -> Option { + env::split_paths(&env::var_os("PATH")?) + .map(|dir| dir.join(program)) + .find(|path| path.is_file()) +} + +/// Total Ir of one `run` process under cachegrind. +/// +/// The environment is empty, not inherited: its size moves the stack layout, +/// and `VALGRIND_OPTS` or a `~/.valgrindrc` would change what is measured. +fn total_ir(valgrind: &Path, exe: &Path, case: &str, n: u64) -> Result { + let out = Command::new(valgrind) + .args([ + "--tool=cachegrind", + "--cache-sim=no", + "--cachegrind-out-file=/dev/null", + ]) + .arg(exe) + .args(["run", case, &n.to_string()]) + .env_clear() + .output() + .map_err(|e| format!("{case} n={n}: cannot start valgrind: {e}"))?; + let stderr = String::from_utf8_lossy(&out.stderr); + if !out.status.success() { + return Err(format!("{case} n={n}: {}\n{stderr}", out.status)); + } + // "==PID== I refs: 1,234,567" (the column padding varies by version) + stderr + .lines() + .find_map(|line| { + let mut words = line.split_whitespace().skip_while(|w| *w != "I").skip(1); + match (words.next(), words.next()) { + (Some("refs:"), Some(count)) => count.replace(',', "").parse().ok(), + _ => None, + } + }) + .ok_or_else(|| format!("{case} n={n}: no 'I refs' line in valgrind output\n{stderr}")) +} + +/// Ir/op for each case, in case order. Runs are independent processes, so they +/// parallelise. +fn measure(valgrind: &Path, cases: &[String], jobs: usize) -> Result, String> { + let exe = env::current_exe().map_err(|e| format!("cannot locate this binary: {e}"))?; + let runs: Vec<(&str, u64)> = cases + .iter() + .flat_map(|case| [(case.as_str(), N_OPS), (case.as_str(), 0)]) + .collect(); + let next = AtomicUsize::new(0); + let counts: Vec>> = runs.iter().map(|_| OnceLock::new()).collect(); + thread::scope(|scope| { + for _ in 0..jobs.clamp(1, runs.len()) { + scope.spawn(|| loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(&(case, n)) = runs.get(i) else { break }; + let _ = counts[i].set(total_ir(valgrind, &exe, case, n)); + }); + } + }); + let counts = counts + .into_iter() + .map(|count| count.into_inner().expect("every run was measured")) + .collect::, String>>()?; + cases + .iter() + .zip(counts.chunks(2)) + .map(|(case, pair)| { + let (full, setup) = (pair[0], pair[1]); + match full.checked_sub(setup) { + Some(delta) if delta > 0 => Ok((case.clone(), (delta + N_OPS / 2) / N_OPS)), + _ => Err(format!( + "{case}: Ir[{N_OPS}] {full} is not above Ir[0] {setup}" + )), + } + }) + .collect() +} + +fn thousands(n: u64) -> String { + let digits = n.to_string(); + let mut out = String::new(); + for (i, digit) in digits.chars().enumerate() { + if i > 0 && (digits.len() - i) % 3 == 0 { + out.push(','); + } + out.push(digit); + } + out +} + +/// Report lines and whether the gate passes. A case with no budget fails: +/// record it first. +fn compare(budgets: &BTreeMap, measured: &[(String, u64)]) -> (Vec, bool) { + let mut ok = true; + let lines = measured + .iter() + .map(|(case, ir)| { + let ir = *ir; + let Some(&budget) = budgets.get(case) else { + ok = false; + return format!( + "FAIL {case:28} {:>10} Ir/op no budget (run make perf-ir-update)", + thousands(ir) + ); + }; + let pct = (ir as f64 - budget as f64) / budget as f64 * 100.0; + let verdict = if pct >= FAIL_PCT { + ok = false; + "FAIL" + } else if pct >= WARN_PCT { + "WARN" + } else if pct <= -FAIL_PCT { + "LOWER" // cheaper: ratchet it down with --update if the change touched this path + } else { + "ok" + }; + format!( + "{verdict:5} {case:28} {:>10} Ir/op budget {:>10} {pct:+.2}%", + thousands(ir), + thousands(budget) + ) + }) + .collect(); + (lines, ok) +} + +/// New budgets: only ever lower, unless the increase is deliberate. +fn ratchet(budgets: &mut BTreeMap, measured: &[(String, u64)], allow_increase: bool) { + for (case, ir) in measured { + let budget = budgets.entry(case.clone()).or_insert(*ir); + if allow_increase || *ir < *budget { + *budget = *ir; + } + } +} + +const USAGE: &str = "\ +usage: cargo bench --features encryption --bench perf_ir -- [options] + (or make perf-ir / make perf-ir-update) + +Measures Ir/op for each case under cachegrind and compares it with the budget +in benches/perf_ir_baselines.json. Exit 0 pass, 1 regression, 2 error. + +options: + --case measure only this case (repeatable) + --update write lower measured figures back as budgets + --allow-increase with --update, also raise budgets + --jobs cachegrind runs at a time (default: min(8, cores)) + +perf_ir run the measured process: setup, one warm-up op, n ops"; + +fn gate(args: &[String]) -> Result { + let (mut update, mut allow_increase, mut cases) = (false, false, Vec::new()); + let mut jobs = thread::available_parallelism() + .map_or(1, |n| n.get()) + .min(8); + let mut args = args.iter(); + while let Some(arg) = args.next() { + match arg.as_str() { + "--update" => update = true, + "--allow-increase" => allow_increase = true, + "--case" => cases.push(args.next().ok_or("--case needs a case id")?.clone()), + "--jobs" => { + jobs = args + .next() + .and_then(|n| n.parse().ok()) + .ok_or("--jobs needs a number")? + } + "-h" | "--help" => { + println!("{USAGE}\n\ncases: {}", case_ids().join(" ")); + return Ok(true); + } + other => return Err(format!("unknown argument {other}\n\n{USAGE}")), + } + } + if allow_increase && !update { + return Err("--allow-increase only applies with --update".into()); + } + let all = case_ids(); + if let Some(unknown) = cases.iter().find(|case| !all.contains(case)) { + return Err(format!("unknown case {unknown}; cases: {}", all.join(" "))); + } + if cases.is_empty() { + cases = all; + } + + let valgrind_bin = + which("valgrind").ok_or("valgrind not found on PATH: install it (apt install valgrind)")?; + let text = fs::read_to_string(BASELINES).map_err(|e| format!("{BASELINES}: {e}"))?; + let mut baselines: Baselines = + serde_json::from_str(&text).map_err(|e| format!("{BASELINES}: {e}"))?; + let key = platform_key(); + let (rustc, valgrind) = (version("rustc", 1), version("valgrind", 0)); + let platform = baselines.platforms.entry(key.clone()).or_default(); + if !platform.budgets.is_empty() && (platform.rustc != rustc || platform.valgrind != valgrind) { + println!( + "note: budgets for {key} were recorded with rustc {} and {}; this is rustc {rustc} and {valgrind}. \ + A toolchain change moves the counts.", + platform.rustc, platform.valgrind + ); + } + + let measured = measure(&valgrind_bin, &cases, jobs)?; + println!( + "cachekit-core Ir/op on {key} (rustc {rustc}, {valgrind}): (Ir[{N_OPS}] - Ir[0]) / {N_OPS}" + ); + // With --update, report against the budgets as written, so the only FAIL + // left is a regression that --allow-increase did not accept. + if update { + ratchet(&mut platform.budgets, &measured, allow_increase); + platform.rustc = rustc; + platform.valgrind = valgrind; + } + let (lines, ok) = compare(&platform.budgets, &measured); + println!("{}", lines.join("\n")); + if update { + baselines.method = format!( + "cachegrind Ir per op, (Ir[{N_OPS}] - Ir[0]) / {N_OPS}, one process per case, \ + empty environment; written by benches/perf_ir.rs --update" + ); + let json = serde_json::to_string_pretty(&baselines).map_err(|e| e.to_string())?; + fs::write(BASELINES, json + "\n").map_err(|e| format!("{BASELINES}: {e}"))?; + println!("budgets written to benches/perf_ir_baselines.json"); + } + println!( + "{}", + match (ok, update) { + (true, _) => "PASS", + (false, false) => "FAIL: a case regressed by 1% or more, or has no budget", + (false, true) => + "FAIL: a regression stays over budget; raise it only with --allow-increase", + } + ); + Ok(ok) +} + +fn main() -> ExitCode { + // `cargo bench` appends --bench to the arguments it passes through. + let args: Vec = env::args().skip(1).filter(|arg| arg != "--bench").collect(); + if let [mode, case, n] = args.as_slice() { + if mode == "run" { + run(case, n.parse().expect("n is a number")); + return ExitCode::SUCCESS; + } + } + match gate(&args) { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::from(1), + Err(err) => { + eprintln!("perf_ir: {err}"); + ExitCode::from(2) + } + } +} diff --git a/benches/perf_ir_baselines.json b/benches/perf_ir_baselines.json new file mode 100644 index 0000000..5f19736 --- /dev/null +++ b/benches/perf_ir_baselines.json @@ -0,0 +1,33 @@ +{ + "method": "cachegrind Ir per op, (Ir[1000] - Ir[0]) / 1000, one process per case, empty environment; written by benches/perf_ir.rs --update", + "platforms": { + "x86_64-linux": { + "rustc": "1.98.1", + "valgrind": "valgrind-3.22.0", + "budgets": { + "decrypt/1024": 5575, + "decrypt/64": 2600, + "decrypt/65536": 235243, + "encrypt/1024": 6432, + "encrypt/64": 3253, + "encrypt/65536": 301457, + "hkdf": 29047, + "keyring_decrypt/1024": 34715, + "keyring_decrypt/64": 31730, + "keyring_decrypt/65536": 264373, + "prescan/1024": 982, + "prescan/64": 880, + "prescan/65536": 932, + "retrieve/1024": 7558, + "retrieve/64": 2722, + "retrieve/65536": 416730, + "store/1024": 29644, + "store/64": 12277, + "store/65536": 1080278, + "tenant_keyring_decrypt/1024": 5598, + "tenant_keyring_decrypt/64": 2623, + "tenant_keyring_decrypt/65536": 235234 + } + } + } +} From 0bad521ab488a90111cd4e5453674a99b0f7b917 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Sun, 4 Oct 2026 00:59:58 +1000 Subject: [PATCH 2/2] fix(bench): keep each Ir budget set to one build and toolchain (LAB-7802) A budget set gates only the instrument that recorded it. Two ways let `--update` mix instruments: - Budgets were keyed by architecture and OS only. A build with `-C target-cpu=x86-64-v3` measures hkdf at 23,913 Ir/op against the default build's 29,047, so ratcheting from it lowered the shared set and then failed every default build of unchanged code. The key now appends any compiled-in CPU features, so such a build records its own set. - `--update` stamped the current rustc and valgrind on every run, including a `--case` subset and a run whose regression stayed over budget, so budgets it never measured carried the new versions and the mismatch note went silent. On a toolchain other than the recorded one, `--update` now refuses unless it re-records every case with `--allow-increase`. --- README.md | 4 +-- benches/perf_ir.rs | 73 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index d81a0dd..4bab124 100644 --- a/README.md +++ b/README.md @@ -350,9 +350,9 @@ Sizes are 64 B, 1 KiB and 64 KiB. One case at a time: `make perf-ir ARGS="--case **Method** (`benches/perf_ir.rs`): each case runs in its own process under `valgrind --tool=cachegrind --cache-sim=no`, once with 1,000 calls and once with none, after the same setup and one warm-up call. Ir/op is `(Ir[1000] - Ir[0]) / 1000`, so process start, setup and exit cancel. The measured process gets an empty environment, so neither the shell nor a `VALGRIND_OPTS` changes what is counted. -**Reproducibility:** two runs of the same build agree exactly on every case. Between builds, the heap moves with the size of the binary, and alignment-dependent code in AES-GCM and memcpy follows it: `encrypt/1024` moves by up to 0.19%, every other case by at most 0.02%. The counts also depend on the compiler, the locked dependencies and the CPU features valgrind passes through, so budgets are keyed by architecture and OS and record the rustc and valgrind versions they were taken with; the gate prints a note when yours differ. +**Reproducibility:** two runs of the same build agree exactly on every case. Between builds, the heap moves with the size of the binary, and alignment-dependent code in AES-GCM and memcpy follows it: `encrypt/1024` moves by up to 0.19%, every other case by at most 0.02%. The counts also depend on the compiler, the locked dependencies, the CPU features the binary is compiled for and the ones valgrind passes through. Budgets are keyed by architecture, OS and any compiled-in CPU features (a `-C target-cpu=x86-64-v3` build gets its own set), and each set records the rustc and valgrind versions it was taken with; the gate prints a note when yours differ. Other `RUSTFLAGS` and profile overrides are not detected, so measure the default build. -**Budgets** live in `benches/perf_ir_baselines.json`. After a change that makes a path cheaper, `make perf-ir-update` ratchets its budget down. It never raises one: an increase you mean takes `make perf-ir-update ARGS=--allow-increase`. Instruction counts ignore cache misses and branch mispredictions, so a claimed wall-clock win still needs a wall-clock A/B. +**Budgets** live in `benches/perf_ir_baselines.json`. After a change that makes a path cheaper, `make perf-ir-update` ratchets its budget down. It never raises one: an increase you mean takes `make perf-ir-update ARGS=--allow-increase`. Ratcheting works only on the rustc and valgrind the budgets were recorded with; on a new toolchain, re-record every case with `make perf-ir-update ARGS=--allow-increase` (no `--case`). Instruction counts ignore cache misses and branch mispredictions, so a claimed wall-clock win still needs a wall-clock A/B. --- diff --git a/benches/perf_ir.rs b/benches/perf_ir.rs index 060eeae..f4cfe47 100644 --- a/benches/perf_ir.rs +++ b/benches/perf_ir.rs @@ -12,15 +12,18 @@ //! - The gate runs each case under `valgrind --tool=cachegrind --cache-sim=no` //! at `n = N_OPS` and `n = 0`. Ir/op is `(Ir[N_OPS] - Ir[0]) / N_OPS`, so //! process start, setup, the warm-up and exit cancel. -//! - Budgets are keyed by architecture and OS in `perf_ir_baselines.json`. -//! A case fails at `FAIL_PCT` over its budget and warns from `WARN_PCT`. -//! `--update` ratchets budgets down to the measured figures, never up -//! unless `--allow-increase` says the increase is deliberate. +//! - Budgets are keyed by architecture, OS and compiled-in CPU features in +//! `perf_ir_baselines.json`. A case fails at `FAIL_PCT` over its budget and +//! warns from `WARN_PCT`. `--update` ratchets budgets down to the measured +//! figures, never up unless `--allow-increase` says the increase is +//! deliberate. //! -//! The counts depend on the compiler, the locked dependencies and the CPU -//! features valgrind passes through (ring and glibc pick their AES, GHASH -//! and memcpy code from them), so the gate notes a rustc or valgrind version -//! other than the one the budgets were recorded with. Instruction counts +//! The counts depend on the compiler, the locked dependencies, the CPU +//! features the binary was compiled for, and the ones valgrind passes through +//! (ring and glibc pick their AES, GHASH and memcpy code from them). Each +//! budget set records the rustc and valgrind it was measured with: the gate +//! notes a different one, and `--update` refuses to ratchet across them, +//! because a set that mixes two toolchains gates neither. Instruction counts //! ignore cache misses and branch mispredictions: a claimed wall-clock win //! still needs a wall-clock A/B. @@ -177,8 +180,43 @@ struct Platform { budgets: BTreeMap, } +/// CPU features this binary was compiled for, beyond the target's baseline. +/// `-C target-cpu` or `-C target-feature` changes the code the compiler and +/// RustCrypto emit (x86-64-v3 takes 18% off `hkdf`), so a build with any of +/// these on is a different instrument with its own budgets. Other RUSTFLAGS +/// and profile overrides are not detected: measure the default build. +const COMPILED_FEATURES: [(&str, bool); 21] = [ + ("sse3", cfg!(target_feature = "sse3")), + ("ssse3", cfg!(target_feature = "ssse3")), + ("sse4.1", cfg!(target_feature = "sse4.1")), + ("sse4.2", cfg!(target_feature = "sse4.2")), + ("popcnt", cfg!(target_feature = "popcnt")), + ("avx", cfg!(target_feature = "avx")), + ("avx2", cfg!(target_feature = "avx2")), + ("bmi1", cfg!(target_feature = "bmi1")), + ("bmi2", cfg!(target_feature = "bmi2")), + ("fma", cfg!(target_feature = "fma")), + ("lzcnt", cfg!(target_feature = "lzcnt")), + ("movbe", cfg!(target_feature = "movbe")), + ("adx", cfg!(target_feature = "adx")), + ("avx512f", cfg!(target_feature = "avx512f")), + ("aes", cfg!(target_feature = "aes")), + ("pclmulqdq", cfg!(target_feature = "pclmulqdq")), + ("sha", cfg!(target_feature = "sha")), + ("sha2", cfg!(target_feature = "sha2")), + ("sha3", cfg!(target_feature = "sha3")), + ("crc", cfg!(target_feature = "crc")), + ("lse", cfg!(target_feature = "lse")), +]; + +/// Budget key: architecture, OS, then any compiled-in CPU features. fn platform_key() -> String { - format!("{}-{}", env::consts::ARCH, env::consts::OS) + let mut key = format!("{}-{}", env::consts::ARCH, env::consts::OS); + for (feature, _) in COMPILED_FEATURES.iter().filter(|(_, on)| *on) { + key.push('+'); + key.push_str(feature); + } + key } /// Word `word` of ` --version`, or "unknown". Only feeds the @@ -376,7 +414,8 @@ fn gate(args: &[String]) -> Result { if let Some(unknown) = cases.iter().find(|case| !all.contains(case)) { return Err(format!("unknown case {unknown}; cases: {}", all.join(" "))); } - if cases.is_empty() { + let every_case = cases.is_empty(); + if every_case { cases = all; } @@ -389,11 +428,19 @@ fn gate(args: &[String]) -> Result { let (rustc, valgrind) = (version("rustc", 1), version("valgrind", 0)); let platform = baselines.platforms.entry(key.clone()).or_default(); if !platform.budgets.is_empty() && (platform.rustc != rustc || platform.valgrind != valgrind) { - println!( - "note: budgets for {key} were recorded with rustc {} and {}; this is rustc {rustc} and {valgrind}. \ - A toolchain change moves the counts.", + let recorded = format!( + "budgets for {key} were recorded with rustc {} and {}; this is rustc {rustc} and {valgrind}", platform.rustc, platform.valgrind ); + // One budget set, one toolchain: ratcheting down across toolchains + // would mix the two, and a partial re-record would label budgets it + // never measured with the new versions. + if update && !(allow_increase && every_case) { + return Err(format!( + "{recorded}. Re-record every case on one toolchain: --update --allow-increase, no --case." + )); + } + println!("note: {recorded}. A toolchain change moves the counts."); } let measured = measure(&valgrind_bin, &cases, jobs)?;