diff --git a/Cargo.toml b/Cargo.toml index 770c5de..458662f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,3 +170,12 @@ debug = "line-tables-only" name = "hot_path" harness = false required-features = ["encryption"] + +# Instruction-count gate (`make perf-ir`). It needs valgrind, so `bench = false` +# keeps it out of a plain `cargo bench`; `--bench perf_ir` still selects it, and +# `clippy --all-targets` still compiles it. +[[bench]] +name = "perf_ir" +harness = false +bench = false +required-features = ["encryption"] diff --git a/Makefile b/Makefile index 752080d..09b3e30 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # cachekit-core - Development Makefile -.PHONY: help check test lint clippy audit deny vet fmt fmt-check bench bench-quick fuzz-quick fuzz-deep sbom clean +.PHONY: help check test lint clippy audit deny vet fmt fmt-check bench bench-quick perf-ir perf-ir-update fuzz-quick fuzz-deep sbom clean .DEFAULT_GOAL := help # Colors for output @@ -84,6 +84,17 @@ bench-quick: ## Quick bench run for CI (1s warmup, 2s measurement, 10 samples) $(call require_binary,cargo,Install Rust: https://rustup.rs) @cargo bench -p cachekit-core --features encryption --bench hot_path -- --warm-up-time 1 --measurement-time 2 --sample-size 10 +# Per-op instruction counts (benches/perf_ir.rs) against the budgets in +# benches/perf_ir_baselines.json. Needs valgrind. Pass gate options through +# ARGS, e.g. make perf-ir ARGS="--case store/1024". See README, "Instruction counts". +perf-ir: ## Instruction-count gate: fail on >=1% Ir/op regression (needs valgrind) + $(call require_binary,cargo,Install Rust: https://rustup.rs) + @cargo bench -p cachekit-core --features encryption --bench perf_ir -- $(ARGS) + +perf-ir-update: ## Ratchet Ir/op budgets down to the measured figures (never up) + $(call require_binary,cargo,Install Rust: https://rustup.rs) + @cargo bench -p cachekit-core --features encryption --bench perf_ir -- --update $(ARGS) + fuzz-quick: ## Quick corpus-only fuzz run (2 min per target) @echo "$(BLUE)Running quick fuzzing (2 min per target)...$(RESET)" $(call require_binary,cargo-fuzz,Install: cargo install cargo-fuzz) diff --git a/README.md b/README.md index c1f61ad..4bab124 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,10 @@ cachekit-core/ ├── include/ │ └── cachekit.h # Generated C header │ +├── benches/ +│ ├── hot_path.rs # Criterion wall-clock suite (make bench) +│ └── perf_ir.rs # Per-op instruction counts and their gate (make perf-ir) +│ ├── fuzz/ # Fuzzing targets (16 targets) │ └── fuzz_targets/ │ @@ -329,6 +333,27 @@ Benchmarks on Apple M2 Max (64KB payload, compressible data): The figures above come from `examples/bench_throughput.rs` on highly compressible data. The Criterion suite in `benches/hot_path.rs` needs the `encryption` feature (`make bench`, or `cargo bench --features encryption`; a plain `cargo bench` skips it). It runs the ByteStorage roundtrip on three corpora side by side: `byte_storage/roundtrip` (synthetic ramp, kept for history), `byte_storage/roundtrip_msgpack` (realistic msgpack records, about 0.38 LZ4 ratio at 64 KB) and `byte_storage/roundtrip_incompressible`. The bench profile keeps symbols, so callgrind and perf attribute cost to functions. +### Instruction counts + +`make perf-ir` counts the instructions each hot path executes per call and fails when a case costs **1% or more** above its committed budget. It warns from 0.2%. Wall clock on a shared machine moves by far more than 1% between identical runs, and Criterion picks its iteration count from wall time; instruction counts at a fixed iteration count are exact. It needs valgrind (`apt install valgrind`). + +| Case | What one call does | +|:-----|:-------------------| +| `store/`, `retrieve/` | `ByteStorage::store` / `retrieve` on the realistic msgpack payload `hot_path` uses | +| `prescan/` | `check_msgpack_structure` on the envelope, the pre-scan `retrieve` runs first | +| `encrypt/`, `decrypt/` | `ZeroKnowledgeEncryptor` AES-256-GCM | +| `keyring_decrypt/` | `Keyring::decrypt_indexed`, which re-derives the tenant key (HKDF) on every call | +| `tenant_keyring_decrypt/` | `TenantKeyring::decrypt_indexed`, key derived once at `for_tenant` | +| `hkdf` | `derive_domain_key` (takes no payload, so it has no size) | + +Sizes are 64 B, 1 KiB and 64 KiB. One case at a time: `make perf-ir ARGS="--case store/1024"`. + +**Method** (`benches/perf_ir.rs`): each case runs in its own process under `valgrind --tool=cachegrind --cache-sim=no`, once with 1,000 calls and once with none, after the same setup and one warm-up call. Ir/op is `(Ir[1000] - Ir[0]) / 1000`, so process start, setup and exit cancel. The measured process gets an empty environment, so neither the shell nor a `VALGRIND_OPTS` changes what is counted. + +**Reproducibility:** two runs of the same build agree exactly on every case. Between builds, the heap moves with the size of the binary, and alignment-dependent code in AES-GCM and memcpy follows it: `encrypt/1024` moves by up to 0.19%, every other case by at most 0.02%. The counts also depend on the compiler, the locked dependencies, the CPU features the binary is compiled for and the ones valgrind passes through. Budgets are keyed by architecture, OS and any compiled-in CPU features (a `-C target-cpu=x86-64-v3` build gets its own set), and each set records the rustc and valgrind versions it was taken with; the gate prints a note when yours differ. Other `RUSTFLAGS` and profile overrides are not detected, so measure the default build. + +**Budgets** live in `benches/perf_ir_baselines.json`. After a change that makes a path cheaper, `make perf-ir-update` ratchets its budget down. It never raises one: an increase you mean takes `make perf-ir-update ARGS=--allow-increase`. Ratcheting works only on the rustc and valgrind the budgets were recorded with; on a new toolchain, re-record every case with `make perf-ir-update ARGS=--allow-increase` (no `--case`). Instruction counts ignore cache misses and branch mispredictions, so a claimed wall-clock win still needs a wall-clock A/B. + --- ## Testing diff --git a/benches/common/mod.rs b/benches/common/mod.rs new file mode 100644 index 0000000..be7f95e --- /dev/null +++ b/benches/common/mod.rs @@ -0,0 +1,60 @@ +//! Payload generators shared by the bench targets. `hot_path` (Criterion wall +//! clock) and `perf_ir` (instruction counts) must measure the same bytes, or +//! their figures cannot be read against each other. + +use serde::Serialize; + +pub fn xorshift(mut state: u64) -> u64 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state +} + +/// A typical cached row: what the SDKs hand ByteStorage after msgpack-encoding +/// a dict/object with string keys. +#[derive(Serialize)] +struct Record { + id: u64, + user: String, + email: String, + score: f64, + active: bool, + tags: Vec<&'static str>, + created_at: String, +} + +/// Realistic payload: a stream of msgpack-named records with deterministic, +/// varied values, truncated to `size`. ByteStorage treats the payload as +/// opaque bytes, so the cut tail does not affect what is measured. +pub fn msgpack_payload(size: usize) -> Vec { + const TAGS: &[&str] = &["free", "pro", "trial", "eu", "us", "beta", "admin"]; + let mut state: u64 = 0x2545f4914f6cdd1d; + let mut out = Vec::with_capacity(size + 256); + let mut id = 0u64; + while out.len() < size { + state = xorshift(state); + id += 1; + let record = Record { + id: 1_000_000 + id, + user: format!("user_{:x}", state % 0xff_ffff), + email: format!("u{}@example{}.com", state % 100_000, state % 7), + score: (state % 10_000) as f64 / 100.0, + active: state % 3 != 0, + tags: (0..(state % 4) as usize) + .map(|i| TAGS[(state as usize >> (8 * i)) % TAGS.len()]) + .collect(), + created_at: format!( + "2026-{:02}-{:02}T{:02}:{:02}:{:02}Z", + 1 + state % 12, + 1 + (state >> 8) % 28, + (state >> 16) % 24, + (state >> 24) % 60, + (state >> 32) % 60 + ), + }; + out.extend_from_slice(&rmp_serde::to_vec_named(&record).unwrap()); + } + out.truncate(size); + out +} diff --git a/benches/hot_path.rs b/benches/hot_path.rs index f15d3cf..0f24fd3 100644 --- a/benches/hot_path.rs +++ b/benches/hot_path.rs @@ -15,9 +15,11 @@ use std::cell::OnceCell; +mod common; + use cachekit_core::{ByteStorage, StorageEnvelope, ZeroKnowledgeEncryptor}; +use common::{msgpack_payload, xorshift}; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use serde::Serialize; const SIZES: &[usize] = &[64, 256, 1024, 4 * 1024, 16 * 1024, 64 * 1024]; @@ -41,61 +43,6 @@ fn incompressible(len: usize) -> Vec { out } -fn xorshift(mut state: u64) -> u64 { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - state -} - -/// A typical cached row: what the SDKs hand ByteStorage after msgpack-encoding -/// a dict/object with string keys. -#[derive(Serialize)] -struct Record { - id: u64, - user: String, - email: String, - score: f64, - active: bool, - tags: Vec<&'static str>, - created_at: String, -} - -/// Realistic payload: a stream of msgpack-named records with deterministic, -/// varied values, truncated to `size`. ByteStorage treats the payload as -/// opaque bytes, so the cut tail does not affect what is measured. -fn msgpack_payload(size: usize) -> Vec { - const TAGS: &[&str] = &["free", "pro", "trial", "eu", "us", "beta", "admin"]; - let mut state: u64 = 0x2545f4914f6cdd1d; - let mut out = Vec::with_capacity(size + 256); - let mut id = 0u64; - while out.len() < size { - state = xorshift(state); - id += 1; - let record = Record { - id: 1_000_000 + id, - user: format!("user_{:x}", state % 0xff_ffff), - email: format!("u{}@example{}.com", state % 100_000, state % 7), - score: (state % 10_000) as f64 / 100.0, - active: state % 3 != 0, - tags: (0..(state % 4) as usize) - .map(|i| TAGS[(state as usize >> (8 * i)) % TAGS.len()]) - .collect(), - created_at: format!( - "2026-{:02}-{:02}T{:02}:{:02}:{:02}Z", - 1 + state % 12, - 1 + (state >> 8) % 28, - (state >> 16) % 24, - (state >> 24) % 60, - (state >> 32) % 60 - ), - }; - out.extend_from_slice(&rmp_serde::to_vec_named(&record).unwrap()); - } - out.truncate(size); - out -} - /// Build a corpus payload and report its LZ4 compressed/raw ratio once, so /// the synthetic and realistic corpora can be compared side by side. fn corpus_fixture(storage: &ByteStorage, id: &str, payload: Vec) -> Vec { diff --git a/benches/perf_ir.rs b/benches/perf_ir.rs new file mode 100644 index 0000000..f4cfe47 --- /dev/null +++ b/benches/perf_ir.rs @@ -0,0 +1,497 @@ +//! Per-op instruction counts (Ir/op) for cachekit-core's hot paths, and the +//! regression gate on them: `make perf-ir`. +//! +//! Wall clock cannot gate a 1% regression on a shared host, and Criterion +//! (`hot_path`) picks its iteration count from wall time, so its total under +//! valgrind cannot be divided by a known count. This target runs every +//! operation a fixed number of times instead: +//! +//! - Each case is its own process, `perf_ir run `. It builds the +//! case's inputs, runs the operation once to warm it (lazy statics, CPU +//! feature detection, the allocator's free lists), then runs it `n` times. +//! - The gate runs each case under `valgrind --tool=cachegrind --cache-sim=no` +//! at `n = N_OPS` and `n = 0`. Ir/op is `(Ir[N_OPS] - Ir[0]) / N_OPS`, so +//! process start, setup, the warm-up and exit cancel. +//! - Budgets are keyed by architecture, OS and compiled-in CPU features in +//! `perf_ir_baselines.json`. A case fails at `FAIL_PCT` over its budget and +//! warns from `WARN_PCT`. `--update` ratchets budgets down to the measured +//! figures, never up unless `--allow-increase` says the increase is +//! deliberate. +//! +//! The counts depend on the compiler, the locked dependencies, the CPU +//! features the binary was compiled for, and the ones valgrind passes through +//! (ring and glibc pick their AES, GHASH and memcpy code from them). Each +//! budget set records the rustc and valgrind it was measured with: the gate +//! notes a different one, and `--update` refuses to ratchet across them, +//! because a set that mixes two toolchains gates neither. Instruction counts +//! ignore cache misses and branch mispredictions: a claimed wall-clock win +//! still needs a wall-clock A/B. + +mod common; + +use std::collections::BTreeMap; +use std::env; +use std::fs; +use std::hint::black_box; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitCode}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::OnceLock; +use std::thread; + +use cachekit_core::{ + check_msgpack_structure, derive_domain_key, ByteStorage, Keyring, ZeroKnowledgeEncryptor, +}; +use common::msgpack_payload; +use serde::{Deserialize, Serialize}; + +/// Iterations in the measured run. Counts are exact, so this sets run time, +/// not precision. +const N_OPS: u64 = 1000; +const FAIL_PCT: f64 = 1.0; +const WARN_PCT: f64 = 0.2; +const BASELINES: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/benches/perf_ir_baselines.json" +); + +/// A small value, a typical value, a large object (as `hot_path`). +const SIZES: [usize; 3] = [64, 1024, 64 * 1024]; + +/// Operations and whether they take a payload. HKDF derives from the master +/// key and tenant id alone, so it has one case with no size. +const OPS: [(&str, bool); 8] = [ + ("store", true), + ("retrieve", true), + ("prescan", true), + ("encrypt", true), + ("decrypt", true), + ("keyring_decrypt", true), + ("tenant_keyring_decrypt", true), + ("hkdf", false), +]; + +const MASTER_KEY: [u8; 32] = [0x42; 32]; +const TENANT: &str = "tenant-123"; +/// Shaped like an SDK's AAD: version byte, then length-prefixed tenant id and +/// cache key. GHASH cost depends on its length, not its bytes. +const AAD: &[u8] = b"\x03\x00\x00\x00\x0atenant-123\x00\x00\x00\x10ns:bench:func:x"; +/// ByteStorage's envelope nesting bound (`decode_envelope`). +const ENVELOPE_MAX_DEPTH: usize = 100; + +fn case_ids() -> Vec { + OPS.iter() + .flat_map(|&(op, sized)| { + if sized { + SIZES.iter().map(|size| format!("{op}/{size}")).collect() + } else { + vec![op.to_string()] + } + }) + .collect() +} + +// ── measured process ──────────────────────────────────────────────────────── + +/// Build the fixtures and return the operation `case` measures. Setup runs in +/// both the `N_OPS` and the 0 run, so only what the closure does is counted. +fn operation(case: &str) -> Box usize> { + let (op, size) = match case.split_once('/') { + Some((op, size)) => (op, size.parse().expect("case size is a number")), + None => (case, 0), + }; + let payload = msgpack_payload(size); + let storage = ByteStorage::new(None); + let envelope = storage.store(&payload, None).unwrap(); + let encryptor = ZeroKnowledgeEncryptor::new().unwrap(); + let key = derive_domain_key(&MASTER_KEY, "encryption", TENANT.as_bytes()).unwrap(); + let ciphertext = encryptor.encrypt_aes_gcm(&payload, &key, AAD).unwrap(); + match op { + "store" => Box::new(move || storage.store(black_box(&payload), None).unwrap().len()), + "retrieve" => Box::new(move || storage.retrieve(black_box(&envelope)).unwrap().0.len()), + "prescan" => Box::new(move || { + check_msgpack_structure(black_box(&envelope), ENVELOPE_MAX_DEPTH).is_ok() as usize + }), + "encrypt" => Box::new(move || { + encryptor + .encrypt_aes_gcm(black_box(&payload), &key, AAD) + .unwrap() + .len() + }), + "decrypt" => Box::new(move || { + encryptor + .decrypt_aes_gcm(black_box(&ciphertext), &key, AAD) + .unwrap() + .len() + }), + // Re-derives the tenant key (HKDF) on every call. + "keyring_decrypt" => { + let keyring = Keyring::new(&MASTER_KEY, &[]).unwrap(); + Box::new(move || { + keyring + .decrypt_indexed(&encryptor, black_box(&ciphertext), TENANT, AAD) + .unwrap() + .0 + .len() + }) + } + // Derived once at for_tenant; the steady-state decrypt surface. + "tenant_keyring_decrypt" => { + let keyring = Keyring::new(&MASTER_KEY, &[]) + .unwrap() + .for_tenant(TENANT) + .unwrap(); + Box::new(move || { + keyring + .decrypt_indexed(&encryptor, black_box(&ciphertext), AAD) + .unwrap() + .0 + .len() + }) + } + "hkdf" => Box::new(|| { + derive_domain_key(black_box(&MASTER_KEY), "encryption", TENANT.as_bytes()).unwrap()[0] + as usize + }), + other => panic!("unknown case {other}"), + } +} + +fn run(case: &str, n: u64) { + let mut op = operation(case); + black_box(op()); + for _ in 0..n { + black_box(op()); + } +} + +// ── gate ──────────────────────────────────────────────────────────────────── + +#[derive(Serialize, Deserialize, Default)] +struct Baselines { + method: String, + platforms: BTreeMap, +} + +#[derive(Serialize, Deserialize, Default)] +struct Platform { + rustc: String, + valgrind: String, + budgets: BTreeMap, +} + +/// CPU features this binary was compiled for, beyond the target's baseline. +/// `-C target-cpu` or `-C target-feature` changes the code the compiler and +/// RustCrypto emit (x86-64-v3 takes 18% off `hkdf`), so a build with any of +/// these on is a different instrument with its own budgets. Other RUSTFLAGS +/// and profile overrides are not detected: measure the default build. +const COMPILED_FEATURES: [(&str, bool); 21] = [ + ("sse3", cfg!(target_feature = "sse3")), + ("ssse3", cfg!(target_feature = "ssse3")), + ("sse4.1", cfg!(target_feature = "sse4.1")), + ("sse4.2", cfg!(target_feature = "sse4.2")), + ("popcnt", cfg!(target_feature = "popcnt")), + ("avx", cfg!(target_feature = "avx")), + ("avx2", cfg!(target_feature = "avx2")), + ("bmi1", cfg!(target_feature = "bmi1")), + ("bmi2", cfg!(target_feature = "bmi2")), + ("fma", cfg!(target_feature = "fma")), + ("lzcnt", cfg!(target_feature = "lzcnt")), + ("movbe", cfg!(target_feature = "movbe")), + ("adx", cfg!(target_feature = "adx")), + ("avx512f", cfg!(target_feature = "avx512f")), + ("aes", cfg!(target_feature = "aes")), + ("pclmulqdq", cfg!(target_feature = "pclmulqdq")), + ("sha", cfg!(target_feature = "sha")), + ("sha2", cfg!(target_feature = "sha2")), + ("sha3", cfg!(target_feature = "sha3")), + ("crc", cfg!(target_feature = "crc")), + ("lse", cfg!(target_feature = "lse")), +]; + +/// Budget key: architecture, OS, then any compiled-in CPU features. +fn platform_key() -> String { + let mut key = format!("{}-{}", env::consts::ARCH, env::consts::OS); + for (feature, _) in COMPILED_FEATURES.iter().filter(|(_, on)| *on) { + key.push('+'); + key.push_str(feature); + } + key +} + +/// Word `word` of ` --version`, or "unknown". Only feeds the +/// recorded-with note, so a missing rustc does not stop the gate. +fn version(program: &str, word: usize) -> String { + Command::new(program) + .arg("--version") + .output() + .ok() + .and_then(|out| { + let text = String::from_utf8_lossy(&out.stdout).into_owned(); + text.split_whitespace().nth(word).map(str::to_string) + }) + .unwrap_or_else(|| "unknown".to_string()) +} + +fn which(program: &str) -> Option { + env::split_paths(&env::var_os("PATH")?) + .map(|dir| dir.join(program)) + .find(|path| path.is_file()) +} + +/// Total Ir of one `run` process under cachegrind. +/// +/// The environment is empty, not inherited: its size moves the stack layout, +/// and `VALGRIND_OPTS` or a `~/.valgrindrc` would change what is measured. +fn total_ir(valgrind: &Path, exe: &Path, case: &str, n: u64) -> Result { + let out = Command::new(valgrind) + .args([ + "--tool=cachegrind", + "--cache-sim=no", + "--cachegrind-out-file=/dev/null", + ]) + .arg(exe) + .args(["run", case, &n.to_string()]) + .env_clear() + .output() + .map_err(|e| format!("{case} n={n}: cannot start valgrind: {e}"))?; + let stderr = String::from_utf8_lossy(&out.stderr); + if !out.status.success() { + return Err(format!("{case} n={n}: {}\n{stderr}", out.status)); + } + // "==PID== I refs: 1,234,567" (the column padding varies by version) + stderr + .lines() + .find_map(|line| { + let mut words = line.split_whitespace().skip_while(|w| *w != "I").skip(1); + match (words.next(), words.next()) { + (Some("refs:"), Some(count)) => count.replace(',', "").parse().ok(), + _ => None, + } + }) + .ok_or_else(|| format!("{case} n={n}: no 'I refs' line in valgrind output\n{stderr}")) +} + +/// Ir/op for each case, in case order. Runs are independent processes, so they +/// parallelise. +fn measure(valgrind: &Path, cases: &[String], jobs: usize) -> Result, String> { + let exe = env::current_exe().map_err(|e| format!("cannot locate this binary: {e}"))?; + let runs: Vec<(&str, u64)> = cases + .iter() + .flat_map(|case| [(case.as_str(), N_OPS), (case.as_str(), 0)]) + .collect(); + let next = AtomicUsize::new(0); + let counts: Vec>> = runs.iter().map(|_| OnceLock::new()).collect(); + thread::scope(|scope| { + for _ in 0..jobs.clamp(1, runs.len()) { + scope.spawn(|| loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(&(case, n)) = runs.get(i) else { break }; + let _ = counts[i].set(total_ir(valgrind, &exe, case, n)); + }); + } + }); + let counts = counts + .into_iter() + .map(|count| count.into_inner().expect("every run was measured")) + .collect::, String>>()?; + cases + .iter() + .zip(counts.chunks(2)) + .map(|(case, pair)| { + let (full, setup) = (pair[0], pair[1]); + match full.checked_sub(setup) { + Some(delta) if delta > 0 => Ok((case.clone(), (delta + N_OPS / 2) / N_OPS)), + _ => Err(format!( + "{case}: Ir[{N_OPS}] {full} is not above Ir[0] {setup}" + )), + } + }) + .collect() +} + +fn thousands(n: u64) -> String { + let digits = n.to_string(); + let mut out = String::new(); + for (i, digit) in digits.chars().enumerate() { + if i > 0 && (digits.len() - i) % 3 == 0 { + out.push(','); + } + out.push(digit); + } + out +} + +/// Report lines and whether the gate passes. A case with no budget fails: +/// record it first. +fn compare(budgets: &BTreeMap, measured: &[(String, u64)]) -> (Vec, bool) { + let mut ok = true; + let lines = measured + .iter() + .map(|(case, ir)| { + let ir = *ir; + let Some(&budget) = budgets.get(case) else { + ok = false; + return format!( + "FAIL {case:28} {:>10} Ir/op no budget (run make perf-ir-update)", + thousands(ir) + ); + }; + let pct = (ir as f64 - budget as f64) / budget as f64 * 100.0; + let verdict = if pct >= FAIL_PCT { + ok = false; + "FAIL" + } else if pct >= WARN_PCT { + "WARN" + } else if pct <= -FAIL_PCT { + "LOWER" // cheaper: ratchet it down with --update if the change touched this path + } else { + "ok" + }; + format!( + "{verdict:5} {case:28} {:>10} Ir/op budget {:>10} {pct:+.2}%", + thousands(ir), + thousands(budget) + ) + }) + .collect(); + (lines, ok) +} + +/// New budgets: only ever lower, unless the increase is deliberate. +fn ratchet(budgets: &mut BTreeMap, measured: &[(String, u64)], allow_increase: bool) { + for (case, ir) in measured { + let budget = budgets.entry(case.clone()).or_insert(*ir); + if allow_increase || *ir < *budget { + *budget = *ir; + } + } +} + +const USAGE: &str = "\ +usage: cargo bench --features encryption --bench perf_ir -- [options] + (or make perf-ir / make perf-ir-update) + +Measures Ir/op for each case under cachegrind and compares it with the budget +in benches/perf_ir_baselines.json. Exit 0 pass, 1 regression, 2 error. + +options: + --case measure only this case (repeatable) + --update write lower measured figures back as budgets + --allow-increase with --update, also raise budgets + --jobs cachegrind runs at a time (default: min(8, cores)) + +perf_ir run the measured process: setup, one warm-up op, n ops"; + +fn gate(args: &[String]) -> Result { + let (mut update, mut allow_increase, mut cases) = (false, false, Vec::new()); + let mut jobs = thread::available_parallelism() + .map_or(1, |n| n.get()) + .min(8); + let mut args = args.iter(); + while let Some(arg) = args.next() { + match arg.as_str() { + "--update" => update = true, + "--allow-increase" => allow_increase = true, + "--case" => cases.push(args.next().ok_or("--case needs a case id")?.clone()), + "--jobs" => { + jobs = args + .next() + .and_then(|n| n.parse().ok()) + .ok_or("--jobs needs a number")? + } + "-h" | "--help" => { + println!("{USAGE}\n\ncases: {}", case_ids().join(" ")); + return Ok(true); + } + other => return Err(format!("unknown argument {other}\n\n{USAGE}")), + } + } + if allow_increase && !update { + return Err("--allow-increase only applies with --update".into()); + } + let all = case_ids(); + if let Some(unknown) = cases.iter().find(|case| !all.contains(case)) { + return Err(format!("unknown case {unknown}; cases: {}", all.join(" "))); + } + let every_case = cases.is_empty(); + if every_case { + cases = all; + } + + let valgrind_bin = + which("valgrind").ok_or("valgrind not found on PATH: install it (apt install valgrind)")?; + let text = fs::read_to_string(BASELINES).map_err(|e| format!("{BASELINES}: {e}"))?; + let mut baselines: Baselines = + serde_json::from_str(&text).map_err(|e| format!("{BASELINES}: {e}"))?; + let key = platform_key(); + let (rustc, valgrind) = (version("rustc", 1), version("valgrind", 0)); + let platform = baselines.platforms.entry(key.clone()).or_default(); + if !platform.budgets.is_empty() && (platform.rustc != rustc || platform.valgrind != valgrind) { + let recorded = format!( + "budgets for {key} were recorded with rustc {} and {}; this is rustc {rustc} and {valgrind}", + platform.rustc, platform.valgrind + ); + // One budget set, one toolchain: ratcheting down across toolchains + // would mix the two, and a partial re-record would label budgets it + // never measured with the new versions. + if update && !(allow_increase && every_case) { + return Err(format!( + "{recorded}. Re-record every case on one toolchain: --update --allow-increase, no --case." + )); + } + println!("note: {recorded}. A toolchain change moves the counts."); + } + + let measured = measure(&valgrind_bin, &cases, jobs)?; + println!( + "cachekit-core Ir/op on {key} (rustc {rustc}, {valgrind}): (Ir[{N_OPS}] - Ir[0]) / {N_OPS}" + ); + // With --update, report against the budgets as written, so the only FAIL + // left is a regression that --allow-increase did not accept. + if update { + ratchet(&mut platform.budgets, &measured, allow_increase); + platform.rustc = rustc; + platform.valgrind = valgrind; + } + let (lines, ok) = compare(&platform.budgets, &measured); + println!("{}", lines.join("\n")); + if update { + baselines.method = format!( + "cachegrind Ir per op, (Ir[{N_OPS}] - Ir[0]) / {N_OPS}, one process per case, \ + empty environment; written by benches/perf_ir.rs --update" + ); + let json = serde_json::to_string_pretty(&baselines).map_err(|e| e.to_string())?; + fs::write(BASELINES, json + "\n").map_err(|e| format!("{BASELINES}: {e}"))?; + println!("budgets written to benches/perf_ir_baselines.json"); + } + println!( + "{}", + match (ok, update) { + (true, _) => "PASS", + (false, false) => "FAIL: a case regressed by 1% or more, or has no budget", + (false, true) => + "FAIL: a regression stays over budget; raise it only with --allow-increase", + } + ); + Ok(ok) +} + +fn main() -> ExitCode { + // `cargo bench` appends --bench to the arguments it passes through. + let args: Vec = env::args().skip(1).filter(|arg| arg != "--bench").collect(); + if let [mode, case, n] = args.as_slice() { + if mode == "run" { + run(case, n.parse().expect("n is a number")); + return ExitCode::SUCCESS; + } + } + match gate(&args) { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::from(1), + Err(err) => { + eprintln!("perf_ir: {err}"); + ExitCode::from(2) + } + } +} diff --git a/benches/perf_ir_baselines.json b/benches/perf_ir_baselines.json new file mode 100644 index 0000000..5f19736 --- /dev/null +++ b/benches/perf_ir_baselines.json @@ -0,0 +1,33 @@ +{ + "method": "cachegrind Ir per op, (Ir[1000] - Ir[0]) / 1000, one process per case, empty environment; written by benches/perf_ir.rs --update", + "platforms": { + "x86_64-linux": { + "rustc": "1.98.1", + "valgrind": "valgrind-3.22.0", + "budgets": { + "decrypt/1024": 5575, + "decrypt/64": 2600, + "decrypt/65536": 235243, + "encrypt/1024": 6432, + "encrypt/64": 3253, + "encrypt/65536": 301457, + "hkdf": 29047, + "keyring_decrypt/1024": 34715, + "keyring_decrypt/64": 31730, + "keyring_decrypt/65536": 264373, + "prescan/1024": 982, + "prescan/64": 880, + "prescan/65536": 932, + "retrieve/1024": 7558, + "retrieve/64": 2722, + "retrieve/65536": 416730, + "store/1024": 29644, + "store/64": 12277, + "store/65536": 1080278, + "tenant_keyring_decrypt/1024": 5598, + "tenant_keyring_decrypt/64": 2623, + "tenant_keyring_decrypt/65536": 235234 + } + } + } +}