From fdd50c727bf906e40453bbc2c12e06c907c5d243 Mon Sep 17 00:00:00 2001 From: Wei Wu Date: Thu, 30 Jul 2026 17:03:42 +0800 Subject: [PATCH 1/3] Add reproducible K-means comparison benchmark Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9e62a713-1904-4c00-92f2-c81851395ecf --- diskann-benchmark/Cargo.toml | 3 + .../example/issue-939-kmeans-disk.json | 33 + .../example/issue-939-kmeans-init.json | 33 + .../issue-939-kmeans-quantization.json | 33 + diskann-benchmark/src/inputs/kmeans.rs | 114 ++ diskann-benchmark/src/inputs/mod.rs | 1 + diskann-benchmark/src/kmeans/comparison.rs | 344 ++++ diskann-benchmark/src/kmeans/mod.rs | 25 + diskann-benchmark/src/main.rs | 2 + issue939-benchmark-results/disk-baseline.json | 1406 +++++++++++++++++ .../kmeans-baseline-results.md | 57 + .../quantization-baseline.json | 1406 +++++++++++++++++ 12 files changed, 3457 insertions(+) create mode 100644 diskann-benchmark/example/issue-939-kmeans-disk.json create mode 100644 diskann-benchmark/example/issue-939-kmeans-init.json create mode 100644 diskann-benchmark/example/issue-939-kmeans-quantization.json create mode 100644 diskann-benchmark/src/inputs/kmeans.rs create mode 100644 diskann-benchmark/src/kmeans/comparison.rs create mode 100644 diskann-benchmark/src/kmeans/mod.rs create mode 100644 issue939-benchmark-results/disk-baseline.json create mode 100644 issue939-benchmark-results/kmeans-baseline-results.md create mode 100644 issue939-benchmark-results/quantization-baseline.json diff --git a/diskann-benchmark/Cargo.toml b/diskann-benchmark/Cargo.toml index 31739244c..564b197ef 100644 --- a/diskann-benchmark/Cargo.toml +++ b/diskann-benchmark/Cargo.toml @@ -82,3 +82,6 @@ disk-index = [ "dep:opentelemetry_sdk", "dep:scopeguard", ] + +# Compare the legacy disk and quantization K-means implementations. +kmeans-comparison = ["dep:diskann-disk"] diff --git a/diskann-benchmark/example/issue-939-kmeans-disk.json b/diskann-benchmark/example/issue-939-kmeans-disk.json new file mode 100644 index 000000000..95ee0028a --- /dev/null +++ b/diskann-benchmark/example/issue-939-kmeans-disk.json @@ -0,0 +1,33 @@ +{ + "search_directories": [], + "jobs": [ + { + "type": "kmeans-comparison", + "content": { + "implementation": "disk", + "phase": "all", + "num_points": 50000, + "dimensions": [4, 32, 128, 384, 768, 1024, 3072], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [1], + "measurements": 30, + "seed": 42 + } + }, + { + "type": "kmeans-comparison", + "content": { + "implementation": "disk", + "phase": "all", + "num_points": 50000, + "dimensions": [4], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [2, 4, 8], + "measurements": 30, + "seed": 42 + } + } + ] +} diff --git a/diskann-benchmark/example/issue-939-kmeans-init.json b/diskann-benchmark/example/issue-939-kmeans-init.json new file mode 100644 index 000000000..95387e815 --- /dev/null +++ b/diskann-benchmark/example/issue-939-kmeans-init.json @@ -0,0 +1,33 @@ +{ + "search_directories": [], + "jobs": [ + { + "type": "kmeans-comparison", + "content": { + "implementation": "disk", + "phase": "init", + "num_points": 50000, + "dimensions": [4, 32, 128, 384, 768, 1024, 3072], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [1], + "measurements": 30, + "seed": 42 + } + }, + { + "type": "kmeans-comparison", + "content": { + "implementation": "quantization", + "phase": "init", + "num_points": 50000, + "dimensions": [4, 32, 128, 384, 768, 1024, 3072], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [1], + "measurements": 30, + "seed": 42 + } + } + ] +} diff --git a/diskann-benchmark/example/issue-939-kmeans-quantization.json b/diskann-benchmark/example/issue-939-kmeans-quantization.json new file mode 100644 index 000000000..d8f23c24f --- /dev/null +++ b/diskann-benchmark/example/issue-939-kmeans-quantization.json @@ -0,0 +1,33 @@ +{ + "search_directories": [], + "jobs": [ + { + "type": "kmeans-comparison", + "content": { + "implementation": "quantization", + "phase": "all", + "num_points": 50000, + "dimensions": [4, 32, 128, 384, 768, 1024, 3072], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [1], + "measurements": 30, + "seed": 42 + } + }, + { + "type": "kmeans-comparison", + "content": { + "implementation": "quantization", + "phase": "all", + "num_points": 50000, + "dimensions": [4], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [2, 4, 8], + "measurements": 30, + "seed": 42 + } + } + ] +} diff --git a/diskann-benchmark/src/inputs/kmeans.rs b/diskann-benchmark/src/inputs/kmeans.rs new file mode 100644 index 000000000..79a40a2d8 --- /dev/null +++ b/diskann-benchmark/src/inputs/kmeans.rs @@ -0,0 +1,114 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +use std::num::NonZeroUsize; + +use diskann_benchmark_runner::Checker; +use serde::{Deserialize, Serialize}; + +use super::{as_input, Example}; + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum KmeansImplementation { + Disk, + Quantization, +} + +impl std::fmt::Display for KmeansImplementation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Disk => write!(f, "disk"), + Self::Quantization => write!(f, "quantization"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum KmeansPhase { + All, + Init, +} + +impl std::fmt::Display for KmeansPhase { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::All => write!(f, "all"), + Self::Init => write!(f, "init"), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct KmeansComparison { + pub(crate) implementation: KmeansImplementation, + pub(crate) phase: KmeansPhase, + pub(crate) num_points: NonZeroUsize, + pub(crate) dimensions: Vec, + pub(crate) center_counts: Vec, + pub(crate) max_iterations: NonZeroUsize, + pub(crate) thread_counts: Vec, + pub(crate) measurements: NonZeroUsize, + pub(crate) seed: u64, +} + +impl KmeansComparison { + pub(crate) const fn tag() -> &'static str { + "kmeans-comparison" + } + + pub(crate) fn validate(&mut self, _checker: &mut Checker) -> anyhow::Result<()> { + anyhow::ensure!(!self.dimensions.is_empty(), "dimensions cannot be empty"); + anyhow::ensure!( + !self.center_counts.is_empty(), + "center_counts cannot be empty" + ); + anyhow::ensure!( + !self.thread_counts.is_empty(), + "thread_counts cannot be empty" + ); + anyhow::ensure!( + self.center_counts + .iter() + .all(|count| count.get() <= self.num_points.get()), + "center counts cannot exceed num_points" + ); + Ok(()) + } +} + +impl Example for KmeansComparison { + fn example() -> Self { + Self { + implementation: KmeansImplementation::Quantization, + phase: KmeansPhase::All, + num_points: NonZeroUsize::new(10_000).unwrap(), + dimensions: vec![NonZeroUsize::new(128).unwrap()], + center_counts: vec![NonZeroUsize::new(64).unwrap()], + max_iterations: NonZeroUsize::new(3).unwrap(), + thread_counts: vec![NonZeroUsize::new(1).unwrap()], + measurements: NonZeroUsize::new(10).unwrap(), + seed: 42, + } + } +} + +as_input!(KmeansComparison); + +impl std::fmt::Display for KmeansComparison { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "K-means Implementation Benchmark\n")?; + writeln!(f, "{:>18}: {}", "implementation", self.implementation)?; + writeln!(f, "{:>18}: {}", "phase", self.phase)?; + writeln!(f, "{:>18}: {}", "points", self.num_points)?; + writeln!(f, "{:>18}: {:?}", "dimensions", self.dimensions)?; + writeln!(f, "{:>18}: {:?}", "center counts", self.center_counts)?; + writeln!(f, "{:>18}: {}", "max iterations", self.max_iterations)?; + writeln!(f, "{:>18}: {:?}", "thread counts", self.thread_counts)?; + writeln!(f, "{:>18}: {}", "measurements", self.measurements)?; + writeln!(f, "{:>18}: {}", "seed", self.seed) + } +} diff --git a/diskann-benchmark/src/inputs/mod.rs b/diskann-benchmark/src/inputs/mod.rs index 978d93cb3..d2c87d667 100644 --- a/diskann-benchmark/src/inputs/mod.rs +++ b/diskann-benchmark/src/inputs/mod.rs @@ -8,6 +8,7 @@ pub(crate) mod exhaustive; pub(crate) mod filters; pub(crate) mod flat; pub(crate) mod graph_index; +pub(crate) mod kmeans; pub(crate) mod multi_vector; pub(crate) mod post_processor; pub(crate) mod save_and_load; diff --git a/diskann-benchmark/src/kmeans/comparison.rs b/diskann-benchmark/src/kmeans/comparison.rs new file mode 100644 index 000000000..cc76633be --- /dev/null +++ b/diskann-benchmark/src/kmeans/comparison.rs @@ -0,0 +1,344 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +use std::{ + fmt, + hash::{DefaultHasher, Hash, Hasher}, + io::Write, + time::Instant, +}; + +use diskann_benchmark_runner::{ + benchmark::{MatchContext, Score}, + utils::{percentiles, MicroSeconds}, + Benchmark, Checkpoint, Output, Registry, +}; +use diskann_disk::utils::{k_meanspp_selecting_pivots, run_lloyds}; +use diskann_providers::utils::{create_thread_pool, RayonThreadPool}; +use diskann_quantization::algorithms::kmeans::{lloyds::lloyds, plusplus::kmeans_plusplus_into}; +use diskann_utils::views::Matrix; +use diskann_vector::{distance::SquaredL2, PureDistanceFunction}; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use serde::Serialize; + +use crate::inputs::kmeans::{KmeansComparison, KmeansImplementation, KmeansPhase}; + +pub(super) fn register(registry: &mut Registry) -> anyhow::Result<()> { + registry.register("kmeans-comparison", Comparison)?; + Ok(()) +} + +struct Comparison; + +impl Benchmark for Comparison { + type Input = KmeansComparison; + type Output = ComparisonOutput; + + fn try_match(&self, _input: &Self::Input, context: &MatchContext) -> Score { + context.success(0) + } + + fn description(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "Runs one K-means implementation with phase telemetry") + } + + fn run( + &self, + input: &Self::Input, + _checkpoint: Checkpoint<'_>, + mut output: &mut dyn Output, + ) -> anyhow::Result { + writeln!(output, "{}", input)?; + let mut workloads = Vec::new(); + + for &threads in &input.thread_counts { + let pool = create_thread_pool(threads.get())?; + for &dim in &input.dimensions { + for &num_centers in &input.center_counts { + let workload = Workload { + num_points: input.num_points.get(), + dim: dim.get(), + num_centers: num_centers.get(), + max_iterations: input.max_iterations.get(), + threads: threads.get(), + }; + let result = run_workload( + input.implementation, + input.phase, + workload, + input.measurements.get(), + input.seed, + &pool, + )?; + writeln!(output, "{}", result)?; + workloads.push(result); + } + } + } + + Ok(ComparisonOutput { + implementation: input.implementation, + phase: input.phase, + workloads, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +struct Workload { + num_points: usize, + dim: usize, + num_centers: usize, + max_iterations: usize, + threads: usize, +} + +#[derive(Debug, Serialize)] +struct PhaseMeasurements { + samples: Vec, + percentiles: percentiles::Percentiles, +} + +#[derive(Debug, Serialize)] +struct WorkloadResult { + workload: Workload, + initialization: PhaseMeasurements, + #[serde(skip_serializing_if = "Option::is_none")] + lloyds: Option, + total: PhaseMeasurements, + center_hash: u64, + objective: f64, +} + +#[derive(Debug, Serialize)] +struct ComparisonOutput { + implementation: KmeansImplementation, + phase: KmeansPhase, + workloads: Vec, +} + +impl fmt::Display for WorkloadResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{} points x {} dimensions x {} centers, {} iterations, {} threads: init median {:.3} ms", + self.workload.num_points, + self.workload.dim, + self.workload.num_centers, + self.workload.max_iterations, + self.workload.threads, + self.initialization.percentiles.median / 1_000.0, + )?; + if let Some(lloyds) = &self.lloyds { + write!( + f, + ", Lloyd median {:.3} ms", + lloyds.percentiles.median / 1_000.0 + )?; + } + writeln!( + f, + ", total median {:.3} ms", + self.total.percentiles.median / 1_000.0 + ) + } +} + +#[derive(Debug)] +struct Measurement { + initialization: MicroSeconds, + lloyds: Option, + total: MicroSeconds, + centers: Vec, +} + +fn run_workload( + implementation: KmeansImplementation, + phase: KmeansPhase, + workload: Workload, + num_measurements: usize, + seed: u64, + pool: &RayonThreadPool, +) -> anyhow::Result { + let data_seed = seed + .wrapping_add(workload.dim as u64) + .wrapping_add(workload.num_centers as u64); + let mut rng = StdRng::seed_from_u64(data_seed); + let data: Vec = (0..workload.num_points * workload.dim) + .map(|_| rng.random_range(-1.0..1.0)) + .collect(); + let data_matrix = Matrix::try_from( + data.clone().into_boxed_slice(), + workload.num_points, + workload.dim, + ) + .map_err(|_| anyhow::anyhow!("generated data matrix has an invalid shape"))?; + + let run = || { + run_measurement( + implementation, + phase, + &data, + &data_matrix, + workload, + data_seed, + pool, + ) + }; + + let quality = run()?; + let objective = objective(&data, &quality.centers, workload); + let center_hash = hash_centers(&quality.centers); + + let mut initialization = Vec::with_capacity(num_measurements); + let mut lloyds = (phase == KmeansPhase::All).then(|| Vec::with_capacity(num_measurements)); + let mut total = Vec::with_capacity(num_measurements); + for _ in 0..num_measurements { + let measurement = run()?; + initialization.push(measurement.initialization); + if let (Some(samples), Some(sample)) = (&mut lloyds, measurement.lloyds) { + samples.push(sample); + } + total.push(measurement.total); + } + + Ok(WorkloadResult { + workload, + initialization: make_phase_measurements(initialization)?, + lloyds: lloyds.map(make_phase_measurements).transpose()?, + total: make_phase_measurements(total)?, + center_hash, + objective, + }) +} + +fn run_measurement( + implementation: KmeansImplementation, + phase: KmeansPhase, + data: &[f32], + data_matrix: &Matrix, + workload: Workload, + seed: u64, + pool: &RayonThreadPool, +) -> anyhow::Result { + match implementation { + KmeansImplementation::Disk => run_disk(phase, data, workload, seed, pool), + KmeansImplementation::Quantization => { + run_quantization(phase, data_matrix, workload, seed, pool) + } + } +} + +fn run_disk( + phase: KmeansPhase, + data: &[f32], + workload: Workload, + seed: u64, + pool: &RayonThreadPool, +) -> anyhow::Result { + let mut centers = vec![0.0; workload.num_centers * workload.dim]; + let mut rng = StdRng::seed_from_u64(seed); + let mut canceled = false; + let total_start = Instant::now(); + let initialization_start = Instant::now(); + k_meanspp_selecting_pivots( + data, + workload.num_points, + workload.dim, + &mut centers, + workload.num_centers, + &mut rng, + &mut canceled, + pool.as_ref(), + )?; + let initialization = initialization_start.elapsed().into(); + let lloyds = if phase == KmeansPhase::All { + let start = Instant::now(); + run_lloyds( + data, + workload.num_points, + workload.dim, + &mut centers, + workload.num_centers, + workload.max_iterations, + &mut canceled, + pool.as_ref(), + )?; + Some(start.elapsed().into()) + } else { + None + }; + Ok(Measurement { + initialization, + lloyds, + total: total_start.elapsed().into(), + centers, + }) +} + +fn run_quantization( + phase: KmeansPhase, + data: &Matrix, + workload: Workload, + seed: u64, + pool: &RayonThreadPool, +) -> anyhow::Result { + let mut centers = Matrix::new(0.0, workload.num_centers, workload.dim); + let mut rng = StdRng::seed_from_u64(seed); + let total_start = Instant::now(); + let initialization_start = Instant::now(); + pool.install(|| kmeans_plusplus_into(centers.as_mut_view(), data.as_view(), &mut rng))?; + let initialization = initialization_start.elapsed().into(); + let lloyds = if phase == KmeansPhase::All { + let start = Instant::now(); + pool.install(|| { + lloyds( + data.as_view(), + centers.as_mut_view(), + workload.max_iterations, + ) + }); + Some(start.elapsed().into()) + } else { + None + }; + Ok(Measurement { + initialization, + lloyds, + total: total_start.elapsed().into(), + centers: centers.into_inner().into(), + }) +} + +fn make_phase_measurements(mut samples: Vec) -> anyhow::Result { + let percentiles = percentiles::compute_percentiles(&mut samples)?; + Ok(PhaseMeasurements { + samples, + percentiles, + }) +} + +fn hash_centers(centers: &[f32]) -> u64 { + let mut hasher = DefaultHasher::new(); + centers + .iter() + .for_each(|value| value.to_bits().hash(&mut hasher)); + hasher.finish() +} + +fn objective(data: &[f32], centers: &[f32], workload: Workload) -> f64 { + data.chunks_exact(workload.dim) + .map(|point| { + centers + .chunks_exact(workload.dim) + .map(|center| { + let distance: f32 = SquaredL2::evaluate(point, center); + f64::from(distance) + }) + .min_by(f64::total_cmp) + .expect("the benchmark requires at least one center") + }) + .sum() +} diff --git a/diskann-benchmark/src/kmeans/mod.rs b/diskann-benchmark/src/kmeans/mod.rs new file mode 100644 index 000000000..7c3d60543 --- /dev/null +++ b/diskann-benchmark/src/kmeans/mod.rs @@ -0,0 +1,25 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +use diskann_benchmark_runner::Registry; + +cfg_if::cfg_if! { + if #[cfg(feature = "kmeans-comparison")] { + mod comparison; + + pub(crate) fn register_benchmarks(registry: &mut Registry) -> anyhow::Result<()> { + comparison::register(registry) + } + } else { + pub(crate) fn register_benchmarks(registry: &mut Registry) -> anyhow::Result<()> { + registry.register_partially_gated::( + "kmeans-comparison", + diskann_benchmark_runner::Features::new("kmeans-comparison"), + "Legacy disk and quantization K-means comparison", + )?; + Ok(()) + } + } +} diff --git a/diskann-benchmark/src/main.rs b/diskann-benchmark/src/main.rs index cf9a0ea52..cc3d46595 100644 --- a/diskann-benchmark/src/main.rs +++ b/diskann-benchmark/src/main.rs @@ -9,6 +9,7 @@ mod filters; mod flat; mod index; mod inputs; +mod kmeans; mod multi_vector; mod utils; @@ -54,6 +55,7 @@ impl Cli { flat::register_benchmarks(&mut registry)?; index::register_benchmarks(&mut registry)?; filters::register_benchmarks(&mut registry)?; + kmeans::register_benchmarks(&mut registry)?; multi_vector::register_benchmarks(&mut registry)?; self.app.run(®istry, output) diff --git a/issue939-benchmark-results/disk-baseline.json b/issue939-benchmark-results/disk-baseline.json new file mode 100644 index 000000000..3c456c726 --- /dev/null +++ b/issue939-benchmark-results/disk-baseline.json @@ -0,0 +1,1406 @@ +[ + { + "input": { + "content": { + "center_counts": [ + 256 + ], + "dimensions": [ + 4, + 32, + 128, + 384, + 768, + 1024, + 3072 + ], + "implementation": "disk", + "max_iterations": 10, + "measurements": 30, + "num_points": 50000, + "phase": "all", + "seed": 42, + "thread_counts": [ + 1 + ] + }, + "type": "kmeans-comparison" + }, + "results": { + "implementation": "disk", + "phase": "all", + "workloads": [ + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 79659.63333333333, + "median": 79131.5, + "minimum": 76601, + "p90": 83854, + "p99": 89952 + }, + "samples": [ + 76601, + 76669, + 76876, + 76878, + 77154, + 77233, + 77281, + 77281, + 77424, + 77439, + 77877, + 77889, + 78169, + 78943, + 78988, + 79275, + 79439, + 79708, + 79943, + 80103, + 80129, + 80575, + 80950, + 81294, + 82331, + 82713, + 82724, + 83854, + 84097, + 89952 + ] + }, + "lloyds": { + "percentiles": { + "mean": 155633.43333333332, + "median": 154482.5, + "minimum": 147034, + "p90": 164978, + "p99": 176861 + }, + "samples": [ + 147034, + 148198, + 148353, + 149056, + 150131, + 150475, + 151038, + 151170, + 151256, + 152991, + 152991, + 153086, + 153932, + 153973, + 154467, + 154498, + 154575, + 154862, + 155085, + 155281, + 155500, + 156629, + 157157, + 157255, + 160326, + 161119, + 162648, + 164978, + 174078, + 176861 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 235294.3, + "median": 233885.0, + "minimum": 223911, + "p90": 251072, + "p99": 258156 + }, + "samples": [ + 223911, + 225777, + 226369, + 228452, + 228915, + 229633, + 229752, + 230527, + 230811, + 230882, + 230965, + 231069, + 231858, + 232017, + 233827, + 233943, + 233986, + 234030, + 235226, + 235630, + 236244, + 236305, + 236733, + 237223, + 239766, + 239882, + 249076, + 251072, + 256792, + 258156 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 94114.1, + "median": 93925.0, + "minimum": 92205, + "p90": 95439, + "p99": 100068 + }, + "samples": [ + 92205, + 92535, + 92883, + 92974, + 93035, + 93195, + 93244, + 93283, + 93315, + 93390, + 93566, + 93648, + 93774, + 93820, + 93914, + 93936, + 93977, + 94037, + 94050, + 94141, + 94271, + 94524, + 94548, + 94773, + 94910, + 95088, + 95201, + 95439, + 95679, + 100068 + ] + }, + "lloyds": { + "percentiles": { + "mean": 239327.06666666668, + "median": 238535.5, + "minimum": 225916, + "p90": 250099, + "p99": 257435 + }, + "samples": [ + 225916, + 228693, + 229775, + 230829, + 232016, + 232965, + 232989, + 233461, + 233712, + 234024, + 234312, + 236521, + 237565, + 237705, + 237825, + 239246, + 240559, + 240901, + 241168, + 241537, + 242690, + 242882, + 243543, + 245916, + 245961, + 246515, + 248121, + 250099, + 254931, + 257435 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 333442.26666666666, + "median": 332902.0, + "minimum": 320058, + "p90": 343344, + "p99": 352875 + }, + "samples": [ + 320058, + 322966, + 323091, + 325300, + 325525, + 325603, + 326902, + 327028, + 327103, + 327286, + 328074, + 329718, + 329910, + 330449, + 332737, + 333067, + 335084, + 335648, + 336658, + 336739, + 338068, + 338370, + 339610, + 340466, + 340493, + 340970, + 342159, + 343344, + 347967, + 352875 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 203285.16666666666, + "median": 202912.0, + "minimum": 179379, + "p90": 226077, + "p99": 234061 + }, + "samples": [ + 179379, + 181875, + 182222, + 182507, + 184072, + 185428, + 185530, + 191201, + 193686, + 195786, + 197330, + 197628, + 199238, + 200141, + 200910, + 204914, + 206783, + 207705, + 211204, + 211390, + 211509, + 212208, + 213889, + 215755, + 216304, + 218347, + 224306, + 226077, + 227170, + 234061 + ] + }, + "lloyds": { + "percentiles": { + "mean": 553646.4666666667, + "median": 553041.0, + "minimum": 546613, + "p90": 562758, + "p99": 570063 + }, + "samples": [ + 546613, + 546671, + 546930, + 547273, + 548561, + 548982, + 549055, + 549234, + 549281, + 549377, + 549576, + 550385, + 550627, + 551412, + 552624, + 553458, + 554046, + 554119, + 554334, + 554773, + 555190, + 555281, + 556255, + 558950, + 559000, + 559456, + 560680, + 562758, + 564430, + 570063 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 756932.8666666667, + "median": 760493.5, + "minimum": 731253, + "p90": 779081, + "p99": 781504 + }, + "samples": [ + 731253, + 733128, + 733499, + 734849, + 736554, + 741684, + 747800, + 748289, + 748743, + 749375, + 750977, + 751087, + 754368, + 758479, + 760196, + 760791, + 761192, + 761265, + 762018, + 763467, + 764961, + 765342, + 766141, + 766240, + 766706, + 773008, + 775255, + 779081, + 780734, + 781504 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 6161376117261214841, + "initialization": { + "percentiles": { + "mean": 895038.2666666667, + "median": 887169.5, + "minimum": 867962, + "p90": 918972, + "p99": 1045898 + }, + "samples": [ + 867962, + 869142, + 870156, + 872226, + 872495, + 873322, + 874188, + 875112, + 875414, + 875681, + 878173, + 880257, + 881962, + 884594, + 885229, + 889110, + 889402, + 891125, + 894271, + 895721, + 897904, + 900007, + 905036, + 905405, + 912686, + 913828, + 918017, + 918972, + 937853, + 1045898 + ] + }, + "lloyds": { + "percentiles": { + "mean": 1380407.7, + "median": 1373521.0, + "minimum": 1349407, + "p90": 1410464, + "p99": 1458439 + }, + "samples": [ + 1349407, + 1360723, + 1365408, + 1366037, + 1366503, + 1367177, + 1367498, + 1368092, + 1368622, + 1369004, + 1369197, + 1370242, + 1371530, + 1372906, + 1372946, + 1374096, + 1374719, + 1375040, + 1376888, + 1379467, + 1383556, + 1384121, + 1390363, + 1392746, + 1394160, + 1396833, + 1402641, + 1410464, + 1413406, + 1458439 + ] + }, + "objective": 6189761.912239075, + "total": { + "percentiles": { + "mean": 2275447.3333333335, + "median": 2264663.5, + "minimum": 2223596, + "p90": 2318812, + "p99": 2419996 + }, + "samples": [ + 2223596, + 2235462, + 2236405, + 2237766, + 2240521, + 2244212, + 2244420, + 2245133, + 2245197, + 2252687, + 2252791, + 2258304, + 2259949, + 2260932, + 2264408, + 2264919, + 2269330, + 2274709, + 2282930, + 2284636, + 2289159, + 2290722, + 2294169, + 2295861, + 2301574, + 2310799, + 2316471, + 2318812, + 2347550, + 2419996 + ] + }, + "workload": { + "dim": 384, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 15316337929231845703, + "initialization": { + "percentiles": { + "mean": 1775828.8333333333, + "median": 1771880.5, + "minimum": 1729521, + "p90": 1849517, + "p99": 1908204 + }, + "samples": [ + 1729521, + 1730366, + 1731422, + 1732395, + 1734035, + 1738329, + 1738598, + 1742626, + 1746081, + 1762640, + 1766394, + 1766678, + 1767849, + 1770057, + 1771639, + 1772122, + 1774084, + 1774322, + 1775715, + 1778884, + 1785078, + 1785598, + 1787744, + 1788354, + 1793227, + 1801558, + 1822079, + 1849517, + 1849749, + 1908204 + ] + }, + "lloyds": { + "percentiles": { + "mean": 2634890.8666666667, + "median": 2635229.5, + "minimum": 2577671, + "p90": 2740652, + "p99": 2790580 + }, + "samples": [ + 2577671, + 2580800, + 2581632, + 2587964, + 2588263, + 2588978, + 2589803, + 2592294, + 2594696, + 2601937, + 2604562, + 2611676, + 2624510, + 2632094, + 2632842, + 2637617, + 2637993, + 2640576, + 2640966, + 2642032, + 2643689, + 2645010, + 2649880, + 2649888, + 2650747, + 2656501, + 2674644, + 2740652, + 2756229, + 2790580 + ] + }, + "objective": 12551465.794921875, + "total": { + "percentiles": { + "mean": 4410721.033333333, + "median": 4404248.5, + "minimum": 4310068, + "p90": 4578309, + "p99": 4698785 + }, + "samples": [ + 4310068, + 4319130, + 4320170, + 4320232, + 4321816, + 4322299, + 4331605, + 4333361, + 4334046, + 4370413, + 4387151, + 4389642, + 4390562, + 4402152, + 4404014, + 4404483, + 4412860, + 4417427, + 4418012, + 4423965, + 4426565, + 4428624, + 4428931, + 4435260, + 4437633, + 4476203, + 4487511, + 4578309, + 4590402, + 4698785 + ] + }, + "workload": { + "dim": 768, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 10519214377489719790, + "initialization": { + "percentiles": { + "mean": 2344325.8, + "median": 2341492.0, + "minimum": 2307992, + "p90": 2378460, + "p99": 2400327 + }, + "samples": [ + 2307992, + 2313691, + 2315361, + 2316423, + 2320088, + 2320938, + 2324819, + 2331982, + 2334549, + 2335596, + 2335782, + 2336978, + 2338426, + 2338502, + 2341034, + 2341950, + 2342020, + 2342229, + 2344795, + 2345911, + 2348853, + 2354183, + 2356079, + 2360734, + 2366677, + 2369739, + 2370482, + 2378460, + 2395174, + 2400327 + ] + }, + "lloyds": { + "percentiles": { + "mean": 3311839.8, + "median": 3305319.0, + "minimum": 3279899, + "p90": 3359864, + "p99": 3366693 + }, + "samples": [ + 3279899, + 3283934, + 3287837, + 3288667, + 3290601, + 3293056, + 3293163, + 3293843, + 3296108, + 3296173, + 3301175, + 3302390, + 3304400, + 3304510, + 3304955, + 3305683, + 3307048, + 3307705, + 3309219, + 3313695, + 3314089, + 3321171, + 3324400, + 3325108, + 3325259, + 3332562, + 3357235, + 3359864, + 3364752, + 3366693 + ] + }, + "objective": 16794760.531051636, + "total": { + "percentiles": { + "mean": 5656166.9, + "median": 5645711.5, + "minimum": 5604293, + "p90": 5730348, + "p99": 5765080 + }, + "samples": [ + 5604293, + 5604720, + 5604873, + 5607926, + 5615041, + 5620934, + 5625646, + 5628394, + 5629451, + 5634373, + 5634601, + 5638338, + 5639678, + 5643303, + 5645002, + 5646421, + 5647347, + 5649751, + 5653792, + 5659607, + 5666143, + 5667211, + 5673255, + 5675423, + 5699240, + 5716348, + 5722773, + 5730348, + 5735695, + 5765080 + ] + }, + "workload": { + "dim": 1024, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 12943831542607517319, + "initialization": { + "percentiles": { + "mean": 6971064.566666666, + "median": 6969815.0, + "minimum": 6829692, + "p90": 7097969, + "p99": 7194953 + }, + "samples": [ + 6829692, + 6856927, + 6860097, + 6864891, + 6869914, + 6902165, + 6904875, + 6907069, + 6923809, + 6924648, + 6936090, + 6939331, + 6940478, + 6949638, + 6969646, + 6969984, + 6979760, + 6982273, + 6982858, + 6998557, + 7002886, + 7004236, + 7025491, + 7026380, + 7039349, + 7041093, + 7082055, + 7097969, + 7124823, + 7194953 + ] + }, + "lloyds": { + "percentiles": { + "mean": 6754397.5, + "median": 6726029.5, + "minimum": 6678597, + "p90": 6892652, + "p99": 7029009 + }, + "samples": [ + 6678597, + 6689924, + 6693280, + 6693345, + 6693556, + 6699429, + 6699938, + 6700776, + 6701777, + 6705380, + 6705858, + 6712317, + 6717181, + 6720224, + 6722644, + 6729415, + 6730183, + 6733185, + 6742381, + 6746151, + 6757462, + 6762267, + 6762500, + 6797526, + 6802683, + 6803046, + 6850737, + 6892652, + 6958502, + 7029009 + ] + }, + "objective": 50768784.480651855, + "total": { + "percentiles": { + "mean": 13725463.233333332, + "median": 13696581.0, + "minimum": 13519617, + "p90": 13927507, + "p99": 14223963 + }, + "samples": [ + 13519617, + 13538695, + 13562787, + 13563260, + 13565669, + 13602105, + 13618205, + 13646455, + 13647257, + 13653273, + 13655019, + 13670662, + 13671762, + 13673041, + 13695176, + 13697986, + 13732148, + 13744541, + 13749038, + 13758677, + 13759574, + 13770509, + 13790069, + 13807283, + 13823907, + 13839518, + 13865572, + 13927507, + 13990622, + 14223963 + ] + }, + "workload": { + "dim": 3072, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + } + ] + } + }, + { + "input": { + "content": { + "center_counts": [ + 256 + ], + "dimensions": [ + 4 + ], + "implementation": "disk", + "max_iterations": 10, + "measurements": 30, + "num_points": 50000, + "phase": "all", + "seed": 42, + "thread_counts": [ + 2, + 4, + 8 + ] + }, + "type": "kmeans-comparison" + }, + "results": { + "implementation": "disk", + "phase": "all", + "workloads": [ + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 57078.566666666666, + "median": 56398.5, + "minimum": 55452, + "p90": 61358, + "p99": 62879 + }, + "samples": [ + 55452, + 55708, + 55798, + 55934, + 55966, + 56022, + 56130, + 56184, + 56236, + 56262, + 56313, + 56319, + 56341, + 56378, + 56381, + 56416, + 56424, + 56510, + 56800, + 56804, + 56955, + 57083, + 57290, + 57484, + 57511, + 57566, + 57676, + 61358, + 62177, + 62879 + ] + }, + "lloyds": { + "percentiles": { + "mean": 130268.4, + "median": 128980.5, + "minimum": 126875, + "p90": 135876, + "p99": 136366 + }, + "samples": [ + 126875, + 127341, + 127486, + 127649, + 127852, + 127890, + 127896, + 128247, + 128449, + 128466, + 128566, + 128583, + 128713, + 128767, + 128896, + 129065, + 129334, + 129478, + 129751, + 129885, + 130110, + 131446, + 132266, + 132638, + 132929, + 135262, + 135778, + 135876, + 136192, + 136366 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 187347.86666666667, + "median": 185514.0, + "minimum": 183578, + "p90": 192289, + "p99": 193625 + }, + "samples": [ + 183578, + 183671, + 183958, + 184159, + 184269, + 184532, + 184605, + 184844, + 184848, + 184897, + 185250, + 185326, + 185364, + 185413, + 185490, + 185538, + 185572, + 186301, + 186819, + 188661, + 189249, + 189833, + 191469, + 191819, + 191991, + 192218, + 192218, + 192289, + 192630, + 193625 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 55639.96666666667, + "median": 54880.5, + "minimum": 52849, + "p90": 59658, + "p99": 64709 + }, + "samples": [ + 52849, + 52957, + 53152, + 53182, + 53196, + 53338, + 53690, + 53848, + 54048, + 54238, + 54289, + 54430, + 54510, + 54610, + 54832, + 54929, + 55346, + 55423, + 55445, + 56125, + 56152, + 56163, + 56504, + 56713, + 57107, + 57917, + 58976, + 59658, + 60863, + 64709 + ] + }, + "lloyds": { + "percentiles": { + "mean": 130297.63333333333, + "median": 128571.0, + "minimum": 127058, + "p90": 135721, + "p99": 141838 + }, + "samples": [ + 127058, + 127114, + 127191, + 127300, + 127492, + 127610, + 127678, + 127789, + 127848, + 128121, + 128219, + 128322, + 128349, + 128369, + 128411, + 128731, + 129269, + 129286, + 129669, + 129670, + 129759, + 130284, + 132960, + 133242, + 133943, + 134343, + 135360, + 135721, + 137983, + 141838 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 185938.7, + "median": 185477.5, + "minimum": 180241, + "p90": 193080, + "p99": 196349 + }, + "samples": [ + 180241, + 180344, + 181301, + 181608, + 181661, + 182109, + 182360, + 182519, + 183148, + 183213, + 183620, + 183809, + 184119, + 184502, + 185116, + 185839, + 185983, + 186199, + 186448, + 186825, + 187152, + 187250, + 188164, + 188954, + 189210, + 191068, + 191861, + 193080, + 194109, + 196349 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 77105.2, + "median": 76757.0, + "minimum": 73008, + "p90": 80970, + "p99": 83776 + }, + "samples": [ + 73008, + 73110, + 73274, + 73505, + 73918, + 74326, + 74635, + 74943, + 75222, + 75790, + 75839, + 76298, + 76348, + 76651, + 76678, + 76836, + 76937, + 76938, + 78158, + 78339, + 78415, + 78490, + 79352, + 79760, + 79825, + 80092, + 80272, + 80970, + 81451, + 83776 + ] + }, + "lloyds": { + "percentiles": { + "mean": 175581.03333333333, + "median": 175218.5, + "minimum": 169025, + "p90": 179559, + "p99": 189482 + }, + "samples": [ + 169025, + 169071, + 170511, + 171260, + 171404, + 172406, + 172651, + 172816, + 173667, + 173667, + 173743, + 173804, + 173962, + 174675, + 174947, + 175490, + 176132, + 176473, + 176869, + 177162, + 177693, + 177973, + 178005, + 178137, + 178879, + 179003, + 179211, + 179559, + 179754, + 189482 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 252687.46666666667, + "median": 252584.5, + "minimum": 245324, + "p90": 259107, + "p99": 264118 + }, + "samples": [ + 245324, + 245415, + 246323, + 246860, + 247365, + 248198, + 248612, + 248950, + 249980, + 250515, + 250800, + 251627, + 252279, + 252378, + 252385, + 252784, + 252848, + 253630, + 253796, + 254531, + 254637, + 255076, + 255196, + 255763, + 257370, + 257371, + 257734, + 259107, + 259652, + 264118 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + } + ] + } + } +] \ No newline at end of file diff --git a/issue939-benchmark-results/kmeans-baseline-results.md b/issue939-benchmark-results/kmeans-baseline-results.md new file mode 100644 index 000000000..fd9c4d54c --- /dev/null +++ b/issue939-benchmark-results/kmeans-baseline-results.md @@ -0,0 +1,57 @@ +# Issue #939 K-means baseline + +## Experiment contract + +- Revision: `59dd0481` (`origin/main` when the baseline worktree was created) +- Worktree: `C:\src\DiskANN-issue-939-baseline` +- Branch: `wuw92/issue-939-benchmark-baseline` +- Dataset: deterministic synthetic `f32` values sampled uniformly from `[-1, 1)`, seed 42 +- Points: 50,000 +- Centers: 256 +- Lloyd iterations: 10 +- Measurements: 30 per workload +- Build: release, `diskann-benchmark` with `kmeans-comparison` + +Each measurement calls K-means++ once and, for `phase: "all"`, Lloyd once using the selected centers. The benchmark records initialization, Lloyd, and total wall-clock latency from that single execution. Fixture generation and quality calculation are outside the timed region. + +The Disk path calls `k_meanspp_selecting_pivots` followed by `run_lloyds`, passing the explicit repository Rayon pool to both functions. The old quantization path calls `kmeans_plusplus_into` followed by `lloyds`; these baseline APIs are sequential, so changing the benchmark thread count does not change their execution. + +## Dimension sweep + +Thread count is 1. Latencies are medians in milliseconds. + +| Dimension | Disk init | Quant init | Init change | Disk Lloyd | Quant Lloyd | Lloyd change | Disk total | Quant total | Total change | +|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| 4 | 79.132 | 12.890 | -83.7% | 154.482 | 26.176 | -83.1% | 233.885 | 39.269 | -83.2% | +| 32 | 93.925 | 36.040 | -61.6% | 238.536 | 147.149 | -38.3% | 332.902 | 183.473 | -44.9% | +| 128 | 202.912 | 173.231 | -14.6% | 553.041 | 663.220 | +19.9% | 760.494 | 836.838 | +10.0% | +| 384 | 887.169 | 767.457 | -13.5% | 1,373.521 | 2,045.529 | +48.9% | 2,264.664 | 2,812.458 | +24.2% | +| 768 | 1,771.880 | 1,688.607 | -4.7% | 2,635.229 | 4,117.453 | +56.2% | 4,404.248 | 5,799.877 | +31.7% | +| 1,024 | 2,341.492 | 2,330.105 | -0.5% | 3,305.319 | 5,515.243 | +66.9% | 5,645.712 | 7,842.756 | +38.9% | +| 3,072 | 6,969.815 | 7,293.668 | +4.6% | 6,726.029 | 16,666.082 | +147.8% | 13,696.581 | 23,970.544 | +75.0% | + +**[measured]** The baseline reproduces the high-dimensional gap from issue #939. Quantization crosses from faster to slower at dimension 128 and reaches a 75.0% total-latency regression at dimension 3,072. The gap is concentrated in Lloyd, which is 147.8% slower at dimension 3,072. + +## Thread sweep + +Dimension is 4. Latencies are medians in milliseconds. + +| Threads | Disk init | Quant init | Disk Lloyd | Quant Lloyd | Disk total | Quant total | +|---:|---:|---:|---:|---:|---:|---:| +| 1 | 79.132 | 12.890 | 154.482 | 26.176 | 233.885 | 39.269 | +| 2 | 56.398 | 13.002 | 128.981 | 27.000 | 185.514 | 40.026 | +| 4 | 54.880 | 13.162 | 128.571 | 26.916 | 185.477 | 40.432 | +| 8 | 76.757 | 13.419 | 175.219 | 26.468 | 252.584 | 39.963 | + +**[measured]** Old quantization total latency remains within 3.0% across 1, 2, 4, and 8 configured threads, confirming that this implementation does not use the benchmark's Rayon pool. Disk improves through four threads on this low-dimensional workload and regresses at eight threads. + +## Quality observations + +The final center hash and objective match exactly through dimension 384 and for every dimension-4 thread workload. They differ at dimensions 768, 1,024, and 3,072, so performance comparisons at those dimensions should not be interpreted as bit-identical algorithm executions. + +## Raw results + +- `disk-baseline.json` +- `quantization-baseline.json` + +The benchmark also accepts `phase: "init"` to execute and report initialization only; `issue-939-kmeans-init.json` provides a ready-to-run configuration. diff --git a/issue939-benchmark-results/quantization-baseline.json b/issue939-benchmark-results/quantization-baseline.json new file mode 100644 index 000000000..ba78edd60 --- /dev/null +++ b/issue939-benchmark-results/quantization-baseline.json @@ -0,0 +1,1406 @@ +[ + { + "input": { + "content": { + "center_counts": [ + 256 + ], + "dimensions": [ + 4, + 32, + 128, + 384, + 768, + 1024, + 3072 + ], + "implementation": "quantization", + "max_iterations": 10, + "measurements": 30, + "num_points": 50000, + "phase": "all", + "seed": 42, + "thread_counts": [ + 1 + ] + }, + "type": "kmeans-comparison" + }, + "results": { + "implementation": "quantization", + "phase": "all", + "workloads": [ + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 13093.266666666666, + "median": 12890.5, + "minimum": 12590, + "p90": 13882, + "p99": 14868 + }, + "samples": [ + 12590, + 12595, + 12610, + 12628, + 12642, + 12651, + 12667, + 12697, + 12733, + 12782, + 12801, + 12819, + 12821, + 12836, + 12879, + 12902, + 12913, + 13097, + 13154, + 13210, + 13263, + 13267, + 13294, + 13369, + 13378, + 13619, + 13750, + 13882, + 14081, + 14868 + ] + }, + "lloyds": { + "percentiles": { + "mean": 26601.7, + "median": 26176.0, + "minimum": 25704, + "p90": 27813, + "p99": 34035 + }, + "samples": [ + 25704, + 25711, + 25780, + 25784, + 25816, + 25863, + 25885, + 25892, + 25942, + 25971, + 26023, + 26057, + 26107, + 26157, + 26175, + 26177, + 26251, + 26314, + 26346, + 26350, + 26358, + 26440, + 26492, + 26742, + 27023, + 27145, + 27496, + 27813, + 28202, + 34035 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 39695.933333333334, + "median": 39269.0, + "minimum": 38364, + "p90": 41026, + "p99": 47405 + }, + "samples": [ + 38364, + 38380, + 38447, + 38505, + 38514, + 38526, + 38634, + 38675, + 38791, + 38885, + 39009, + 39055, + 39111, + 39134, + 39210, + 39328, + 39407, + 39544, + 39651, + 39736, + 39796, + 40101, + 40229, + 40259, + 40402, + 40761, + 40911, + 41026, + 41082, + 47405 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 36423.76666666667, + "median": 36040.0, + "minimum": 34810, + "p90": 39532, + "p99": 39718 + }, + "samples": [ + 34810, + 34908, + 35000, + 35108, + 35186, + 35424, + 35446, + 35566, + 35586, + 35652, + 35658, + 35705, + 35724, + 35778, + 35967, + 36113, + 36211, + 36227, + 36377, + 36378, + 36461, + 36543, + 36675, + 37424, + 37649, + 37691, + 38504, + 39532, + 39692, + 39718 + ] + }, + "lloyds": { + "percentiles": { + "mean": 147563.8, + "median": 147148.5, + "minimum": 145458, + "p90": 149065, + "p99": 158325 + }, + "samples": [ + 145458, + 145651, + 145786, + 146037, + 146063, + 146070, + 146548, + 146610, + 146633, + 146655, + 146669, + 146737, + 146753, + 146776, + 147100, + 147197, + 147421, + 147612, + 147642, + 147653, + 147838, + 147867, + 147965, + 148029, + 148058, + 148182, + 148554, + 149065, + 149960, + 158325 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 183988.83333333334, + "median": 183473.0, + "minimum": 180462, + "p90": 187335, + "p99": 194704 + }, + "samples": [ + 180462, + 181146, + 181543, + 181736, + 181900, + 182200, + 182236, + 182389, + 182533, + 182753, + 182923, + 183115, + 183240, + 183332, + 183391, + 183555, + 183717, + 183736, + 183755, + 183834, + 183964, + 184067, + 184201, + 184393, + 185072, + 186504, + 187331, + 187335, + 188598, + 194704 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 175995.96666666667, + "median": 173230.5, + "minimum": 162613, + "p90": 188954, + "p99": 218934 + }, + "samples": [ + 162613, + 163729, + 166644, + 166766, + 166928, + 166950, + 167000, + 167613, + 168615, + 169056, + 169114, + 169989, + 170169, + 171065, + 171656, + 174805, + 174857, + 175221, + 177412, + 177422, + 178665, + 181445, + 182139, + 183985, + 184953, + 185889, + 187711, + 188954, + 189580, + 218934 + ] + }, + "lloyds": { + "percentiles": { + "mean": 663505.5, + "median": 663219.5, + "minimum": 659915, + "p90": 666756, + "p99": 667433 + }, + "samples": [ + 659915, + 661219, + 662163, + 662273, + 662336, + 662509, + 662552, + 662559, + 662577, + 662577, + 662613, + 662756, + 662830, + 662898, + 663187, + 663252, + 663532, + 663615, + 663754, + 663780, + 663789, + 663884, + 664022, + 664358, + 664546, + 664803, + 665444, + 666756, + 667233, + 667433 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 839503.6666666666, + "median": 836838.5, + "minimum": 825370, + "p90": 852091, + "p99": 886368 + }, + "samples": [ + 825370, + 825893, + 826916, + 830191, + 831193, + 831388, + 831570, + 832395, + 832604, + 833356, + 833402, + 833416, + 833994, + 834162, + 835117, + 838560, + 838648, + 839003, + 840956, + 841028, + 841218, + 844344, + 846546, + 846686, + 848720, + 848838, + 848931, + 852091, + 852206, + 886368 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 6161376117261214841, + "initialization": { + "percentiles": { + "mean": 769465.5666666667, + "median": 767456.5, + "minimum": 755988, + "p90": 782713, + "p99": 800520 + }, + "samples": [ + 755988, + 758069, + 759617, + 759803, + 760377, + 761240, + 761681, + 762062, + 762525, + 762741, + 763926, + 764592, + 765516, + 765562, + 766317, + 768596, + 769937, + 770691, + 772052, + 772711, + 773004, + 773826, + 774172, + 775096, + 777029, + 779299, + 780667, + 782713, + 783638, + 800520 + ] + }, + "lloyds": { + "percentiles": { + "mean": 2045415.4333333333, + "median": 2045529.0, + "minimum": 2042191, + "p90": 2047474, + "p99": 2050228 + }, + "samples": [ + 2042191, + 2042416, + 2042593, + 2042653, + 2043341, + 2043502, + 2043676, + 2043796, + 2044051, + 2044308, + 2044408, + 2044543, + 2044781, + 2044801, + 2045356, + 2045702, + 2045772, + 2045972, + 2046475, + 2046685, + 2046703, + 2046989, + 2047042, + 2047122, + 2047132, + 2047309, + 2047471, + 2047474, + 2047971, + 2050228 + ] + }, + "objective": 6189761.912239075, + "total": { + "percentiles": { + "mean": 2814882.3333333335, + "median": 2812458.0, + "minimum": 2801961, + "p90": 2826292, + "p99": 2847642 + }, + "samples": [ + 2801961, + 2805380, + 2806080, + 2807014, + 2807276, + 2807933, + 2808538, + 2808724, + 2809001, + 2809846, + 2810216, + 2810498, + 2811059, + 2811119, + 2812267, + 2812649, + 2812883, + 2815730, + 2816346, + 2816623, + 2817255, + 2817623, + 2818482, + 2819623, + 2819879, + 2824657, + 2826218, + 2826292, + 2827656, + 2847642 + ] + }, + "workload": { + "dim": 384, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 14566160934793959414, + "initialization": { + "percentiles": { + "mean": 1696962.2333333334, + "median": 1688607.0, + "minimum": 1650823, + "p90": 1729076, + "p99": 1860238 + }, + "samples": [ + 1650823, + 1658197, + 1658233, + 1665562, + 1666267, + 1667541, + 1668434, + 1670128, + 1671154, + 1675488, + 1676122, + 1680987, + 1682994, + 1683111, + 1688004, + 1689210, + 1690134, + 1693381, + 1708517, + 1710475, + 1711042, + 1714726, + 1715402, + 1716812, + 1718861, + 1722193, + 1727809, + 1729076, + 1737946, + 1860238 + ] + }, + "lloyds": { + "percentiles": { + "mean": 4119677.3666666667, + "median": 4117452.5, + "minimum": 4101898, + "p90": 4133925, + "p99": 4136807 + }, + "samples": [ + 4101898, + 4103161, + 4109324, + 4110755, + 4113907, + 4114024, + 4114099, + 4114591, + 4114753, + 4115561, + 4115821, + 4115900, + 4116451, + 4116697, + 4116872, + 4118033, + 4118800, + 4119289, + 4121173, + 4121610, + 4126000, + 4126538, + 4126702, + 4126878, + 4127384, + 4127738, + 4131557, + 4133925, + 4134073, + 4136807 + ] + }, + "objective": 12551421.084503174, + "total": { + "percentiles": { + "mean": 5816640.933333334, + "median": 5799877.5, + "minimum": 5766724, + "p90": 5855670, + "p99": 5991796 + }, + "samples": [ + 5766724, + 5774887, + 5775071, + 5782089, + 5782459, + 5784036, + 5785112, + 5785576, + 5786156, + 5789904, + 5790242, + 5795412, + 5797694, + 5798673, + 5799789, + 5799966, + 5804235, + 5820084, + 5829994, + 5835900, + 5835902, + 5838215, + 5838892, + 5845116, + 5845528, + 5849420, + 5850738, + 5855670, + 5863948, + 5991796 + ] + }, + "workload": { + "dim": 768, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 18077977999153830296, + "initialization": { + "percentiles": { + "mean": 2329945.7333333334, + "median": 2330105.0, + "minimum": 2294280, + "p90": 2356371, + "p99": 2383549 + }, + "samples": [ + 2294280, + 2303697, + 2305339, + 2305669, + 2309027, + 2309896, + 2310610, + 2312125, + 2314649, + 2317666, + 2318330, + 2319183, + 2326239, + 2326995, + 2329769, + 2330441, + 2330678, + 2332067, + 2335466, + 2335967, + 2336688, + 2337062, + 2337522, + 2340404, + 2348113, + 2351492, + 2355929, + 2356371, + 2383149, + 2383549 + ] + }, + "lloyds": { + "percentiles": { + "mean": 5515752.433333334, + "median": 5515242.5, + "minimum": 5500971, + "p90": 5527831, + "p99": 5535827 + }, + "samples": [ + 5500971, + 5501024, + 5501642, + 5503907, + 5505187, + 5508187, + 5508363, + 5509669, + 5511690, + 5511768, + 5512275, + 5513724, + 5513764, + 5514634, + 5514980, + 5515505, + 5517262, + 5517558, + 5517804, + 5519914, + 5520844, + 5520869, + 5521251, + 5521312, + 5524630, + 5525226, + 5525997, + 5527831, + 5528958, + 5535827 + ] + }, + "objective": 16794756.62135315, + "total": { + "percentiles": { + "mean": 7845699.4, + "median": 7842755.5, + "minimum": 7799468, + "p90": 7884204, + "p99": 7908776 + }, + "samples": [ + 7799468, + 7807312, + 7814518, + 7818638, + 7819355, + 7823620, + 7824949, + 7826419, + 7829872, + 7833818, + 7834427, + 7836757, + 7838805, + 7841168, + 7841977, + 7843534, + 7844344, + 7847156, + 7850074, + 7854247, + 7854868, + 7856837, + 7859637, + 7863520, + 7868028, + 7868756, + 7871436, + 7884204, + 7904462, + 7908776 + ] + }, + "workload": { + "dim": 1024, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 571235301484887008, + "initialization": { + "percentiles": { + "mean": 7326393.8, + "median": 7293668.5, + "minimum": 7176322, + "p90": 7485590, + "p99": 7580349 + }, + "samples": [ + 7176322, + 7219695, + 7225947, + 7235262, + 7235270, + 7243231, + 7256728, + 7263396, + 7265813, + 7269106, + 7271179, + 7276415, + 7286493, + 7291970, + 7293155, + 7294182, + 7304200, + 7306860, + 7326126, + 7332088, + 7334595, + 7342484, + 7343923, + 7423923, + 7428231, + 7448687, + 7478711, + 7485590, + 7551883, + 7580349 + ] + }, + "lloyds": { + "percentiles": { + "mean": 16668619.166666666, + "median": 16666081.5, + "minimum": 16586869, + "p90": 16728774, + "p99": 16804942 + }, + "samples": [ + 16586869, + 16593323, + 16628608, + 16630320, + 16636152, + 16637893, + 16639131, + 16639207, + 16641850, + 16648074, + 16650442, + 16651373, + 16653559, + 16658798, + 16663668, + 16668495, + 16669581, + 16673570, + 16679692, + 16683027, + 16685549, + 16685646, + 16688670, + 16692085, + 16701854, + 16702197, + 16706041, + 16728774, + 16729185, + 16804942 + ] + }, + "objective": 50767967.460754395, + "total": { + "percentiles": { + "mean": 23995014.166666668, + "median": 23970544.5, + "minimum": 23818173, + "p90": 24134334, + "p99": 24254082 + }, + "samples": [ + 23818173, + 23862431, + 23863879, + 23865080, + 23878494, + 23901290, + 23921465, + 23928781, + 23929482, + 23937867, + 23940761, + 23945398, + 23959443, + 23964037, + 23968241, + 23972848, + 23974201, + 23977772, + 23998946, + 24000224, + 24012419, + 24071670, + 24074367, + 24096913, + 24115912, + 24117919, + 24130087, + 24134334, + 24233909, + 24254082 + ] + }, + "workload": { + "dim": 3072, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + } + ] + } + }, + { + "input": { + "content": { + "center_counts": [ + 256 + ], + "dimensions": [ + 4 + ], + "implementation": "quantization", + "max_iterations": 10, + "measurements": 30, + "num_points": 50000, + "phase": "all", + "seed": 42, + "thread_counts": [ + 2, + 4, + 8 + ] + }, + "type": "kmeans-comparison" + }, + "results": { + "implementation": "quantization", + "phase": "all", + "workloads": [ + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 13173.933333333332, + "median": 13002.5, + "minimum": 12725, + "p90": 13784, + "p99": 14859 + }, + "samples": [ + 12725, + 12762, + 12767, + 12767, + 12787, + 12803, + 12830, + 12845, + 12868, + 12878, + 12910, + 12938, + 12939, + 12951, + 12992, + 13013, + 13064, + 13074, + 13102, + 13110, + 13161, + 13206, + 13554, + 13647, + 13650, + 13653, + 13781, + 13784, + 13798, + 14859 + ] + }, + "lloyds": { + "percentiles": { + "mean": 26976.166666666668, + "median": 27000.5, + "minimum": 26495, + "p90": 27500, + "p99": 27679 + }, + "samples": [ + 26495, + 26503, + 26530, + 26543, + 26548, + 26556, + 26676, + 26695, + 26722, + 26722, + 26751, + 26825, + 26853, + 26884, + 26959, + 27042, + 27048, + 27049, + 27063, + 27119, + 27123, + 27176, + 27271, + 27282, + 27324, + 27367, + 27386, + 27500, + 27594, + 27679 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 40151.03333333333, + "median": 40026.5, + "minimum": 39270, + "p90": 40933, + "p99": 41415 + }, + "samples": [ + 39270, + 39427, + 39496, + 39499, + 39615, + 39691, + 39825, + 39833, + 39851, + 39867, + 39891, + 39949, + 39958, + 39959, + 39966, + 40087, + 40102, + 40124, + 40294, + 40318, + 40439, + 40446, + 40463, + 40610, + 40615, + 40823, + 40824, + 40933, + 40941, + 41415 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 13394.3, + "median": 13162.0, + "minimum": 12717, + "p90": 14583, + "p99": 15273 + }, + "samples": [ + 12717, + 12769, + 12818, + 12828, + 12857, + 12875, + 12884, + 12920, + 12927, + 12999, + 13013, + 13070, + 13072, + 13088, + 13116, + 13208, + 13290, + 13291, + 13312, + 13314, + 13343, + 13359, + 13391, + 13422, + 14438, + 14488, + 14508, + 14583, + 14656, + 15273 + ] + }, + "lloyds": { + "percentiles": { + "mean": 27199.3, + "median": 26916.5, + "minimum": 26507, + "p90": 28484, + "p99": 29977 + }, + "samples": [ + 26507, + 26550, + 26557, + 26581, + 26613, + 26665, + 26697, + 26698, + 26767, + 26786, + 26789, + 26803, + 26894, + 26900, + 26916, + 26917, + 26997, + 27029, + 27098, + 27163, + 27169, + 27228, + 27286, + 27313, + 27571, + 27792, + 27972, + 28484, + 29260, + 29977 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 40594.53333333333, + "median": 40432.0, + "minimum": 39327, + "p90": 42089, + "p99": 42977 + }, + "samples": [ + 39327, + 39434, + 39457, + 39579, + 39626, + 39710, + 39715, + 39724, + 39754, + 39917, + 40103, + 40110, + 40207, + 40285, + 40344, + 40520, + 40554, + 40600, + 40829, + 40994, + 41122, + 41152, + 41225, + 41373, + 41406, + 41557, + 41558, + 42089, + 42588, + 42977 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 13723.7, + "median": 13419.5, + "minimum": 12605, + "p90": 14892, + "p99": 16436 + }, + "samples": [ + 12605, + 12606, + 12632, + 12717, + 12722, + 12726, + 12766, + 12768, + 12778, + 12795, + 12898, + 13028, + 13062, + 13094, + 13258, + 13581, + 13699, + 14039, + 14049, + 14213, + 14348, + 14408, + 14461, + 14471, + 14689, + 14775, + 14878, + 14892, + 16317, + 16436 + ] + }, + "lloyds": { + "percentiles": { + "mean": 27505.833333333332, + "median": 26468.0, + "minimum": 25928, + "p90": 29921, + "p99": 30402 + }, + "samples": [ + 25928, + 26001, + 26014, + 26020, + 26020, + 26023, + 26055, + 26064, + 26067, + 26072, + 26110, + 26130, + 26369, + 26407, + 26462, + 26474, + 27113, + 27613, + 28552, + 28627, + 28684, + 28782, + 28991, + 29190, + 29237, + 29601, + 29904, + 29921, + 30342, + 30402 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 41230.53333333333, + "median": 39963.0, + "minimum": 38535, + "p90": 44751, + "p99": 46779 + }, + "samples": [ + 38535, + 38619, + 38749, + 38763, + 38787, + 38788, + 39009, + 39129, + 39176, + 39258, + 39464, + 39583, + 39719, + 39734, + 39831, + 40095, + 40392, + 40481, + 42832, + 42933, + 43243, + 43403, + 43709, + 43884, + 44063, + 44070, + 44135, + 44751, + 45002, + 46779 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + } + ] + } + } +] \ No newline at end of file From c9f84023689288e0a6d009b3cd9304d2a1e56797 Mon Sep 17 00:00:00 2001 From: Wei Wu Date: Thu, 30 Jul 2026 19:47:11 +0800 Subject: [PATCH 2/3] Add labeled K-means comparison charts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9e62a713-1904-4c00-92f2-c81851395ecf --- .../dimension-comparison.svg | 67 ++++++++++++++++++ .../thread-comparison.svg | 68 +++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 issue939-benchmark-results/dimension-comparison.svg create mode 100644 issue939-benchmark-results/thread-comparison.svg diff --git a/issue939-benchmark-results/dimension-comparison.svg b/issue939-benchmark-results/dimension-comparison.svg new file mode 100644 index 000000000..d670baee8 --- /dev/null +++ b/issue939-benchmark-results/dimension-comparison.svg @@ -0,0 +1,67 @@ + + Total K-means latency by dimension + Disk and quantization median total latency for 50000 points, 256 centers, 10 iterations, and one thread. + + Total K-means latency by dimension (1 thread) + + + + + + + + + + + + + + + 0 + 5 + 10 + 15 + 20 + 25 + + + + 4 + 32 + 128 + 384 + 768 + 1024 + 3072 + + + Dimensions + Median latency (seconds) + + + + + + Disk + + + Quantization + + + + + + + + + + + + + + + + 13.70 s + 23.97 s + + diff --git a/issue939-benchmark-results/thread-comparison.svg b/issue939-benchmark-results/thread-comparison.svg new file mode 100644 index 000000000..050f55dda --- /dev/null +++ b/issue939-benchmark-results/thread-comparison.svg @@ -0,0 +1,68 @@ + + Total K-means latency by configured thread count + Disk and quantization median total latency for 50000 four-dimensional points, 256 centers, and 10 iterations. + + Total K-means latency by configured threads (dimension 4) + + + + + + + + + + + + + + + 0 + 50 + 100 + 150 + 200 + 250 + + + + 1 + 2 + 4 + 8 + + + Configured threads + Median latency (ms) + + + + + + Disk + + + Quantization + + + + + + + + + + + + + + 233.9 + 185.5 + 185.5 + 252.6 + 39.3 + 40.0 + 40.4 + 40.0 + + From 24f981a53b00361b7f2dbe4e48fcbaec03b0c9fa Mon Sep 17 00:00:00 2001 From: Wei Wu Date: Thu, 30 Jul 2026 20:06:03 +0800 Subject: [PATCH 3/3] Expand K-means thread and phase comparisons Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9e62a713-1904-4c00-92f2-c81851395ecf --- diskann-benchmark/Cargo.toml | 2 +- .../example/issue-939-kmeans-disk.json | 2 +- .../issue-939-kmeans-quantization.json | 2 +- diskann-benchmark/src/kmeans/mod.rs | 2 +- issue939-benchmark-results/disk-baseline.json | 1440 +++++++++++++---- .../kmeans-baseline-results.md | 32 +- .../quantization-baseline.json | 1434 ++++++++++++---- .../stage-comparison.svg | 30 + .../thread-comparison.svg | 85 +- 9 files changed, 2322 insertions(+), 707 deletions(-) create mode 100644 issue939-benchmark-results/stage-comparison.svg diff --git a/diskann-benchmark/Cargo.toml b/diskann-benchmark/Cargo.toml index 564b197ef..919305a48 100644 --- a/diskann-benchmark/Cargo.toml +++ b/diskann-benchmark/Cargo.toml @@ -83,5 +83,5 @@ disk-index = [ "dep:scopeguard", ] -# Compare the legacy disk and quantization K-means implementations. +# Compare the Disk and quantization K-means implementations. kmeans-comparison = ["dep:diskann-disk"] diff --git a/diskann-benchmark/example/issue-939-kmeans-disk.json b/diskann-benchmark/example/issue-939-kmeans-disk.json index 95ee0028a..aa1b8b1cf 100644 --- a/diskann-benchmark/example/issue-939-kmeans-disk.json +++ b/diskann-benchmark/example/issue-939-kmeans-disk.json @@ -21,7 +21,7 @@ "implementation": "disk", "phase": "all", "num_points": 50000, - "dimensions": [4], + "dimensions": [4, 32, 128], "center_counts": [256], "max_iterations": 10, "thread_counts": [2, 4, 8], diff --git a/diskann-benchmark/example/issue-939-kmeans-quantization.json b/diskann-benchmark/example/issue-939-kmeans-quantization.json index d8f23c24f..f741a1514 100644 --- a/diskann-benchmark/example/issue-939-kmeans-quantization.json +++ b/diskann-benchmark/example/issue-939-kmeans-quantization.json @@ -21,7 +21,7 @@ "implementation": "quantization", "phase": "all", "num_points": 50000, - "dimensions": [4], + "dimensions": [4, 32, 128], "center_counts": [256], "max_iterations": 10, "thread_counts": [2, 4, 8], diff --git a/diskann-benchmark/src/kmeans/mod.rs b/diskann-benchmark/src/kmeans/mod.rs index 7c3d60543..bf672318e 100644 --- a/diskann-benchmark/src/kmeans/mod.rs +++ b/diskann-benchmark/src/kmeans/mod.rs @@ -17,7 +17,7 @@ cfg_if::cfg_if! { registry.register_partially_gated::( "kmeans-comparison", diskann_benchmark_runner::Features::new("kmeans-comparison"), - "Legacy disk and quantization K-means comparison", + "Disk and quantization K-means comparison", )?; Ok(()) } diff --git a/issue939-benchmark-results/disk-baseline.json b/issue939-benchmark-results/disk-baseline.json index 3c456c726..8c50ecf17 100644 --- a/issue939-benchmark-results/disk-baseline.json +++ b/issue939-benchmark-results/disk-baseline.json @@ -978,7 +978,9 @@ 256 ], "dimensions": [ - 4 + 4, + 32, + 128 ], "implementation": "disk", "max_iterations": 10, @@ -1002,126 +1004,126 @@ "center_hash": 12536658597732962046, "initialization": { "percentiles": { - "mean": 57078.566666666666, - "median": 56398.5, - "minimum": 55452, - "p90": 61358, - "p99": 62879 + "mean": 103743.56666666667, + "median": 93420.0, + "minimum": 77049, + "p90": 157419, + "p99": 184676 }, "samples": [ - 55452, - 55708, - 55798, - 55934, - 55966, - 56022, - 56130, - 56184, - 56236, - 56262, - 56313, - 56319, - 56341, - 56378, - 56381, - 56416, - 56424, - 56510, - 56800, - 56804, - 56955, - 57083, - 57290, - 57484, - 57511, - 57566, - 57676, - 61358, - 62177, - 62879 + 77049, + 77749, + 80164, + 80323, + 80357, + 82078, + 82492, + 82955, + 83714, + 88226, + 89391, + 89817, + 90248, + 90379, + 92978, + 93862, + 94125, + 95850, + 99842, + 101313, + 105170, + 111335, + 112334, + 119523, + 132688, + 134830, + 141730, + 157419, + 159690, + 184676 ] }, "lloyds": { "percentiles": { - "mean": 130268.4, - "median": 128980.5, - "minimum": 126875, - "p90": 135876, - "p99": 136366 + "mean": 230422.73333333334, + "median": 191393.0, + "minimum": 164178, + "p90": 455847, + "p99": 471622 }, "samples": [ - 126875, - 127341, - 127486, - 127649, - 127852, - 127890, - 127896, - 128247, - 128449, - 128466, - 128566, - 128583, - 128713, - 128767, - 128896, - 129065, - 129334, - 129478, - 129751, - 129885, - 130110, - 131446, - 132266, - 132638, - 132929, - 135262, - 135778, - 135876, - 136192, - 136366 + 164178, + 172693, + 174131, + 178912, + 179808, + 180217, + 181655, + 182001, + 182146, + 182555, + 184364, + 186070, + 187101, + 191088, + 191333, + 191453, + 198027, + 198910, + 207170, + 207464, + 207932, + 213648, + 222290, + 225560, + 229524, + 349625, + 355956, + 455847, + 459402, + 471622 ] }, "objective": 4127.379323303059, "total": { "percentiles": { - "mean": 187347.86666666667, - "median": 185514.0, - "minimum": 183578, - "p90": 192289, - "p99": 193625 + "mean": 334167.4666666667, + "median": 291924.5, + "minimum": 256210, + "p90": 594233, + "p99": 640524 }, "samples": [ - 183578, - 183671, - 183958, - 184159, - 184269, - 184532, - 184605, - 184844, - 184848, - 184897, - 185250, - 185326, - 185364, - 185413, - 185490, - 185538, - 185572, - 186301, - 186819, - 188661, - 189249, - 189833, - 191469, - 191819, - 191991, - 192218, - 192218, - 192289, - 192630, - 193625 + 256210, + 256661, + 259605, + 261820, + 262359, + 263073, + 265492, + 269026, + 270373, + 273671, + 274803, + 278491, + 279234, + 280725, + 291297, + 292552, + 303466, + 312017, + 312342, + 312539, + 318538, + 318801, + 327457, + 330716, + 346793, + 445476, + 513377, + 594233, + 613353, + 640524 ] }, "workload": { @@ -1132,130 +1134,398 @@ "threads": 2 } }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 117536.96666666666, + "median": 94530.0, + "minimum": 87947, + "p90": 120104, + "p99": 415949 + }, + "samples": [ + 87947, + 89027, + 89138, + 89249, + 89376, + 89418, + 89970, + 90039, + 91150, + 91645, + 91674, + 92369, + 92425, + 92519, + 93311, + 95749, + 96945, + 97170, + 99643, + 101829, + 104756, + 105090, + 105792, + 106085, + 106497, + 106679, + 110842, + 120104, + 393722, + 415949 + ] + }, + "lloyds": { + "percentiles": { + "mean": 312395.3, + "median": 286001.5, + "minimum": 261637, + "p90": 464581, + "p99": 562121 + }, + "samples": [ + 261637, + 262575, + 265617, + 266367, + 267494, + 268823, + 269759, + 270285, + 272186, + 275912, + 276004, + 277943, + 283104, + 283697, + 284921, + 287082, + 287422, + 287505, + 291142, + 291278, + 291294, + 291680, + 296122, + 304908, + 333473, + 355876, + 406818, + 464581, + 534233, + 562121 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 429933.5333333333, + "median": 380600.0, + "minimum": 354250, + "p90": 668801, + "p99": 880531 + }, + "samples": [ + 354250, + 358200, + 359731, + 364666, + 364707, + 367430, + 371458, + 372115, + 373144, + 374848, + 377587, + 377833, + 379090, + 379151, + 380528, + 380672, + 380713, + 380735, + 381128, + 381999, + 384051, + 385150, + 391839, + 401854, + 448302, + 453578, + 623373, + 668801, + 800542, + 880531 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 198453.73333333334, + "median": 193639.5, + "minimum": 166190, + "p90": 212224, + "p99": 343011 + }, + "samples": [ + 166190, + 175871, + 178637, + 180068, + 183414, + 183840, + 184568, + 185398, + 185435, + 186784, + 187836, + 189358, + 190229, + 192472, + 193582, + 193697, + 195452, + 195452, + 196820, + 199012, + 200194, + 203115, + 203480, + 206293, + 206644, + 209615, + 209838, + 212224, + 215083, + 343011 + ] + }, + "lloyds": { + "percentiles": { + "mean": 612818.2333333333, + "median": 586743.0, + "minimum": 568229, + "p90": 830296, + "p99": 876639 + }, + "samples": [ + 568229, + 570557, + 571243, + 573524, + 573670, + 578173, + 579619, + 580868, + 580881, + 582130, + 582594, + 583074, + 583597, + 585135, + 586454, + 587032, + 587321, + 588920, + 590286, + 592542, + 593318, + 595191, + 595958, + 596349, + 596685, + 601255, + 626081, + 830296, + 846926, + 876639 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 811273.2, + "median": 777325.0, + "minimum": 739861, + "p90": 1033777, + "p99": 1189938 + }, + "samples": [ + 739861, + 746429, + 756678, + 760950, + 765050, + 766267, + 766915, + 767012, + 767223, + 767456, + 767533, + 771180, + 772361, + 773240, + 774855, + 779795, + 784148, + 785710, + 788771, + 790645, + 793677, + 794838, + 796544, + 801145, + 805798, + 806301, + 832375, + 1033777, + 1091724, + 1189938 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, { "center_hash": 12536658597732962046, "initialization": { "percentiles": { - "mean": 55639.96666666667, - "median": 54880.5, - "minimum": 52849, - "p90": 59658, - "p99": 64709 + "mean": 109540.23333333334, + "median": 91437.5, + "minimum": 82366, + "p90": 160856, + "p99": 349553 }, "samples": [ - 52849, - 52957, - 53152, - 53182, - 53196, - 53338, - 53690, - 53848, - 54048, - 54238, - 54289, - 54430, - 54510, - 54610, - 54832, - 54929, - 55346, - 55423, - 55445, - 56125, - 56152, - 56163, - 56504, - 56713, - 57107, - 57917, - 58976, - 59658, - 60863, - 64709 + 82366, + 82618, + 82632, + 82871, + 83010, + 83517, + 84574, + 87407, + 87793, + 87859, + 88180, + 89685, + 91041, + 91397, + 91405, + 91470, + 91741, + 92378, + 93172, + 94819, + 95093, + 95673, + 97189, + 99794, + 100013, + 100368, + 116086, + 160856, + 311647, + 349553 ] }, "lloyds": { "percentiles": { - "mean": 130297.63333333333, - "median": 128571.0, - "minimum": 127058, - "p90": 135721, - "p99": 141838 + "mean": 203150.66666666666, + "median": 192178.5, + "minimum": 180613, + "p90": 223034, + "p99": 408678 }, "samples": [ - 127058, - 127114, - 127191, - 127300, - 127492, - 127610, - 127678, - 127789, - 127848, - 128121, - 128219, - 128322, - 128349, - 128369, - 128411, - 128731, - 129269, - 129286, - 129669, - 129670, - 129759, - 130284, - 132960, - 133242, - 133943, - 134343, - 135360, - 135721, - 137983, - 141838 + 180613, + 182012, + 183375, + 183379, + 184137, + 184732, + 185010, + 185562, + 186854, + 188239, + 188858, + 190156, + 191378, + 191404, + 191683, + 192674, + 192961, + 193657, + 195139, + 196993, + 199193, + 199769, + 200856, + 201694, + 205295, + 207912, + 210076, + 223034, + 269197, + 408678 ] }, "objective": 4127.379323303059, "total": { "percentiles": { - "mean": 185938.7, - "median": 185477.5, - "minimum": 180241, - "p90": 193080, - "p99": 196349 + "mean": 312692.13333333336, + "median": 284324.5, + "minimum": 266770, + "p90": 383892, + "p99": 758232 }, "samples": [ - 180241, - 180344, - 181301, - 181608, - 181661, - 182109, - 182360, - 182519, - 183148, - 183213, - 183620, - 183809, - 184119, - 184502, - 185116, - 185839, - 185983, - 186199, - 186448, - 186825, - 187152, - 187250, - 188164, - 188954, - 189210, - 191068, - 191861, - 193080, - 194109, - 196349 + 266770, + 268407, + 273065, + 273191, + 273755, + 274781, + 275953, + 276529, + 276604, + 277758, + 279552, + 279711, + 282535, + 283953, + 284072, + 284577, + 287053, + 287223, + 290280, + 291698, + 292703, + 293087, + 296788, + 296959, + 297368, + 309048, + 352715, + 383892, + 512505, + 758232 ] }, "workload": { @@ -1266,130 +1536,398 @@ "threads": 4 } }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 99598.06666666667, + "median": 96983.5, + "minimum": 89280, + "p90": 116472, + "p99": 121812 + }, + "samples": [ + 89280, + 90846, + 90883, + 91202, + 91254, + 91365, + 92030, + 92219, + 94031, + 94855, + 95347, + 95552, + 95646, + 95700, + 95932, + 98035, + 98874, + 100139, + 101515, + 102278, + 102761, + 102989, + 103222, + 103536, + 104395, + 106835, + 109018, + 116472, + 119919, + 121812 + ] + }, + "lloyds": { + "percentiles": { + "mean": 292384.5333333333, + "median": 289568.0, + "minimum": 273737, + "p90": 313800, + "p99": 315629 + }, + "samples": [ + 273737, + 274019, + 275816, + 280297, + 281170, + 281471, + 281980, + 284071, + 285207, + 285226, + 285461, + 285510, + 286549, + 287639, + 288854, + 290282, + 291010, + 292419, + 292974, + 293828, + 296446, + 301403, + 301449, + 302477, + 305991, + 309952, + 311702, + 313800, + 315167, + 315629 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 391983.6, + "median": 390559.0, + "minimum": 369951, + "p90": 408285, + "p99": 424290 + }, + "samples": [ + 369951, + 371500, + 371517, + 372865, + 377238, + 380809, + 382946, + 384340, + 384394, + 384585, + 384675, + 385930, + 386450, + 387505, + 390489, + 390629, + 391616, + 391999, + 393657, + 395406, + 401207, + 401543, + 401983, + 404448, + 406020, + 406558, + 407507, + 408285, + 419166, + 424290 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 144421.76666666666, + "median": 143072.5, + "minimum": 133527, + "p90": 158342, + "p99": 166230 + }, + "samples": [ + 133527, + 134038, + 134686, + 135479, + 136131, + 137234, + 137979, + 138344, + 139064, + 139700, + 140581, + 142725, + 142850, + 142936, + 143067, + 143078, + 143549, + 144574, + 144818, + 146232, + 146498, + 147693, + 147697, + 148662, + 150331, + 153169, + 154434, + 158342, + 159005, + 166230 + ] + }, + "lloyds": { + "percentiles": { + "mean": 597731.3666666667, + "median": 596196.5, + "minimum": 585548, + "p90": 606760, + "p99": 626390 + }, + "samples": [ + 585548, + 587730, + 588673, + 590739, + 591921, + 591984, + 592778, + 592916, + 593346, + 594343, + 595007, + 595498, + 595623, + 595731, + 595843, + 596550, + 597203, + 597652, + 599586, + 600002, + 600701, + 601228, + 601266, + 601269, + 601539, + 602451, + 604519, + 606760, + 607145, + 626390 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 742154.2333333333, + "median": 739211.0, + "minimum": 719587, + "p90": 756658, + "p99": 792621 + }, + "samples": [ + 719587, + 728396, + 728534, + 729219, + 730440, + 733336, + 733408, + 734188, + 734647, + 735424, + 736227, + 736312, + 736371, + 738567, + 739173, + 739249, + 739277, + 742009, + 743780, + 744162, + 744204, + 746882, + 747462, + 749611, + 751963, + 754437, + 755621, + 756658, + 762862, + 792621 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, { "center_hash": 12536658597732962046, "initialization": { "percentiles": { - "mean": 77105.2, - "median": 76757.0, - "minimum": 73008, - "p90": 80970, - "p99": 83776 + "mean": 218459.6, + "median": 216815.0, + "minimum": 183775, + "p90": 246588, + "p99": 267153 }, "samples": [ - 73008, - 73110, - 73274, - 73505, - 73918, - 74326, - 74635, - 74943, - 75222, - 75790, - 75839, - 76298, - 76348, - 76651, - 76678, - 76836, - 76937, - 76938, - 78158, - 78339, - 78415, - 78490, - 79352, - 79760, - 79825, - 80092, - 80272, - 80970, - 81451, - 83776 + 183775, + 190887, + 195568, + 196428, + 197432, + 201789, + 201843, + 202760, + 203073, + 205429, + 207953, + 209798, + 211094, + 211911, + 216270, + 217360, + 219428, + 221523, + 223208, + 223292, + 225672, + 229366, + 231073, + 232242, + 233118, + 236968, + 245522, + 246588, + 265265, + 267153 ] }, "lloyds": { "percentiles": { - "mean": 175581.03333333333, - "median": 175218.5, - "minimum": 169025, - "p90": 179559, - "p99": 189482 + "mean": 431326.7, + "median": 432785.0, + "minimum": 400281, + "p90": 448728, + "p99": 461766 }, "samples": [ - 169025, - 169071, - 170511, - 171260, - 171404, - 172406, - 172651, - 172816, - 173667, - 173667, - 173743, - 173804, - 173962, - 174675, - 174947, - 175490, - 176132, - 176473, - 176869, - 177162, - 177693, - 177973, - 178005, - 178137, - 178879, - 179003, - 179211, - 179559, - 179754, - 189482 + 400281, + 400693, + 410748, + 415670, + 418634, + 419555, + 422313, + 422662, + 423203, + 424421, + 425465, + 429111, + 430273, + 430983, + 432233, + 433337, + 434510, + 434530, + 437768, + 439016, + 439335, + 439626, + 440175, + 441877, + 442642, + 442985, + 443741, + 448728, + 453520, + 461766 ] }, "objective": 4127.379323303059, "total": { "percentiles": { - "mean": 252687.46666666667, - "median": 252584.5, - "minimum": 245324, - "p90": 259107, - "p99": 264118 + "mean": 649787.8, + "median": 646387.5, + "minimum": 616067, + "p90": 676014, + "p99": 701663 }, "samples": [ - 245324, - 245415, - 246323, - 246860, - 247365, - 248198, - 248612, - 248950, - 249980, - 250515, - 250800, - 251627, - 252279, - 252378, - 252385, - 252784, - 252848, - 253630, - 253796, - 254531, - 254637, - 255076, - 255196, - 255763, - 257370, - 257371, - 257734, - 259107, - 259652, - 264118 + 616067, + 616964, + 617112, + 624452, + 625047, + 632185, + 637194, + 639616, + 640170, + 641369, + 642935, + 643198, + 644346, + 645522, + 645805, + 646970, + 648344, + 651538, + 655495, + 657335, + 662414, + 662434, + 665007, + 665352, + 665935, + 666144, + 671244, + 676014, + 685763, + 701663 ] }, "workload": { @@ -1399,8 +1937,276 @@ "num_points": 50000, "threads": 8 } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 253998.66666666666, + "median": 251475.0, + "minimum": 190532, + "p90": 296480, + "p99": 299838 + }, + "samples": [ + 190532, + 206957, + 210416, + 215497, + 221938, + 223092, + 229880, + 238388, + 239019, + 244173, + 245377, + 245970, + 246138, + 246585, + 249547, + 253403, + 254544, + 262054, + 265896, + 267961, + 269255, + 271311, + 280265, + 283744, + 284929, + 287047, + 290514, + 296480, + 299210, + 299838 + ] + }, + "lloyds": { + "percentiles": { + "mean": 584898.8333333334, + "median": 580572.5, + "minimum": 549303, + "p90": 607362, + "p99": 634541 + }, + "samples": [ + 549303, + 552744, + 556848, + 561289, + 564126, + 567256, + 568066, + 570455, + 571935, + 572120, + 573147, + 575130, + 575589, + 576519, + 580242, + 580903, + 583645, + 588722, + 591441, + 599095, + 601832, + 604526, + 604601, + 604742, + 605432, + 605697, + 605991, + 607362, + 613666, + 634541 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 838898.7666666667, + "median": 835157.0, + "minimum": 762468, + "p90": 902503, + "p99": 905830 + }, + "samples": [ + 762468, + 772346, + 787861, + 793549, + 801858, + 802516, + 806400, + 811141, + 812634, + 819762, + 823343, + 826213, + 826465, + 833011, + 833963, + 836351, + 842127, + 845235, + 851329, + 854149, + 856378, + 858077, + 871612, + 874160, + 876618, + 884978, + 889443, + 902503, + 904643, + 905830 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 284052.5333333333, + "median": 278502.0, + "minimum": 228579, + "p90": 334081, + "p99": 399971 + }, + "samples": [ + 228579, + 230762, + 231351, + 235065, + 237048, + 249763, + 255927, + 256635, + 257913, + 259800, + 261440, + 263874, + 264181, + 266459, + 277512, + 279492, + 283973, + 285478, + 288775, + 294872, + 303097, + 305212, + 317479, + 319993, + 328297, + 329322, + 332595, + 334081, + 342630, + 399971 + ] + }, + "lloyds": { + "percentiles": { + "mean": 955415.8, + "median": 953971.5, + "minimum": 854034, + "p90": 1030320, + "p99": 1109617 + }, + "samples": [ + 854034, + 862089, + 870909, + 899627, + 900599, + 901642, + 912977, + 923940, + 924284, + 937307, + 939222, + 947248, + 951517, + 951978, + 952611, + 955332, + 956533, + 957562, + 967244, + 968828, + 968932, + 974416, + 987080, + 995403, + 996334, + 1008330, + 1020536, + 1030320, + 1036023, + 1109617 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 1239469.6666666667, + "median": 1241718.0, + "minimum": 1091083, + "p90": 1358619, + "p99": 1452248 + }, + "samples": [ + 1091083, + 1092852, + 1099489, + 1134694, + 1160400, + 1165517, + 1168905, + 1208154, + 1209763, + 1213246, + 1218697, + 1221744, + 1226742, + 1227039, + 1235952, + 1247484, + 1251888, + 1257301, + 1261746, + 1262794, + 1263192, + 1264593, + 1268545, + 1297466, + 1301327, + 1301424, + 1325810, + 1358619, + 1395376, + 1452248 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } } ] } } -] \ No newline at end of file +] diff --git a/issue939-benchmark-results/kmeans-baseline-results.md b/issue939-benchmark-results/kmeans-baseline-results.md index fd9c4d54c..1c72a1aba 100644 --- a/issue939-benchmark-results/kmeans-baseline-results.md +++ b/issue939-benchmark-results/kmeans-baseline-results.md @@ -14,7 +14,7 @@ Each measurement calls K-means++ once and, for `phase: "all"`, Lloyd once using the selected centers. The benchmark records initialization, Lloyd, and total wall-clock latency from that single execution. Fixture generation and quality calculation are outside the timed region. -The Disk path calls `k_meanspp_selecting_pivots` followed by `run_lloyds`, passing the explicit repository Rayon pool to both functions. The old quantization path calls `kmeans_plusplus_into` followed by `lloyds`; these baseline APIs are sequential, so changing the benchmark thread count does not change their execution. +The Disk path calls `k_meanspp_selecting_pivots` followed by `run_lloyds`, passing the explicit repository Rayon pool to both functions. The quantization path calls `kmeans_plusplus_into` followed by `lloyds`; these APIs are sequential, so changing the benchmark thread count does not change their execution. ## Dimension sweep @@ -34,20 +34,28 @@ Thread count is 1. Latencies are medians in milliseconds. ## Thread sweep -Dimension is 4. Latencies are medians in milliseconds. - -| Threads | Disk init | Quant init | Disk Lloyd | Quant Lloyd | Disk total | Quant total | -|---:|---:|---:|---:|---:|---:|---:| -| 1 | 79.132 | 12.890 | 154.482 | 26.176 | 233.885 | 39.269 | -| 2 | 56.398 | 13.002 | 128.981 | 27.000 | 185.514 | 40.026 | -| 4 | 54.880 | 13.162 | 128.571 | 26.916 | 185.477 | 40.432 | -| 8 | 76.757 | 13.419 | 175.219 | 26.468 | 252.584 | 39.963 | - -**[measured]** Old quantization total latency remains within 3.0% across 1, 2, 4, and 8 configured threads, confirming that this implementation does not use the benchmark's Rayon pool. Disk improves through four threads on this low-dimensional workload and regresses at eight threads. +Dimensions are 4, 32, and 128. Latencies are medians in milliseconds. + +| Dimension | Threads | Disk init | Quant init | Disk Lloyd | Quant Lloyd | Disk total | Quant total | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 4 | 1 | 79.132 | 12.890 | 154.482 | 26.176 | 233.885 | 39.269 | +| 4 | 2 | 93.420 | 14.013 | 191.393 | 28.925 | 291.925 | 43.144 | +| 4 | 4 | 91.438 | 14.634 | 192.179 | 29.078 | 284.325 | 44.184 | +| 4 | 8 | 216.815 | 14.135 | 432.785 | 29.009 | 646.388 | 43.416 | +| 32 | 1 | 93.925 | 36.040 | 238.536 | 147.149 | 332.902 | 183.473 | +| 32 | 2 | 94.530 | 41.224 | 286.002 | 160.463 | 380.600 | 201.898 | +| 32 | 4 | 96.984 | 41.086 | 289.568 | 165.916 | 390.559 | 206.356 | +| 32 | 8 | 251.475 | 41.028 | 580.572 | 158.886 | 835.157 | 201.424 | +| 128 | 1 | 202.912 | 173.231 | 553.041 | 663.220 | 760.494 | 836.838 | +| 128 | 2 | 193.639 | 239.483 | 586.743 | 711.697 | 777.325 | 955.052 | +| 128 | 4 | 143.072 | 250.808 | 596.197 | 713.639 | 739.211 | 965.208 | +| 128 | 8 | 278.502 | 261.207 | 953.971 | 728.626 | 1,241.718 | 986.318 | + +**[measured]** Quantization remains effectively insensitive to the configured thread count at dimensions 4 and 32; dimension 128 varies more but does not scale down with additional threads. Disk reaches its best dimension-128 total at four threads and regresses sharply at eight threads across all three dimensions on this host. ## Quality observations -The final center hash and objective match exactly through dimension 384 and for every dimension-4 thread workload. They differ at dimensions 768, 1,024, and 3,072, so performance comparisons at those dimensions should not be interpreted as bit-identical algorithm executions. +The final center hash and objective match exactly through dimension 384 and for every thread-sweep workload. They differ at dimensions 768, 1,024, and 3,072, so performance comparisons at those dimensions should not be interpreted as bit-identical algorithm executions. ## Raw results diff --git a/issue939-benchmark-results/quantization-baseline.json b/issue939-benchmark-results/quantization-baseline.json index ba78edd60..f866db829 100644 --- a/issue939-benchmark-results/quantization-baseline.json +++ b/issue939-benchmark-results/quantization-baseline.json @@ -978,7 +978,9 @@ 256 ], "dimensions": [ - 4 + 4, + 32, + 128 ], "implementation": "quantization", "max_iterations": 10, @@ -1002,126 +1004,126 @@ "center_hash": 12536658597732962046, "initialization": { "percentiles": { - "mean": 13173.933333333332, - "median": 13002.5, - "minimum": 12725, - "p90": 13784, - "p99": 14859 + "mean": 17185.666666666668, + "median": 14013.0, + "minimum": 13397, + "p90": 19924, + "p99": 68087 }, "samples": [ - 12725, - 12762, - 12767, - 12767, - 12787, - 12803, - 12830, - 12845, - 12868, - 12878, - 12910, - 12938, - 12939, - 12951, - 12992, - 13013, - 13064, - 13074, - 13102, - 13110, - 13161, - 13206, - 13554, - 13647, - 13650, - 13653, - 13781, - 13784, - 13798, - 14859 + 13397, + 13516, + 13534, + 13594, + 13707, + 13722, + 13723, + 13728, + 13797, + 13834, + 13836, + 13879, + 13888, + 13935, + 13954, + 14072, + 14162, + 14201, + 14447, + 14587, + 14851, + 15378, + 18487, + 19297, + 19556, + 19623, + 19785, + 19924, + 23069, + 68087 ] }, "lloyds": { "percentiles": { - "mean": 26976.166666666668, - "median": 27000.5, - "minimum": 26495, - "p90": 27500, - "p99": 27679 + "mean": 37465.433333333334, + "median": 28925.0, + "minimum": 26879, + "p90": 49681, + "p99": 166068 }, "samples": [ - 26495, - 26503, - 26530, - 26543, - 26548, - 26556, - 26676, - 26695, - 26722, - 26722, - 26751, - 26825, - 26853, - 26884, - 26959, - 27042, - 27048, - 27049, - 27063, - 27119, + 26879, + 26915, 27123, 27176, - 27271, - 27282, - 27324, - 27367, - 27386, - 27500, - 27594, - 27679 + 27373, + 27407, + 27495, + 27560, + 27610, + 27879, + 27938, + 27954, + 28495, + 28548, + 28683, + 29167, + 29368, + 29586, + 29831, + 29929, + 31676, + 31959, + 36260, + 42056, + 45224, + 45344, + 47519, + 49681, + 65260, + 166068 ] }, "objective": 4127.379323303059, "total": { "percentiles": { - "mean": 40151.03333333333, - "median": 40026.5, - "minimum": 39270, - "p90": 40933, - "p99": 41415 + "mean": 54652.4, + "median": 43143.5, + "minimum": 40641, + "p90": 69606, + "p99": 179467 }, "samples": [ - 39270, - 39427, - 39496, - 39499, - 39615, - 39691, - 39825, - 39833, - 39851, - 39867, - 39891, - 39949, - 39958, - 39959, - 39966, - 40087, - 40102, - 40124, - 40294, - 40318, - 40439, - 40446, - 40463, - 40610, - 40615, - 40823, - 40824, - 40933, - 40941, - 41415 + 40641, + 40768, + 41013, + 41203, + 41395, + 41503, + 41533, + 41570, + 41677, + 41952, + 42224, + 42484, + 42563, + 42891, + 43121, + 43166, + 44034, + 44781, + 45632, + 46672, + 47167, + 47339, + 56047, + 59793, + 60543, + 67144, + 68294, + 69606, + 133349, + 179467 ] }, "workload": { @@ -1132,130 +1134,398 @@ "threads": 2 } }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 41495.4, + "median": 41224.5, + "minimum": 37700, + "p90": 46087, + "p99": 46960 + }, + "samples": [ + 37700, + 38128, + 38331, + 38417, + 38614, + 38676, + 39201, + 39264, + 40150, + 40170, + 40589, + 40931, + 41016, + 41109, + 41115, + 41334, + 41365, + 41436, + 41579, + 41626, + 42107, + 42238, + 42554, + 43097, + 44498, + 44913, + 45396, + 46087, + 46261, + 46960 + ] + }, + "lloyds": { + "percentiles": { + "mean": 161288.46666666667, + "median": 160462.5, + "minimum": 154615, + "p90": 170258, + "p99": 178819 + }, + "samples": [ + 154615, + 155986, + 156001, + 156057, + 156588, + 156610, + 157120, + 157166, + 157633, + 158327, + 159756, + 159834, + 160130, + 160313, + 160416, + 160509, + 160541, + 160615, + 160921, + 160936, + 161311, + 162343, + 162770, + 163300, + 163830, + 165600, + 169222, + 170258, + 171127, + 178819 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 202784.93333333332, + "median": 201897.5, + "minimum": 193688, + "p90": 211275, + "p99": 220399 + }, + "samples": [ + 193688, + 195797, + 196052, + 196172, + 196242, + 198828, + 199165, + 199259, + 199274, + 199396, + 199440, + 200675, + 200872, + 201204, + 201651, + 202144, + 202256, + 202502, + 203514, + 204136, + 204215, + 204812, + 205424, + 206096, + 207896, + 209227, + 210659, + 211275, + 211278, + 220399 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 242522.76666666666, + "median": 239483.5, + "minimum": 222652, + "p90": 260651, + "p99": 263946 + }, + "samples": [ + 222652, + 227578, + 230223, + 230527, + 231677, + 233676, + 234155, + 234824, + 236595, + 236876, + 238192, + 238392, + 238467, + 238844, + 239300, + 239667, + 241920, + 245330, + 246065, + 246474, + 247164, + 248418, + 250077, + 250728, + 252380, + 253921, + 254804, + 260651, + 262160, + 263946 + ] + }, + "lloyds": { + "percentiles": { + "mean": 713255.9333333333, + "median": 711696.5, + "minimum": 703226, + "p90": 721753, + "p99": 738898 + }, + "samples": [ + 703226, + 703694, + 704042, + 706484, + 706671, + 706799, + 706846, + 707345, + 707951, + 708028, + 708181, + 708491, + 708653, + 709135, + 711522, + 711871, + 711979, + 712429, + 716167, + 716988, + 717146, + 718053, + 718171, + 718312, + 718445, + 718494, + 719390, + 721753, + 732514, + 738898 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 955779.9666666667, + "median": 955051.5, + "minimum": 935082, + "p90": 973082, + "p99": 983915 + }, + "samples": [ + 935082, + 937375, + 938349, + 941502, + 942072, + 945615, + 945632, + 946418, + 946685, + 948668, + 951172, + 952171, + 953865, + 954720, + 954909, + 955194, + 956816, + 956853, + 958455, + 959180, + 961289, + 962103, + 964586, + 964694, + 967339, + 968900, + 969468, + 973082, + 977290, + 983915 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, { "center_hash": 12536658597732962046, "initialization": { "percentiles": { - "mean": 13394.3, - "median": 13162.0, - "minimum": 12717, - "p90": 14583, - "p99": 15273 + "mean": 14830.5, + "median": 14634.0, + "minimum": 13422, + "p90": 16136, + "p99": 19770 }, "samples": [ - 12717, - 12769, - 12818, - 12828, - 12857, - 12875, - 12884, - 12920, - 12927, - 12999, - 13013, - 13070, - 13072, - 13088, - 13116, - 13208, - 13290, - 13291, - 13312, - 13314, - 13343, - 13359, - 13391, 13422, - 14438, - 14488, - 14508, - 14583, - 14656, - 15273 + 13524, + 13698, + 13775, + 13834, + 13924, + 13935, + 13968, + 14121, + 14156, + 14190, + 14272, + 14456, + 14581, + 14613, + 14655, + 14663, + 14767, + 14775, + 14981, + 15064, + 15096, + 15356, + 15526, + 15593, + 15753, + 16103, + 16136, + 16208, + 19770 ] }, "lloyds": { "percentiles": { - "mean": 27199.3, - "median": 26916.5, - "minimum": 26507, - "p90": 28484, - "p99": 29977 + "mean": 30321.133333333335, + "median": 29077.5, + "minimum": 27903, + "p90": 34193, + "p99": 42565 }, "samples": [ - 26507, - 26550, - 26557, - 26581, - 26613, - 26665, - 26697, - 26698, - 26767, - 26786, - 26789, - 26803, - 26894, - 26900, - 26916, - 26917, - 26997, - 27029, - 27098, - 27163, - 27169, - 27228, - 27286, - 27313, - 27571, - 27792, - 27972, - 28484, - 29260, - 29977 + 27903, + 28091, + 28109, + 28213, + 28284, + 28316, + 28323, + 28413, + 28505, + 28592, + 28757, + 28819, + 28841, + 28853, + 28894, + 29261, + 30024, + 30142, + 30308, + 30378, + 30701, + 30792, + 31280, + 31344, + 31746, + 32561, + 32644, + 34193, + 34782, + 42565 ] }, "objective": 4127.379323303059, "total": { "percentiles": { - "mean": 40594.53333333333, - "median": 40432.0, - "minimum": 39327, - "p90": 42089, - "p99": 42977 + "mean": 45152.833333333336, + "median": 44184.5, + "minimum": 42100, + "p90": 50376, + "p99": 58702 }, "samples": [ - 39327, - 39434, - 39457, - 39579, - 39626, - 39710, - 39715, - 39724, - 39754, - 39917, - 40103, - 40110, - 40207, - 40285, - 40344, - 40520, - 40554, - 40600, - 40829, - 40994, - 41122, - 41152, - 41225, - 41373, - 41406, - 41557, - 41558, - 42089, - 42588, - 42977 + 42100, + 42338, + 42405, + 42680, + 42877, + 42914, + 42930, + 43173, + 43310, + 43349, + 43383, + 43447, + 43487, + 43550, + 44143, + 44226, + 44297, + 44346, + 44491, + 44945, + 45043, + 45216, + 45313, + 46068, + 46844, + 48316, + 49914, + 50376, + 50402, + 58702 ] }, "workload": { @@ -1266,130 +1536,398 @@ "threads": 4 } }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 41797.333333333336, + "median": 41086.0, + "minimum": 37551, + "p90": 46474, + "p99": 49521 + }, + "samples": [ + 37551, + 37668, + 38970, + 39283, + 39388, + 39422, + 40022, + 40267, + 40362, + 40630, + 40804, + 40841, + 40865, + 40920, + 41077, + 41095, + 41568, + 41675, + 41854, + 41858, + 41885, + 42359, + 42831, + 43397, + 43668, + 44429, + 44546, + 46474, + 48690, + 49521 + ] + }, + "lloyds": { + "percentiles": { + "mean": 192288.93333333332, + "median": 165915.5, + "minimum": 155857, + "p90": 338068, + "p99": 378457 + }, + "samples": [ + 155857, + 156582, + 157433, + 157676, + 157905, + 158905, + 159975, + 160051, + 160148, + 160667, + 161103, + 162115, + 162680, + 163555, + 165018, + 166813, + 167152, + 168868, + 173437, + 176994, + 179234, + 180229, + 182317, + 182996, + 194447, + 197313, + 335709, + 338068, + 346964, + 378457 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 234087.6, + "median": 206355.5, + "minimum": 195280, + "p90": 384400, + "p99": 419263 + }, + "samples": [ + 195280, + 196822, + 197701, + 198827, + 199771, + 200338, + 200349, + 201129, + 201148, + 202746, + 202962, + 203027, + 203824, + 204151, + 206124, + 206587, + 207892, + 211700, + 215323, + 217836, + 218518, + 222340, + 223899, + 226394, + 241743, + 243970, + 379745, + 384400, + 388819, + 419263 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 252115.93333333332, + "median": 250807.5, + "minimum": 215024, + "p90": 271767, + "p99": 310375 + }, + "samples": [ + 215024, + 225289, + 234875, + 237623, + 238325, + 239624, + 239922, + 240313, + 240314, + 242584, + 244038, + 244040, + 245314, + 247590, + 249988, + 251627, + 251739, + 252526, + 252649, + 253449, + 253569, + 254404, + 256727, + 258998, + 263945, + 269965, + 271256, + 271767, + 305619, + 310375 + ] + }, + "lloyds": { + "percentiles": { + "mean": 720034.1, + "median": 713639.5, + "minimum": 693751, + "p90": 730099, + "p99": 879119 + }, + "samples": [ + 693751, + 698405, + 698903, + 701029, + 701532, + 701604, + 703989, + 704111, + 704333, + 704705, + 706426, + 707538, + 710642, + 711687, + 712878, + 714401, + 714881, + 716589, + 717579, + 719104, + 719222, + 721685, + 722695, + 723206, + 726978, + 729448, + 729579, + 730099, + 774905, + 879119 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 972151.4, + "median": 965208.0, + "minimum": 916055, + "p90": 996945, + "p99": 1184739 + }, + "samples": [ + 916055, + 929402, + 936029, + 938529, + 944648, + 944752, + 947321, + 948744, + 949757, + 954182, + 956516, + 956903, + 958193, + 960633, + 965136, + 965280, + 966537, + 968806, + 969277, + 970022, + 970963, + 972862, + 973489, + 979935, + 981207, + 981525, + 990872, + 996945, + 1085283, + 1184739 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, { "center_hash": 12536658597732962046, "initialization": { "percentiles": { - "mean": 13723.7, - "median": 13419.5, - "minimum": 12605, - "p90": 14892, - "p99": 16436 + "mean": 14667.066666666668, + "median": 14135.0, + "minimum": 13180, + "p90": 17515, + "p99": 19109 }, "samples": [ - 12605, - 12606, - 12632, - 12717, - 12722, - 12726, - 12766, - 12768, - 12778, - 12795, - 12898, - 13028, - 13062, - 13094, - 13258, - 13581, - 13699, - 14039, - 14049, - 14213, - 14348, - 14408, - 14461, - 14471, - 14689, - 14775, - 14878, - 14892, - 16317, - 16436 + 13180, + 13246, + 13360, + 13404, + 13408, + 13470, + 13472, + 13610, + 13641, + 13676, + 13881, + 13897, + 13906, + 14050, + 14121, + 14149, + 14177, + 14287, + 14303, + 14304, + 14640, + 14673, + 14963, + 15279, + 16158, + 16954, + 17224, + 17515, + 17955, + 19109 ] }, "lloyds": { "percentiles": { - "mean": 27505.833333333332, - "median": 26468.0, - "minimum": 25928, - "p90": 29921, - "p99": 30402 + "mean": 30603.733333333334, + "median": 29008.5, + "minimum": 27730, + "p90": 37408, + "p99": 42911 }, "samples": [ - 25928, - 26001, - 26014, - 26020, - 26020, - 26023, - 26055, - 26064, - 26067, - 26072, - 26110, - 26130, - 26369, - 26407, - 26462, - 26474, - 27113, - 27613, - 28552, - 28627, - 28684, - 28782, - 28991, - 29190, - 29237, - 29601, - 29904, - 29921, - 30342, - 30402 + 27730, + 27958, + 28004, + 28041, + 28041, + 28338, + 28378, + 28467, + 28551, + 28707, + 28833, + 28840, + 28869, + 28910, + 28946, + 29071, + 29099, + 29263, + 29354, + 29623, + 29840, + 30449, + 30542, + 30852, + 31753, + 32887, + 35725, + 37408, + 42722, + 42911 ] }, "objective": 4127.379323303059, "total": { "percentiles": { - "mean": 41230.53333333333, - "median": 39963.0, - "minimum": 38535, - "p90": 44751, - "p99": 46779 + "mean": 45271.8, + "median": 43415.5, + "minimum": 41288, + "p90": 56518, + "p99": 58881 }, "samples": [ - 38535, - 38619, - 38749, - 38763, - 38787, - 38788, - 39009, - 39129, - 39176, - 39258, - 39464, - 39583, - 39719, - 39734, - 39831, - 40095, - 40392, - 40481, - 42832, - 42933, - 43243, - 43403, - 43709, - 43884, - 44063, - 44070, - 44135, - 44751, - 45002, - 46779 + 41288, + 41600, + 41781, + 41809, + 41901, + 42181, + 42280, + 42350, + 42374, + 42624, + 42666, + 43031, + 43138, + 43225, + 43321, + 43510, + 43722, + 43928, + 44261, + 44571, + 44692, + 45866, + 46356, + 47027, + 47851, + 48977, + 49336, + 56518, + 57089, + 58881 ] }, "workload": { @@ -1399,8 +1937,276 @@ "num_points": 50000, "threads": 8 } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 41054.2, + "median": 41028.0, + "minimum": 36883, + "p90": 44385, + "p99": 51314 + }, + "samples": [ + 36883, + 37332, + 37927, + 38437, + 38499, + 38503, + 38522, + 38671, + 38715, + 39253, + 39340, + 39495, + 40265, + 40457, + 40758, + 41298, + 41392, + 41502, + 41597, + 41621, + 41786, + 42446, + 42548, + 42947, + 43249, + 43438, + 44025, + 44385, + 45021, + 51314 + ] + }, + "lloyds": { + "percentiles": { + "mean": 160811.73333333334, + "median": 158885.5, + "minimum": 153092, + "p90": 169693, + "p99": 170019 + }, + "samples": [ + 153092, + 154501, + 155113, + 155246, + 155408, + 155664, + 155941, + 156125, + 156874, + 157335, + 157480, + 157533, + 158013, + 158194, + 158444, + 159327, + 159549, + 160550, + 162143, + 163108, + 163182, + 164333, + 164928, + 165414, + 168769, + 168921, + 169692, + 169693, + 169761, + 170019 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 201867.2, + "median": 201424.5, + "minimum": 192434, + "p90": 210779, + "p99": 211154 + }, + "samples": [ + 192434, + 193458, + 193614, + 193847, + 194741, + 194918, + 196517, + 197477, + 197850, + 198903, + 199155, + 199884, + 200327, + 200428, + 201216, + 201633, + 201897, + 202337, + 205194, + 205393, + 205556, + 205817, + 207013, + 207209, + 207440, + 208410, + 210424, + 210779, + 210991, + 211154 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 299173.63333333336, + "median": 261206.5, + "minimum": 235709, + "p90": 372930, + "p99": 1076387 + }, + "samples": [ + 235709, + 236347, + 236625, + 239721, + 246784, + 247345, + 247517, + 249082, + 252236, + 252437, + 252816, + 252827, + 253825, + 257898, + 258788, + 263625, + 267377, + 267604, + 267690, + 268953, + 269407, + 276253, + 283904, + 299980, + 301729, + 310009, + 347251, + 372930, + 382153, + 1076387 + ] + }, + "lloyds": { + "percentiles": { + "mean": 766883.0666666667, + "median": 728626.5, + "minimum": 705553, + "p90": 933792, + "p99": 999121 + }, + "samples": [ + 705553, + 707225, + 708154, + 711401, + 714584, + 714608, + 716129, + 717032, + 718768, + 721786, + 722170, + 722798, + 723455, + 725608, + 728529, + 728724, + 732473, + 732483, + 733897, + 737420, + 755079, + 761810, + 771230, + 779823, + 854825, + 864363, + 894288, + 933792, + 969364, + 999121 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 1066058.1333333333, + "median": 986318.5, + "minimum": 942180, + "p90": 1223191, + "p99": 1847618 + }, + "samples": [ + 942180, + 956754, + 957880, + 959146, + 962914, + 967436, + 973126, + 973374, + 975875, + 976180, + 978779, + 982394, + 984409, + 985300, + 986134, + 986503, + 989478, + 998132, + 1016379, + 1033466, + 1063541, + 1079804, + 1131969, + 1137233, + 1146727, + 1191691, + 1202078, + 1223191, + 1372053, + 1847618 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } } ] } } -] \ No newline at end of file +] diff --git a/issue939-benchmark-results/stage-comparison.svg b/issue939-benchmark-results/stage-comparison.svg new file mode 100644 index 000000000..c2cad7d19 --- /dev/null +++ b/issue939-benchmark-results/stage-comparison.svg @@ -0,0 +1,30 @@ + + K-means phase latency by dimension + Disk and quantization median initialization and Lloyd latency for one thread. + + K-means phase latency by dimension (1 thread) + + Disk + Quantization + + + + K-means++ initialization + + 048 s + + + + Lloyd iterations + + 0918 s + + + + + + + 43212838476810243072 + Dimensions + + diff --git a/issue939-benchmark-results/thread-comparison.svg b/issue939-benchmark-results/thread-comparison.svg index 050f55dda..803be728c 100644 --- a/issue939-benchmark-results/thread-comparison.svg +++ b/issue939-benchmark-results/thread-comparison.svg @@ -1,68 +1,33 @@ - + Total K-means latency by configured thread count - Disk and quantization median total latency for 50000 four-dimensional points, 256 centers, and 10 iterations. - - Total K-means latency by configured threads (dimension 4) - - - - - - - - - - - - - - - 0 - 50 - 100 - 150 - 200 - 250 - - - - 1 - 2 - 4 - 8 - - - Configured threads - Median latency (ms) - - + Disk and quantization median total latency for dimensions 4, 32, and 128. + + Total K-means latency by configured threads - - - Disk - - - Quantization + Disk + Quantization - - + + Dimension 4 + + 0350700 ms + - - - - - - + Dimension 32 + + 0450900 ms + + + Dimension 128 + + 06501300 ms + + + + - - 233.9 - 185.5 - 185.5 - 252.6 - 39.3 - 40.0 - 40.4 - 40.0 + 1248 + Configured threads