diff --git a/diskann-benchmark/Cargo.toml b/diskann-benchmark/Cargo.toml index 31739244c..919305a48 100644 --- a/diskann-benchmark/Cargo.toml +++ b/diskann-benchmark/Cargo.toml @@ -82,3 +82,6 @@ disk-index = [ "dep:opentelemetry_sdk", "dep:scopeguard", ] + +# Compare the Disk and quantization K-means implementations. +kmeans-comparison = ["dep:diskann-disk"] diff --git a/diskann-benchmark/example/issue-939-kmeans-disk.json b/diskann-benchmark/example/issue-939-kmeans-disk.json new file mode 100644 index 000000000..aa1b8b1cf --- /dev/null +++ b/diskann-benchmark/example/issue-939-kmeans-disk.json @@ -0,0 +1,33 @@ +{ + "search_directories": [], + "jobs": [ + { + "type": "kmeans-comparison", + "content": { + "implementation": "disk", + "phase": "all", + "num_points": 50000, + "dimensions": [4, 32, 128, 384, 768, 1024, 3072], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [1], + "measurements": 30, + "seed": 42 + } + }, + { + "type": "kmeans-comparison", + "content": { + "implementation": "disk", + "phase": "all", + "num_points": 50000, + "dimensions": [4, 32, 128], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [2, 4, 8], + "measurements": 30, + "seed": 42 + } + } + ] +} diff --git a/diskann-benchmark/example/issue-939-kmeans-init.json b/diskann-benchmark/example/issue-939-kmeans-init.json new file mode 100644 index 000000000..95387e815 --- /dev/null +++ b/diskann-benchmark/example/issue-939-kmeans-init.json @@ -0,0 +1,33 @@ +{ + "search_directories": [], + "jobs": [ + { + "type": "kmeans-comparison", + "content": { + "implementation": "disk", + "phase": "init", + "num_points": 50000, + "dimensions": [4, 32, 128, 384, 768, 1024, 3072], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [1], + "measurements": 30, + "seed": 42 + } + }, + { + "type": "kmeans-comparison", + "content": { + "implementation": "quantization", + "phase": "init", + "num_points": 50000, + "dimensions": [4, 32, 128, 384, 768, 1024, 3072], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [1], + "measurements": 30, + "seed": 42 + } + } + ] +} diff --git a/diskann-benchmark/example/issue-939-kmeans-quantization.json b/diskann-benchmark/example/issue-939-kmeans-quantization.json new file mode 100644 index 000000000..f741a1514 --- /dev/null +++ b/diskann-benchmark/example/issue-939-kmeans-quantization.json @@ -0,0 +1,33 @@ +{ + "search_directories": [], + "jobs": [ + { + "type": "kmeans-comparison", + "content": { + "implementation": "quantization", + "phase": "all", + "num_points": 50000, + "dimensions": [4, 32, 128, 384, 768, 1024, 3072], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [1], + "measurements": 30, + "seed": 42 + } + }, + { + "type": "kmeans-comparison", + "content": { + "implementation": "quantization", + "phase": "all", + "num_points": 50000, + "dimensions": [4, 32, 128], + "center_counts": [256], + "max_iterations": 10, + "thread_counts": [2, 4, 8], + "measurements": 30, + "seed": 42 + } + } + ] +} diff --git a/diskann-benchmark/src/inputs/kmeans.rs b/diskann-benchmark/src/inputs/kmeans.rs new file mode 100644 index 000000000..79a40a2d8 --- /dev/null +++ b/diskann-benchmark/src/inputs/kmeans.rs @@ -0,0 +1,114 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +use std::num::NonZeroUsize; + +use diskann_benchmark_runner::Checker; +use serde::{Deserialize, Serialize}; + +use super::{as_input, Example}; + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum KmeansImplementation { + Disk, + Quantization, +} + +impl std::fmt::Display for KmeansImplementation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Disk => write!(f, "disk"), + Self::Quantization => write!(f, "quantization"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum KmeansPhase { + All, + Init, +} + +impl std::fmt::Display for KmeansPhase { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::All => write!(f, "all"), + Self::Init => write!(f, "init"), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct KmeansComparison { + pub(crate) implementation: KmeansImplementation, + pub(crate) phase: KmeansPhase, + pub(crate) num_points: NonZeroUsize, + pub(crate) dimensions: Vec, + pub(crate) center_counts: Vec, + pub(crate) max_iterations: NonZeroUsize, + pub(crate) thread_counts: Vec, + pub(crate) measurements: NonZeroUsize, + pub(crate) seed: u64, +} + +impl KmeansComparison { + pub(crate) const fn tag() -> &'static str { + "kmeans-comparison" + } + + pub(crate) fn validate(&mut self, _checker: &mut Checker) -> anyhow::Result<()> { + anyhow::ensure!(!self.dimensions.is_empty(), "dimensions cannot be empty"); + anyhow::ensure!( + !self.center_counts.is_empty(), + "center_counts cannot be empty" + ); + anyhow::ensure!( + !self.thread_counts.is_empty(), + "thread_counts cannot be empty" + ); + anyhow::ensure!( + self.center_counts + .iter() + .all(|count| count.get() <= self.num_points.get()), + "center counts cannot exceed num_points" + ); + Ok(()) + } +} + +impl Example for KmeansComparison { + fn example() -> Self { + Self { + implementation: KmeansImplementation::Quantization, + phase: KmeansPhase::All, + num_points: NonZeroUsize::new(10_000).unwrap(), + dimensions: vec![NonZeroUsize::new(128).unwrap()], + center_counts: vec![NonZeroUsize::new(64).unwrap()], + max_iterations: NonZeroUsize::new(3).unwrap(), + thread_counts: vec![NonZeroUsize::new(1).unwrap()], + measurements: NonZeroUsize::new(10).unwrap(), + seed: 42, + } + } +} + +as_input!(KmeansComparison); + +impl std::fmt::Display for KmeansComparison { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "K-means Implementation Benchmark\n")?; + writeln!(f, "{:>18}: {}", "implementation", self.implementation)?; + writeln!(f, "{:>18}: {}", "phase", self.phase)?; + writeln!(f, "{:>18}: {}", "points", self.num_points)?; + writeln!(f, "{:>18}: {:?}", "dimensions", self.dimensions)?; + writeln!(f, "{:>18}: {:?}", "center counts", self.center_counts)?; + writeln!(f, "{:>18}: {}", "max iterations", self.max_iterations)?; + writeln!(f, "{:>18}: {:?}", "thread counts", self.thread_counts)?; + writeln!(f, "{:>18}: {}", "measurements", self.measurements)?; + writeln!(f, "{:>18}: {}", "seed", self.seed) + } +} diff --git a/diskann-benchmark/src/inputs/mod.rs b/diskann-benchmark/src/inputs/mod.rs index 978d93cb3..d2c87d667 100644 --- a/diskann-benchmark/src/inputs/mod.rs +++ b/diskann-benchmark/src/inputs/mod.rs @@ -8,6 +8,7 @@ pub(crate) mod exhaustive; pub(crate) mod filters; pub(crate) mod flat; pub(crate) mod graph_index; +pub(crate) mod kmeans; pub(crate) mod multi_vector; pub(crate) mod post_processor; pub(crate) mod save_and_load; diff --git a/diskann-benchmark/src/kmeans/comparison.rs b/diskann-benchmark/src/kmeans/comparison.rs new file mode 100644 index 000000000..cc76633be --- /dev/null +++ b/diskann-benchmark/src/kmeans/comparison.rs @@ -0,0 +1,344 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +use std::{ + fmt, + hash::{DefaultHasher, Hash, Hasher}, + io::Write, + time::Instant, +}; + +use diskann_benchmark_runner::{ + benchmark::{MatchContext, Score}, + utils::{percentiles, MicroSeconds}, + Benchmark, Checkpoint, Output, Registry, +}; +use diskann_disk::utils::{k_meanspp_selecting_pivots, run_lloyds}; +use diskann_providers::utils::{create_thread_pool, RayonThreadPool}; +use diskann_quantization::algorithms::kmeans::{lloyds::lloyds, plusplus::kmeans_plusplus_into}; +use diskann_utils::views::Matrix; +use diskann_vector::{distance::SquaredL2, PureDistanceFunction}; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use serde::Serialize; + +use crate::inputs::kmeans::{KmeansComparison, KmeansImplementation, KmeansPhase}; + +pub(super) fn register(registry: &mut Registry) -> anyhow::Result<()> { + registry.register("kmeans-comparison", Comparison)?; + Ok(()) +} + +struct Comparison; + +impl Benchmark for Comparison { + type Input = KmeansComparison; + type Output = ComparisonOutput; + + fn try_match(&self, _input: &Self::Input, context: &MatchContext) -> Score { + context.success(0) + } + + fn description(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "Runs one K-means implementation with phase telemetry") + } + + fn run( + &self, + input: &Self::Input, + _checkpoint: Checkpoint<'_>, + mut output: &mut dyn Output, + ) -> anyhow::Result { + writeln!(output, "{}", input)?; + let mut workloads = Vec::new(); + + for &threads in &input.thread_counts { + let pool = create_thread_pool(threads.get())?; + for &dim in &input.dimensions { + for &num_centers in &input.center_counts { + let workload = Workload { + num_points: input.num_points.get(), + dim: dim.get(), + num_centers: num_centers.get(), + max_iterations: input.max_iterations.get(), + threads: threads.get(), + }; + let result = run_workload( + input.implementation, + input.phase, + workload, + input.measurements.get(), + input.seed, + &pool, + )?; + writeln!(output, "{}", result)?; + workloads.push(result); + } + } + } + + Ok(ComparisonOutput { + implementation: input.implementation, + phase: input.phase, + workloads, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +struct Workload { + num_points: usize, + dim: usize, + num_centers: usize, + max_iterations: usize, + threads: usize, +} + +#[derive(Debug, Serialize)] +struct PhaseMeasurements { + samples: Vec, + percentiles: percentiles::Percentiles, +} + +#[derive(Debug, Serialize)] +struct WorkloadResult { + workload: Workload, + initialization: PhaseMeasurements, + #[serde(skip_serializing_if = "Option::is_none")] + lloyds: Option, + total: PhaseMeasurements, + center_hash: u64, + objective: f64, +} + +#[derive(Debug, Serialize)] +struct ComparisonOutput { + implementation: KmeansImplementation, + phase: KmeansPhase, + workloads: Vec, +} + +impl fmt::Display for WorkloadResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{} points x {} dimensions x {} centers, {} iterations, {} threads: init median {:.3} ms", + self.workload.num_points, + self.workload.dim, + self.workload.num_centers, + self.workload.max_iterations, + self.workload.threads, + self.initialization.percentiles.median / 1_000.0, + )?; + if let Some(lloyds) = &self.lloyds { + write!( + f, + ", Lloyd median {:.3} ms", + lloyds.percentiles.median / 1_000.0 + )?; + } + writeln!( + f, + ", total median {:.3} ms", + self.total.percentiles.median / 1_000.0 + ) + } +} + +#[derive(Debug)] +struct Measurement { + initialization: MicroSeconds, + lloyds: Option, + total: MicroSeconds, + centers: Vec, +} + +fn run_workload( + implementation: KmeansImplementation, + phase: KmeansPhase, + workload: Workload, + num_measurements: usize, + seed: u64, + pool: &RayonThreadPool, +) -> anyhow::Result { + let data_seed = seed + .wrapping_add(workload.dim as u64) + .wrapping_add(workload.num_centers as u64); + let mut rng = StdRng::seed_from_u64(data_seed); + let data: Vec = (0..workload.num_points * workload.dim) + .map(|_| rng.random_range(-1.0..1.0)) + .collect(); + let data_matrix = Matrix::try_from( + data.clone().into_boxed_slice(), + workload.num_points, + workload.dim, + ) + .map_err(|_| anyhow::anyhow!("generated data matrix has an invalid shape"))?; + + let run = || { + run_measurement( + implementation, + phase, + &data, + &data_matrix, + workload, + data_seed, + pool, + ) + }; + + let quality = run()?; + let objective = objective(&data, &quality.centers, workload); + let center_hash = hash_centers(&quality.centers); + + let mut initialization = Vec::with_capacity(num_measurements); + let mut lloyds = (phase == KmeansPhase::All).then(|| Vec::with_capacity(num_measurements)); + let mut total = Vec::with_capacity(num_measurements); + for _ in 0..num_measurements { + let measurement = run()?; + initialization.push(measurement.initialization); + if let (Some(samples), Some(sample)) = (&mut lloyds, measurement.lloyds) { + samples.push(sample); + } + total.push(measurement.total); + } + + Ok(WorkloadResult { + workload, + initialization: make_phase_measurements(initialization)?, + lloyds: lloyds.map(make_phase_measurements).transpose()?, + total: make_phase_measurements(total)?, + center_hash, + objective, + }) +} + +fn run_measurement( + implementation: KmeansImplementation, + phase: KmeansPhase, + data: &[f32], + data_matrix: &Matrix, + workload: Workload, + seed: u64, + pool: &RayonThreadPool, +) -> anyhow::Result { + match implementation { + KmeansImplementation::Disk => run_disk(phase, data, workload, seed, pool), + KmeansImplementation::Quantization => { + run_quantization(phase, data_matrix, workload, seed, pool) + } + } +} + +fn run_disk( + phase: KmeansPhase, + data: &[f32], + workload: Workload, + seed: u64, + pool: &RayonThreadPool, +) -> anyhow::Result { + let mut centers = vec![0.0; workload.num_centers * workload.dim]; + let mut rng = StdRng::seed_from_u64(seed); + let mut canceled = false; + let total_start = Instant::now(); + let initialization_start = Instant::now(); + k_meanspp_selecting_pivots( + data, + workload.num_points, + workload.dim, + &mut centers, + workload.num_centers, + &mut rng, + &mut canceled, + pool.as_ref(), + )?; + let initialization = initialization_start.elapsed().into(); + let lloyds = if phase == KmeansPhase::All { + let start = Instant::now(); + run_lloyds( + data, + workload.num_points, + workload.dim, + &mut centers, + workload.num_centers, + workload.max_iterations, + &mut canceled, + pool.as_ref(), + )?; + Some(start.elapsed().into()) + } else { + None + }; + Ok(Measurement { + initialization, + lloyds, + total: total_start.elapsed().into(), + centers, + }) +} + +fn run_quantization( + phase: KmeansPhase, + data: &Matrix, + workload: Workload, + seed: u64, + pool: &RayonThreadPool, +) -> anyhow::Result { + let mut centers = Matrix::new(0.0, workload.num_centers, workload.dim); + let mut rng = StdRng::seed_from_u64(seed); + let total_start = Instant::now(); + let initialization_start = Instant::now(); + pool.install(|| kmeans_plusplus_into(centers.as_mut_view(), data.as_view(), &mut rng))?; + let initialization = initialization_start.elapsed().into(); + let lloyds = if phase == KmeansPhase::All { + let start = Instant::now(); + pool.install(|| { + lloyds( + data.as_view(), + centers.as_mut_view(), + workload.max_iterations, + ) + }); + Some(start.elapsed().into()) + } else { + None + }; + Ok(Measurement { + initialization, + lloyds, + total: total_start.elapsed().into(), + centers: centers.into_inner().into(), + }) +} + +fn make_phase_measurements(mut samples: Vec) -> anyhow::Result { + let percentiles = percentiles::compute_percentiles(&mut samples)?; + Ok(PhaseMeasurements { + samples, + percentiles, + }) +} + +fn hash_centers(centers: &[f32]) -> u64 { + let mut hasher = DefaultHasher::new(); + centers + .iter() + .for_each(|value| value.to_bits().hash(&mut hasher)); + hasher.finish() +} + +fn objective(data: &[f32], centers: &[f32], workload: Workload) -> f64 { + data.chunks_exact(workload.dim) + .map(|point| { + centers + .chunks_exact(workload.dim) + .map(|center| { + let distance: f32 = SquaredL2::evaluate(point, center); + f64::from(distance) + }) + .min_by(f64::total_cmp) + .expect("the benchmark requires at least one center") + }) + .sum() +} diff --git a/diskann-benchmark/src/kmeans/mod.rs b/diskann-benchmark/src/kmeans/mod.rs new file mode 100644 index 000000000..bf672318e --- /dev/null +++ b/diskann-benchmark/src/kmeans/mod.rs @@ -0,0 +1,25 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +use diskann_benchmark_runner::Registry; + +cfg_if::cfg_if! { + if #[cfg(feature = "kmeans-comparison")] { + mod comparison; + + pub(crate) fn register_benchmarks(registry: &mut Registry) -> anyhow::Result<()> { + comparison::register(registry) + } + } else { + pub(crate) fn register_benchmarks(registry: &mut Registry) -> anyhow::Result<()> { + registry.register_partially_gated::( + "kmeans-comparison", + diskann_benchmark_runner::Features::new("kmeans-comparison"), + "Disk and quantization K-means comparison", + )?; + Ok(()) + } + } +} diff --git a/diskann-benchmark/src/main.rs b/diskann-benchmark/src/main.rs index cf9a0ea52..cc3d46595 100644 --- a/diskann-benchmark/src/main.rs +++ b/diskann-benchmark/src/main.rs @@ -9,6 +9,7 @@ mod filters; mod flat; mod index; mod inputs; +mod kmeans; mod multi_vector; mod utils; @@ -54,6 +55,7 @@ impl Cli { flat::register_benchmarks(&mut registry)?; index::register_benchmarks(&mut registry)?; filters::register_benchmarks(&mut registry)?; + kmeans::register_benchmarks(&mut registry)?; multi_vector::register_benchmarks(&mut registry)?; self.app.run(®istry, output) diff --git a/issue939-benchmark-results/dimension-comparison.svg b/issue939-benchmark-results/dimension-comparison.svg new file mode 100644 index 000000000..d670baee8 --- /dev/null +++ b/issue939-benchmark-results/dimension-comparison.svg @@ -0,0 +1,67 @@ + + Total K-means latency by dimension + Disk and quantization median total latency for 50000 points, 256 centers, 10 iterations, and one thread. + + Total K-means latency by dimension (1 thread) + + + + + + + + + + + + + + + 0 + 5 + 10 + 15 + 20 + 25 + + + + 4 + 32 + 128 + 384 + 768 + 1024 + 3072 + + + Dimensions + Median latency (seconds) + + + + + + Disk + + + Quantization + + + + + + + + + + + + + + + + 13.70 s + 23.97 s + + diff --git a/issue939-benchmark-results/disk-baseline.json b/issue939-benchmark-results/disk-baseline.json new file mode 100644 index 000000000..8c50ecf17 --- /dev/null +++ b/issue939-benchmark-results/disk-baseline.json @@ -0,0 +1,2212 @@ +[ + { + "input": { + "content": { + "center_counts": [ + 256 + ], + "dimensions": [ + 4, + 32, + 128, + 384, + 768, + 1024, + 3072 + ], + "implementation": "disk", + "max_iterations": 10, + "measurements": 30, + "num_points": 50000, + "phase": "all", + "seed": 42, + "thread_counts": [ + 1 + ] + }, + "type": "kmeans-comparison" + }, + "results": { + "implementation": "disk", + "phase": "all", + "workloads": [ + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 79659.63333333333, + "median": 79131.5, + "minimum": 76601, + "p90": 83854, + "p99": 89952 + }, + "samples": [ + 76601, + 76669, + 76876, + 76878, + 77154, + 77233, + 77281, + 77281, + 77424, + 77439, + 77877, + 77889, + 78169, + 78943, + 78988, + 79275, + 79439, + 79708, + 79943, + 80103, + 80129, + 80575, + 80950, + 81294, + 82331, + 82713, + 82724, + 83854, + 84097, + 89952 + ] + }, + "lloyds": { + "percentiles": { + "mean": 155633.43333333332, + "median": 154482.5, + "minimum": 147034, + "p90": 164978, + "p99": 176861 + }, + "samples": [ + 147034, + 148198, + 148353, + 149056, + 150131, + 150475, + 151038, + 151170, + 151256, + 152991, + 152991, + 153086, + 153932, + 153973, + 154467, + 154498, + 154575, + 154862, + 155085, + 155281, + 155500, + 156629, + 157157, + 157255, + 160326, + 161119, + 162648, + 164978, + 174078, + 176861 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 235294.3, + "median": 233885.0, + "minimum": 223911, + "p90": 251072, + "p99": 258156 + }, + "samples": [ + 223911, + 225777, + 226369, + 228452, + 228915, + 229633, + 229752, + 230527, + 230811, + 230882, + 230965, + 231069, + 231858, + 232017, + 233827, + 233943, + 233986, + 234030, + 235226, + 235630, + 236244, + 236305, + 236733, + 237223, + 239766, + 239882, + 249076, + 251072, + 256792, + 258156 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 94114.1, + "median": 93925.0, + "minimum": 92205, + "p90": 95439, + "p99": 100068 + }, + "samples": [ + 92205, + 92535, + 92883, + 92974, + 93035, + 93195, + 93244, + 93283, + 93315, + 93390, + 93566, + 93648, + 93774, + 93820, + 93914, + 93936, + 93977, + 94037, + 94050, + 94141, + 94271, + 94524, + 94548, + 94773, + 94910, + 95088, + 95201, + 95439, + 95679, + 100068 + ] + }, + "lloyds": { + "percentiles": { + "mean": 239327.06666666668, + "median": 238535.5, + "minimum": 225916, + "p90": 250099, + "p99": 257435 + }, + "samples": [ + 225916, + 228693, + 229775, + 230829, + 232016, + 232965, + 232989, + 233461, + 233712, + 234024, + 234312, + 236521, + 237565, + 237705, + 237825, + 239246, + 240559, + 240901, + 241168, + 241537, + 242690, + 242882, + 243543, + 245916, + 245961, + 246515, + 248121, + 250099, + 254931, + 257435 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 333442.26666666666, + "median": 332902.0, + "minimum": 320058, + "p90": 343344, + "p99": 352875 + }, + "samples": [ + 320058, + 322966, + 323091, + 325300, + 325525, + 325603, + 326902, + 327028, + 327103, + 327286, + 328074, + 329718, + 329910, + 330449, + 332737, + 333067, + 335084, + 335648, + 336658, + 336739, + 338068, + 338370, + 339610, + 340466, + 340493, + 340970, + 342159, + 343344, + 347967, + 352875 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 203285.16666666666, + "median": 202912.0, + "minimum": 179379, + "p90": 226077, + "p99": 234061 + }, + "samples": [ + 179379, + 181875, + 182222, + 182507, + 184072, + 185428, + 185530, + 191201, + 193686, + 195786, + 197330, + 197628, + 199238, + 200141, + 200910, + 204914, + 206783, + 207705, + 211204, + 211390, + 211509, + 212208, + 213889, + 215755, + 216304, + 218347, + 224306, + 226077, + 227170, + 234061 + ] + }, + "lloyds": { + "percentiles": { + "mean": 553646.4666666667, + "median": 553041.0, + "minimum": 546613, + "p90": 562758, + "p99": 570063 + }, + "samples": [ + 546613, + 546671, + 546930, + 547273, + 548561, + 548982, + 549055, + 549234, + 549281, + 549377, + 549576, + 550385, + 550627, + 551412, + 552624, + 553458, + 554046, + 554119, + 554334, + 554773, + 555190, + 555281, + 556255, + 558950, + 559000, + 559456, + 560680, + 562758, + 564430, + 570063 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 756932.8666666667, + "median": 760493.5, + "minimum": 731253, + "p90": 779081, + "p99": 781504 + }, + "samples": [ + 731253, + 733128, + 733499, + 734849, + 736554, + 741684, + 747800, + 748289, + 748743, + 749375, + 750977, + 751087, + 754368, + 758479, + 760196, + 760791, + 761192, + 761265, + 762018, + 763467, + 764961, + 765342, + 766141, + 766240, + 766706, + 773008, + 775255, + 779081, + 780734, + 781504 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 6161376117261214841, + "initialization": { + "percentiles": { + "mean": 895038.2666666667, + "median": 887169.5, + "minimum": 867962, + "p90": 918972, + "p99": 1045898 + }, + "samples": [ + 867962, + 869142, + 870156, + 872226, + 872495, + 873322, + 874188, + 875112, + 875414, + 875681, + 878173, + 880257, + 881962, + 884594, + 885229, + 889110, + 889402, + 891125, + 894271, + 895721, + 897904, + 900007, + 905036, + 905405, + 912686, + 913828, + 918017, + 918972, + 937853, + 1045898 + ] + }, + "lloyds": { + "percentiles": { + "mean": 1380407.7, + "median": 1373521.0, + "minimum": 1349407, + "p90": 1410464, + "p99": 1458439 + }, + "samples": [ + 1349407, + 1360723, + 1365408, + 1366037, + 1366503, + 1367177, + 1367498, + 1368092, + 1368622, + 1369004, + 1369197, + 1370242, + 1371530, + 1372906, + 1372946, + 1374096, + 1374719, + 1375040, + 1376888, + 1379467, + 1383556, + 1384121, + 1390363, + 1392746, + 1394160, + 1396833, + 1402641, + 1410464, + 1413406, + 1458439 + ] + }, + "objective": 6189761.912239075, + "total": { + "percentiles": { + "mean": 2275447.3333333335, + "median": 2264663.5, + "minimum": 2223596, + "p90": 2318812, + "p99": 2419996 + }, + "samples": [ + 2223596, + 2235462, + 2236405, + 2237766, + 2240521, + 2244212, + 2244420, + 2245133, + 2245197, + 2252687, + 2252791, + 2258304, + 2259949, + 2260932, + 2264408, + 2264919, + 2269330, + 2274709, + 2282930, + 2284636, + 2289159, + 2290722, + 2294169, + 2295861, + 2301574, + 2310799, + 2316471, + 2318812, + 2347550, + 2419996 + ] + }, + "workload": { + "dim": 384, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 15316337929231845703, + "initialization": { + "percentiles": { + "mean": 1775828.8333333333, + "median": 1771880.5, + "minimum": 1729521, + "p90": 1849517, + "p99": 1908204 + }, + "samples": [ + 1729521, + 1730366, + 1731422, + 1732395, + 1734035, + 1738329, + 1738598, + 1742626, + 1746081, + 1762640, + 1766394, + 1766678, + 1767849, + 1770057, + 1771639, + 1772122, + 1774084, + 1774322, + 1775715, + 1778884, + 1785078, + 1785598, + 1787744, + 1788354, + 1793227, + 1801558, + 1822079, + 1849517, + 1849749, + 1908204 + ] + }, + "lloyds": { + "percentiles": { + "mean": 2634890.8666666667, + "median": 2635229.5, + "minimum": 2577671, + "p90": 2740652, + "p99": 2790580 + }, + "samples": [ + 2577671, + 2580800, + 2581632, + 2587964, + 2588263, + 2588978, + 2589803, + 2592294, + 2594696, + 2601937, + 2604562, + 2611676, + 2624510, + 2632094, + 2632842, + 2637617, + 2637993, + 2640576, + 2640966, + 2642032, + 2643689, + 2645010, + 2649880, + 2649888, + 2650747, + 2656501, + 2674644, + 2740652, + 2756229, + 2790580 + ] + }, + "objective": 12551465.794921875, + "total": { + "percentiles": { + "mean": 4410721.033333333, + "median": 4404248.5, + "minimum": 4310068, + "p90": 4578309, + "p99": 4698785 + }, + "samples": [ + 4310068, + 4319130, + 4320170, + 4320232, + 4321816, + 4322299, + 4331605, + 4333361, + 4334046, + 4370413, + 4387151, + 4389642, + 4390562, + 4402152, + 4404014, + 4404483, + 4412860, + 4417427, + 4418012, + 4423965, + 4426565, + 4428624, + 4428931, + 4435260, + 4437633, + 4476203, + 4487511, + 4578309, + 4590402, + 4698785 + ] + }, + "workload": { + "dim": 768, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 10519214377489719790, + "initialization": { + "percentiles": { + "mean": 2344325.8, + "median": 2341492.0, + "minimum": 2307992, + "p90": 2378460, + "p99": 2400327 + }, + "samples": [ + 2307992, + 2313691, + 2315361, + 2316423, + 2320088, + 2320938, + 2324819, + 2331982, + 2334549, + 2335596, + 2335782, + 2336978, + 2338426, + 2338502, + 2341034, + 2341950, + 2342020, + 2342229, + 2344795, + 2345911, + 2348853, + 2354183, + 2356079, + 2360734, + 2366677, + 2369739, + 2370482, + 2378460, + 2395174, + 2400327 + ] + }, + "lloyds": { + "percentiles": { + "mean": 3311839.8, + "median": 3305319.0, + "minimum": 3279899, + "p90": 3359864, + "p99": 3366693 + }, + "samples": [ + 3279899, + 3283934, + 3287837, + 3288667, + 3290601, + 3293056, + 3293163, + 3293843, + 3296108, + 3296173, + 3301175, + 3302390, + 3304400, + 3304510, + 3304955, + 3305683, + 3307048, + 3307705, + 3309219, + 3313695, + 3314089, + 3321171, + 3324400, + 3325108, + 3325259, + 3332562, + 3357235, + 3359864, + 3364752, + 3366693 + ] + }, + "objective": 16794760.531051636, + "total": { + "percentiles": { + "mean": 5656166.9, + "median": 5645711.5, + "minimum": 5604293, + "p90": 5730348, + "p99": 5765080 + }, + "samples": [ + 5604293, + 5604720, + 5604873, + 5607926, + 5615041, + 5620934, + 5625646, + 5628394, + 5629451, + 5634373, + 5634601, + 5638338, + 5639678, + 5643303, + 5645002, + 5646421, + 5647347, + 5649751, + 5653792, + 5659607, + 5666143, + 5667211, + 5673255, + 5675423, + 5699240, + 5716348, + 5722773, + 5730348, + 5735695, + 5765080 + ] + }, + "workload": { + "dim": 1024, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 12943831542607517319, + "initialization": { + "percentiles": { + "mean": 6971064.566666666, + "median": 6969815.0, + "minimum": 6829692, + "p90": 7097969, + "p99": 7194953 + }, + "samples": [ + 6829692, + 6856927, + 6860097, + 6864891, + 6869914, + 6902165, + 6904875, + 6907069, + 6923809, + 6924648, + 6936090, + 6939331, + 6940478, + 6949638, + 6969646, + 6969984, + 6979760, + 6982273, + 6982858, + 6998557, + 7002886, + 7004236, + 7025491, + 7026380, + 7039349, + 7041093, + 7082055, + 7097969, + 7124823, + 7194953 + ] + }, + "lloyds": { + "percentiles": { + "mean": 6754397.5, + "median": 6726029.5, + "minimum": 6678597, + "p90": 6892652, + "p99": 7029009 + }, + "samples": [ + 6678597, + 6689924, + 6693280, + 6693345, + 6693556, + 6699429, + 6699938, + 6700776, + 6701777, + 6705380, + 6705858, + 6712317, + 6717181, + 6720224, + 6722644, + 6729415, + 6730183, + 6733185, + 6742381, + 6746151, + 6757462, + 6762267, + 6762500, + 6797526, + 6802683, + 6803046, + 6850737, + 6892652, + 6958502, + 7029009 + ] + }, + "objective": 50768784.480651855, + "total": { + "percentiles": { + "mean": 13725463.233333332, + "median": 13696581.0, + "minimum": 13519617, + "p90": 13927507, + "p99": 14223963 + }, + "samples": [ + 13519617, + 13538695, + 13562787, + 13563260, + 13565669, + 13602105, + 13618205, + 13646455, + 13647257, + 13653273, + 13655019, + 13670662, + 13671762, + 13673041, + 13695176, + 13697986, + 13732148, + 13744541, + 13749038, + 13758677, + 13759574, + 13770509, + 13790069, + 13807283, + 13823907, + 13839518, + 13865572, + 13927507, + 13990622, + 14223963 + ] + }, + "workload": { + "dim": 3072, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + } + ] + } + }, + { + "input": { + "content": { + "center_counts": [ + 256 + ], + "dimensions": [ + 4, + 32, + 128 + ], + "implementation": "disk", + "max_iterations": 10, + "measurements": 30, + "num_points": 50000, + "phase": "all", + "seed": 42, + "thread_counts": [ + 2, + 4, + 8 + ] + }, + "type": "kmeans-comparison" + }, + "results": { + "implementation": "disk", + "phase": "all", + "workloads": [ + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 103743.56666666667, + "median": 93420.0, + "minimum": 77049, + "p90": 157419, + "p99": 184676 + }, + "samples": [ + 77049, + 77749, + 80164, + 80323, + 80357, + 82078, + 82492, + 82955, + 83714, + 88226, + 89391, + 89817, + 90248, + 90379, + 92978, + 93862, + 94125, + 95850, + 99842, + 101313, + 105170, + 111335, + 112334, + 119523, + 132688, + 134830, + 141730, + 157419, + 159690, + 184676 + ] + }, + "lloyds": { + "percentiles": { + "mean": 230422.73333333334, + "median": 191393.0, + "minimum": 164178, + "p90": 455847, + "p99": 471622 + }, + "samples": [ + 164178, + 172693, + 174131, + 178912, + 179808, + 180217, + 181655, + 182001, + 182146, + 182555, + 184364, + 186070, + 187101, + 191088, + 191333, + 191453, + 198027, + 198910, + 207170, + 207464, + 207932, + 213648, + 222290, + 225560, + 229524, + 349625, + 355956, + 455847, + 459402, + 471622 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 334167.4666666667, + "median": 291924.5, + "minimum": 256210, + "p90": 594233, + "p99": 640524 + }, + "samples": [ + 256210, + 256661, + 259605, + 261820, + 262359, + 263073, + 265492, + 269026, + 270373, + 273671, + 274803, + 278491, + 279234, + 280725, + 291297, + 292552, + 303466, + 312017, + 312342, + 312539, + 318538, + 318801, + 327457, + 330716, + 346793, + 445476, + 513377, + 594233, + 613353, + 640524 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 117536.96666666666, + "median": 94530.0, + "minimum": 87947, + "p90": 120104, + "p99": 415949 + }, + "samples": [ + 87947, + 89027, + 89138, + 89249, + 89376, + 89418, + 89970, + 90039, + 91150, + 91645, + 91674, + 92369, + 92425, + 92519, + 93311, + 95749, + 96945, + 97170, + 99643, + 101829, + 104756, + 105090, + 105792, + 106085, + 106497, + 106679, + 110842, + 120104, + 393722, + 415949 + ] + }, + "lloyds": { + "percentiles": { + "mean": 312395.3, + "median": 286001.5, + "minimum": 261637, + "p90": 464581, + "p99": 562121 + }, + "samples": [ + 261637, + 262575, + 265617, + 266367, + 267494, + 268823, + 269759, + 270285, + 272186, + 275912, + 276004, + 277943, + 283104, + 283697, + 284921, + 287082, + 287422, + 287505, + 291142, + 291278, + 291294, + 291680, + 296122, + 304908, + 333473, + 355876, + 406818, + 464581, + 534233, + 562121 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 429933.5333333333, + "median": 380600.0, + "minimum": 354250, + "p90": 668801, + "p99": 880531 + }, + "samples": [ + 354250, + 358200, + 359731, + 364666, + 364707, + 367430, + 371458, + 372115, + 373144, + 374848, + 377587, + 377833, + 379090, + 379151, + 380528, + 380672, + 380713, + 380735, + 381128, + 381999, + 384051, + 385150, + 391839, + 401854, + 448302, + 453578, + 623373, + 668801, + 800542, + 880531 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 198453.73333333334, + "median": 193639.5, + "minimum": 166190, + "p90": 212224, + "p99": 343011 + }, + "samples": [ + 166190, + 175871, + 178637, + 180068, + 183414, + 183840, + 184568, + 185398, + 185435, + 186784, + 187836, + 189358, + 190229, + 192472, + 193582, + 193697, + 195452, + 195452, + 196820, + 199012, + 200194, + 203115, + 203480, + 206293, + 206644, + 209615, + 209838, + 212224, + 215083, + 343011 + ] + }, + "lloyds": { + "percentiles": { + "mean": 612818.2333333333, + "median": 586743.0, + "minimum": 568229, + "p90": 830296, + "p99": 876639 + }, + "samples": [ + 568229, + 570557, + 571243, + 573524, + 573670, + 578173, + 579619, + 580868, + 580881, + 582130, + 582594, + 583074, + 583597, + 585135, + 586454, + 587032, + 587321, + 588920, + 590286, + 592542, + 593318, + 595191, + 595958, + 596349, + 596685, + 601255, + 626081, + 830296, + 846926, + 876639 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 811273.2, + "median": 777325.0, + "minimum": 739861, + "p90": 1033777, + "p99": 1189938 + }, + "samples": [ + 739861, + 746429, + 756678, + 760950, + 765050, + 766267, + 766915, + 767012, + 767223, + 767456, + 767533, + 771180, + 772361, + 773240, + 774855, + 779795, + 784148, + 785710, + 788771, + 790645, + 793677, + 794838, + 796544, + 801145, + 805798, + 806301, + 832375, + 1033777, + 1091724, + 1189938 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 109540.23333333334, + "median": 91437.5, + "minimum": 82366, + "p90": 160856, + "p99": 349553 + }, + "samples": [ + 82366, + 82618, + 82632, + 82871, + 83010, + 83517, + 84574, + 87407, + 87793, + 87859, + 88180, + 89685, + 91041, + 91397, + 91405, + 91470, + 91741, + 92378, + 93172, + 94819, + 95093, + 95673, + 97189, + 99794, + 100013, + 100368, + 116086, + 160856, + 311647, + 349553 + ] + }, + "lloyds": { + "percentiles": { + "mean": 203150.66666666666, + "median": 192178.5, + "minimum": 180613, + "p90": 223034, + "p99": 408678 + }, + "samples": [ + 180613, + 182012, + 183375, + 183379, + 184137, + 184732, + 185010, + 185562, + 186854, + 188239, + 188858, + 190156, + 191378, + 191404, + 191683, + 192674, + 192961, + 193657, + 195139, + 196993, + 199193, + 199769, + 200856, + 201694, + 205295, + 207912, + 210076, + 223034, + 269197, + 408678 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 312692.13333333336, + "median": 284324.5, + "minimum": 266770, + "p90": 383892, + "p99": 758232 + }, + "samples": [ + 266770, + 268407, + 273065, + 273191, + 273755, + 274781, + 275953, + 276529, + 276604, + 277758, + 279552, + 279711, + 282535, + 283953, + 284072, + 284577, + 287053, + 287223, + 290280, + 291698, + 292703, + 293087, + 296788, + 296959, + 297368, + 309048, + 352715, + 383892, + 512505, + 758232 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 99598.06666666667, + "median": 96983.5, + "minimum": 89280, + "p90": 116472, + "p99": 121812 + }, + "samples": [ + 89280, + 90846, + 90883, + 91202, + 91254, + 91365, + 92030, + 92219, + 94031, + 94855, + 95347, + 95552, + 95646, + 95700, + 95932, + 98035, + 98874, + 100139, + 101515, + 102278, + 102761, + 102989, + 103222, + 103536, + 104395, + 106835, + 109018, + 116472, + 119919, + 121812 + ] + }, + "lloyds": { + "percentiles": { + "mean": 292384.5333333333, + "median": 289568.0, + "minimum": 273737, + "p90": 313800, + "p99": 315629 + }, + "samples": [ + 273737, + 274019, + 275816, + 280297, + 281170, + 281471, + 281980, + 284071, + 285207, + 285226, + 285461, + 285510, + 286549, + 287639, + 288854, + 290282, + 291010, + 292419, + 292974, + 293828, + 296446, + 301403, + 301449, + 302477, + 305991, + 309952, + 311702, + 313800, + 315167, + 315629 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 391983.6, + "median": 390559.0, + "minimum": 369951, + "p90": 408285, + "p99": 424290 + }, + "samples": [ + 369951, + 371500, + 371517, + 372865, + 377238, + 380809, + 382946, + 384340, + 384394, + 384585, + 384675, + 385930, + 386450, + 387505, + 390489, + 390629, + 391616, + 391999, + 393657, + 395406, + 401207, + 401543, + 401983, + 404448, + 406020, + 406558, + 407507, + 408285, + 419166, + 424290 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 144421.76666666666, + "median": 143072.5, + "minimum": 133527, + "p90": 158342, + "p99": 166230 + }, + "samples": [ + 133527, + 134038, + 134686, + 135479, + 136131, + 137234, + 137979, + 138344, + 139064, + 139700, + 140581, + 142725, + 142850, + 142936, + 143067, + 143078, + 143549, + 144574, + 144818, + 146232, + 146498, + 147693, + 147697, + 148662, + 150331, + 153169, + 154434, + 158342, + 159005, + 166230 + ] + }, + "lloyds": { + "percentiles": { + "mean": 597731.3666666667, + "median": 596196.5, + "minimum": 585548, + "p90": 606760, + "p99": 626390 + }, + "samples": [ + 585548, + 587730, + 588673, + 590739, + 591921, + 591984, + 592778, + 592916, + 593346, + 594343, + 595007, + 595498, + 595623, + 595731, + 595843, + 596550, + 597203, + 597652, + 599586, + 600002, + 600701, + 601228, + 601266, + 601269, + 601539, + 602451, + 604519, + 606760, + 607145, + 626390 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 742154.2333333333, + "median": 739211.0, + "minimum": 719587, + "p90": 756658, + "p99": 792621 + }, + "samples": [ + 719587, + 728396, + 728534, + 729219, + 730440, + 733336, + 733408, + 734188, + 734647, + 735424, + 736227, + 736312, + 736371, + 738567, + 739173, + 739249, + 739277, + 742009, + 743780, + 744162, + 744204, + 746882, + 747462, + 749611, + 751963, + 754437, + 755621, + 756658, + 762862, + 792621 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 218459.6, + "median": 216815.0, + "minimum": 183775, + "p90": 246588, + "p99": 267153 + }, + "samples": [ + 183775, + 190887, + 195568, + 196428, + 197432, + 201789, + 201843, + 202760, + 203073, + 205429, + 207953, + 209798, + 211094, + 211911, + 216270, + 217360, + 219428, + 221523, + 223208, + 223292, + 225672, + 229366, + 231073, + 232242, + 233118, + 236968, + 245522, + 246588, + 265265, + 267153 + ] + }, + "lloyds": { + "percentiles": { + "mean": 431326.7, + "median": 432785.0, + "minimum": 400281, + "p90": 448728, + "p99": 461766 + }, + "samples": [ + 400281, + 400693, + 410748, + 415670, + 418634, + 419555, + 422313, + 422662, + 423203, + 424421, + 425465, + 429111, + 430273, + 430983, + 432233, + 433337, + 434510, + 434530, + 437768, + 439016, + 439335, + 439626, + 440175, + 441877, + 442642, + 442985, + 443741, + 448728, + 453520, + 461766 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 649787.8, + "median": 646387.5, + "minimum": 616067, + "p90": 676014, + "p99": 701663 + }, + "samples": [ + 616067, + 616964, + 617112, + 624452, + 625047, + 632185, + 637194, + 639616, + 640170, + 641369, + 642935, + 643198, + 644346, + 645522, + 645805, + 646970, + 648344, + 651538, + 655495, + 657335, + 662414, + 662434, + 665007, + 665352, + 665935, + 666144, + 671244, + 676014, + 685763, + 701663 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 253998.66666666666, + "median": 251475.0, + "minimum": 190532, + "p90": 296480, + "p99": 299838 + }, + "samples": [ + 190532, + 206957, + 210416, + 215497, + 221938, + 223092, + 229880, + 238388, + 239019, + 244173, + 245377, + 245970, + 246138, + 246585, + 249547, + 253403, + 254544, + 262054, + 265896, + 267961, + 269255, + 271311, + 280265, + 283744, + 284929, + 287047, + 290514, + 296480, + 299210, + 299838 + ] + }, + "lloyds": { + "percentiles": { + "mean": 584898.8333333334, + "median": 580572.5, + "minimum": 549303, + "p90": 607362, + "p99": 634541 + }, + "samples": [ + 549303, + 552744, + 556848, + 561289, + 564126, + 567256, + 568066, + 570455, + 571935, + 572120, + 573147, + 575130, + 575589, + 576519, + 580242, + 580903, + 583645, + 588722, + 591441, + 599095, + 601832, + 604526, + 604601, + 604742, + 605432, + 605697, + 605991, + 607362, + 613666, + 634541 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 838898.7666666667, + "median": 835157.0, + "minimum": 762468, + "p90": 902503, + "p99": 905830 + }, + "samples": [ + 762468, + 772346, + 787861, + 793549, + 801858, + 802516, + 806400, + 811141, + 812634, + 819762, + 823343, + 826213, + 826465, + 833011, + 833963, + 836351, + 842127, + 845235, + 851329, + 854149, + 856378, + 858077, + 871612, + 874160, + 876618, + 884978, + 889443, + 902503, + 904643, + 905830 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 284052.5333333333, + "median": 278502.0, + "minimum": 228579, + "p90": 334081, + "p99": 399971 + }, + "samples": [ + 228579, + 230762, + 231351, + 235065, + 237048, + 249763, + 255927, + 256635, + 257913, + 259800, + 261440, + 263874, + 264181, + 266459, + 277512, + 279492, + 283973, + 285478, + 288775, + 294872, + 303097, + 305212, + 317479, + 319993, + 328297, + 329322, + 332595, + 334081, + 342630, + 399971 + ] + }, + "lloyds": { + "percentiles": { + "mean": 955415.8, + "median": 953971.5, + "minimum": 854034, + "p90": 1030320, + "p99": 1109617 + }, + "samples": [ + 854034, + 862089, + 870909, + 899627, + 900599, + 901642, + 912977, + 923940, + 924284, + 937307, + 939222, + 947248, + 951517, + 951978, + 952611, + 955332, + 956533, + 957562, + 967244, + 968828, + 968932, + 974416, + 987080, + 995403, + 996334, + 1008330, + 1020536, + 1030320, + 1036023, + 1109617 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 1239469.6666666667, + "median": 1241718.0, + "minimum": 1091083, + "p90": 1358619, + "p99": 1452248 + }, + "samples": [ + 1091083, + 1092852, + 1099489, + 1134694, + 1160400, + 1165517, + 1168905, + 1208154, + 1209763, + 1213246, + 1218697, + 1221744, + 1226742, + 1227039, + 1235952, + 1247484, + 1251888, + 1257301, + 1261746, + 1262794, + 1263192, + 1264593, + 1268545, + 1297466, + 1301327, + 1301424, + 1325810, + 1358619, + 1395376, + 1452248 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + } + ] + } + } +] diff --git a/issue939-benchmark-results/kmeans-baseline-results.md b/issue939-benchmark-results/kmeans-baseline-results.md new file mode 100644 index 000000000..1c72a1aba --- /dev/null +++ b/issue939-benchmark-results/kmeans-baseline-results.md @@ -0,0 +1,65 @@ +# Issue #939 K-means baseline + +## Experiment contract + +- Revision: `59dd0481` (`origin/main` when the baseline worktree was created) +- Worktree: `C:\src\DiskANN-issue-939-baseline` +- Branch: `wuw92/issue-939-benchmark-baseline` +- Dataset: deterministic synthetic `f32` values sampled uniformly from `[-1, 1)`, seed 42 +- Points: 50,000 +- Centers: 256 +- Lloyd iterations: 10 +- Measurements: 30 per workload +- Build: release, `diskann-benchmark` with `kmeans-comparison` + +Each measurement calls K-means++ once and, for `phase: "all"`, Lloyd once using the selected centers. The benchmark records initialization, Lloyd, and total wall-clock latency from that single execution. Fixture generation and quality calculation are outside the timed region. + +The Disk path calls `k_meanspp_selecting_pivots` followed by `run_lloyds`, passing the explicit repository Rayon pool to both functions. The quantization path calls `kmeans_plusplus_into` followed by `lloyds`; these APIs are sequential, so changing the benchmark thread count does not change their execution. + +## Dimension sweep + +Thread count is 1. Latencies are medians in milliseconds. + +| Dimension | Disk init | Quant init | Init change | Disk Lloyd | Quant Lloyd | Lloyd change | Disk total | Quant total | Total change | +|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| 4 | 79.132 | 12.890 | -83.7% | 154.482 | 26.176 | -83.1% | 233.885 | 39.269 | -83.2% | +| 32 | 93.925 | 36.040 | -61.6% | 238.536 | 147.149 | -38.3% | 332.902 | 183.473 | -44.9% | +| 128 | 202.912 | 173.231 | -14.6% | 553.041 | 663.220 | +19.9% | 760.494 | 836.838 | +10.0% | +| 384 | 887.169 | 767.457 | -13.5% | 1,373.521 | 2,045.529 | +48.9% | 2,264.664 | 2,812.458 | +24.2% | +| 768 | 1,771.880 | 1,688.607 | -4.7% | 2,635.229 | 4,117.453 | +56.2% | 4,404.248 | 5,799.877 | +31.7% | +| 1,024 | 2,341.492 | 2,330.105 | -0.5% | 3,305.319 | 5,515.243 | +66.9% | 5,645.712 | 7,842.756 | +38.9% | +| 3,072 | 6,969.815 | 7,293.668 | +4.6% | 6,726.029 | 16,666.082 | +147.8% | 13,696.581 | 23,970.544 | +75.0% | + +**[measured]** The baseline reproduces the high-dimensional gap from issue #939. Quantization crosses from faster to slower at dimension 128 and reaches a 75.0% total-latency regression at dimension 3,072. The gap is concentrated in Lloyd, which is 147.8% slower at dimension 3,072. + +## Thread sweep + +Dimensions are 4, 32, and 128. Latencies are medians in milliseconds. + +| Dimension | Threads | Disk init | Quant init | Disk Lloyd | Quant Lloyd | Disk total | Quant total | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 4 | 1 | 79.132 | 12.890 | 154.482 | 26.176 | 233.885 | 39.269 | +| 4 | 2 | 93.420 | 14.013 | 191.393 | 28.925 | 291.925 | 43.144 | +| 4 | 4 | 91.438 | 14.634 | 192.179 | 29.078 | 284.325 | 44.184 | +| 4 | 8 | 216.815 | 14.135 | 432.785 | 29.009 | 646.388 | 43.416 | +| 32 | 1 | 93.925 | 36.040 | 238.536 | 147.149 | 332.902 | 183.473 | +| 32 | 2 | 94.530 | 41.224 | 286.002 | 160.463 | 380.600 | 201.898 | +| 32 | 4 | 96.984 | 41.086 | 289.568 | 165.916 | 390.559 | 206.356 | +| 32 | 8 | 251.475 | 41.028 | 580.572 | 158.886 | 835.157 | 201.424 | +| 128 | 1 | 202.912 | 173.231 | 553.041 | 663.220 | 760.494 | 836.838 | +| 128 | 2 | 193.639 | 239.483 | 586.743 | 711.697 | 777.325 | 955.052 | +| 128 | 4 | 143.072 | 250.808 | 596.197 | 713.639 | 739.211 | 965.208 | +| 128 | 8 | 278.502 | 261.207 | 953.971 | 728.626 | 1,241.718 | 986.318 | + +**[measured]** Quantization remains effectively insensitive to the configured thread count at dimensions 4 and 32; dimension 128 varies more but does not scale down with additional threads. Disk reaches its best dimension-128 total at four threads and regresses sharply at eight threads across all three dimensions on this host. + +## Quality observations + +The final center hash and objective match exactly through dimension 384 and for every thread-sweep workload. They differ at dimensions 768, 1,024, and 3,072, so performance comparisons at those dimensions should not be interpreted as bit-identical algorithm executions. + +## Raw results + +- `disk-baseline.json` +- `quantization-baseline.json` + +The benchmark also accepts `phase: "init"` to execute and report initialization only; `issue-939-kmeans-init.json` provides a ready-to-run configuration. diff --git a/issue939-benchmark-results/quantization-baseline.json b/issue939-benchmark-results/quantization-baseline.json new file mode 100644 index 000000000..f866db829 --- /dev/null +++ b/issue939-benchmark-results/quantization-baseline.json @@ -0,0 +1,2212 @@ +[ + { + "input": { + "content": { + "center_counts": [ + 256 + ], + "dimensions": [ + 4, + 32, + 128, + 384, + 768, + 1024, + 3072 + ], + "implementation": "quantization", + "max_iterations": 10, + "measurements": 30, + "num_points": 50000, + "phase": "all", + "seed": 42, + "thread_counts": [ + 1 + ] + }, + "type": "kmeans-comparison" + }, + "results": { + "implementation": "quantization", + "phase": "all", + "workloads": [ + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 13093.266666666666, + "median": 12890.5, + "minimum": 12590, + "p90": 13882, + "p99": 14868 + }, + "samples": [ + 12590, + 12595, + 12610, + 12628, + 12642, + 12651, + 12667, + 12697, + 12733, + 12782, + 12801, + 12819, + 12821, + 12836, + 12879, + 12902, + 12913, + 13097, + 13154, + 13210, + 13263, + 13267, + 13294, + 13369, + 13378, + 13619, + 13750, + 13882, + 14081, + 14868 + ] + }, + "lloyds": { + "percentiles": { + "mean": 26601.7, + "median": 26176.0, + "minimum": 25704, + "p90": 27813, + "p99": 34035 + }, + "samples": [ + 25704, + 25711, + 25780, + 25784, + 25816, + 25863, + 25885, + 25892, + 25942, + 25971, + 26023, + 26057, + 26107, + 26157, + 26175, + 26177, + 26251, + 26314, + 26346, + 26350, + 26358, + 26440, + 26492, + 26742, + 27023, + 27145, + 27496, + 27813, + 28202, + 34035 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 39695.933333333334, + "median": 39269.0, + "minimum": 38364, + "p90": 41026, + "p99": 47405 + }, + "samples": [ + 38364, + 38380, + 38447, + 38505, + 38514, + 38526, + 38634, + 38675, + 38791, + 38885, + 39009, + 39055, + 39111, + 39134, + 39210, + 39328, + 39407, + 39544, + 39651, + 39736, + 39796, + 40101, + 40229, + 40259, + 40402, + 40761, + 40911, + 41026, + 41082, + 47405 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 36423.76666666667, + "median": 36040.0, + "minimum": 34810, + "p90": 39532, + "p99": 39718 + }, + "samples": [ + 34810, + 34908, + 35000, + 35108, + 35186, + 35424, + 35446, + 35566, + 35586, + 35652, + 35658, + 35705, + 35724, + 35778, + 35967, + 36113, + 36211, + 36227, + 36377, + 36378, + 36461, + 36543, + 36675, + 37424, + 37649, + 37691, + 38504, + 39532, + 39692, + 39718 + ] + }, + "lloyds": { + "percentiles": { + "mean": 147563.8, + "median": 147148.5, + "minimum": 145458, + "p90": 149065, + "p99": 158325 + }, + "samples": [ + 145458, + 145651, + 145786, + 146037, + 146063, + 146070, + 146548, + 146610, + 146633, + 146655, + 146669, + 146737, + 146753, + 146776, + 147100, + 147197, + 147421, + 147612, + 147642, + 147653, + 147838, + 147867, + 147965, + 148029, + 148058, + 148182, + 148554, + 149065, + 149960, + 158325 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 183988.83333333334, + "median": 183473.0, + "minimum": 180462, + "p90": 187335, + "p99": 194704 + }, + "samples": [ + 180462, + 181146, + 181543, + 181736, + 181900, + 182200, + 182236, + 182389, + 182533, + 182753, + 182923, + 183115, + 183240, + 183332, + 183391, + 183555, + 183717, + 183736, + 183755, + 183834, + 183964, + 184067, + 184201, + 184393, + 185072, + 186504, + 187331, + 187335, + 188598, + 194704 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 175995.96666666667, + "median": 173230.5, + "minimum": 162613, + "p90": 188954, + "p99": 218934 + }, + "samples": [ + 162613, + 163729, + 166644, + 166766, + 166928, + 166950, + 167000, + 167613, + 168615, + 169056, + 169114, + 169989, + 170169, + 171065, + 171656, + 174805, + 174857, + 175221, + 177412, + 177422, + 178665, + 181445, + 182139, + 183985, + 184953, + 185889, + 187711, + 188954, + 189580, + 218934 + ] + }, + "lloyds": { + "percentiles": { + "mean": 663505.5, + "median": 663219.5, + "minimum": 659915, + "p90": 666756, + "p99": 667433 + }, + "samples": [ + 659915, + 661219, + 662163, + 662273, + 662336, + 662509, + 662552, + 662559, + 662577, + 662577, + 662613, + 662756, + 662830, + 662898, + 663187, + 663252, + 663532, + 663615, + 663754, + 663780, + 663789, + 663884, + 664022, + 664358, + 664546, + 664803, + 665444, + 666756, + 667233, + 667433 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 839503.6666666666, + "median": 836838.5, + "minimum": 825370, + "p90": 852091, + "p99": 886368 + }, + "samples": [ + 825370, + 825893, + 826916, + 830191, + 831193, + 831388, + 831570, + 832395, + 832604, + 833356, + 833402, + 833416, + 833994, + 834162, + 835117, + 838560, + 838648, + 839003, + 840956, + 841028, + 841218, + 844344, + 846546, + 846686, + 848720, + 848838, + 848931, + 852091, + 852206, + 886368 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 6161376117261214841, + "initialization": { + "percentiles": { + "mean": 769465.5666666667, + "median": 767456.5, + "minimum": 755988, + "p90": 782713, + "p99": 800520 + }, + "samples": [ + 755988, + 758069, + 759617, + 759803, + 760377, + 761240, + 761681, + 762062, + 762525, + 762741, + 763926, + 764592, + 765516, + 765562, + 766317, + 768596, + 769937, + 770691, + 772052, + 772711, + 773004, + 773826, + 774172, + 775096, + 777029, + 779299, + 780667, + 782713, + 783638, + 800520 + ] + }, + "lloyds": { + "percentiles": { + "mean": 2045415.4333333333, + "median": 2045529.0, + "minimum": 2042191, + "p90": 2047474, + "p99": 2050228 + }, + "samples": [ + 2042191, + 2042416, + 2042593, + 2042653, + 2043341, + 2043502, + 2043676, + 2043796, + 2044051, + 2044308, + 2044408, + 2044543, + 2044781, + 2044801, + 2045356, + 2045702, + 2045772, + 2045972, + 2046475, + 2046685, + 2046703, + 2046989, + 2047042, + 2047122, + 2047132, + 2047309, + 2047471, + 2047474, + 2047971, + 2050228 + ] + }, + "objective": 6189761.912239075, + "total": { + "percentiles": { + "mean": 2814882.3333333335, + "median": 2812458.0, + "minimum": 2801961, + "p90": 2826292, + "p99": 2847642 + }, + "samples": [ + 2801961, + 2805380, + 2806080, + 2807014, + 2807276, + 2807933, + 2808538, + 2808724, + 2809001, + 2809846, + 2810216, + 2810498, + 2811059, + 2811119, + 2812267, + 2812649, + 2812883, + 2815730, + 2816346, + 2816623, + 2817255, + 2817623, + 2818482, + 2819623, + 2819879, + 2824657, + 2826218, + 2826292, + 2827656, + 2847642 + ] + }, + "workload": { + "dim": 384, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 14566160934793959414, + "initialization": { + "percentiles": { + "mean": 1696962.2333333334, + "median": 1688607.0, + "minimum": 1650823, + "p90": 1729076, + "p99": 1860238 + }, + "samples": [ + 1650823, + 1658197, + 1658233, + 1665562, + 1666267, + 1667541, + 1668434, + 1670128, + 1671154, + 1675488, + 1676122, + 1680987, + 1682994, + 1683111, + 1688004, + 1689210, + 1690134, + 1693381, + 1708517, + 1710475, + 1711042, + 1714726, + 1715402, + 1716812, + 1718861, + 1722193, + 1727809, + 1729076, + 1737946, + 1860238 + ] + }, + "lloyds": { + "percentiles": { + "mean": 4119677.3666666667, + "median": 4117452.5, + "minimum": 4101898, + "p90": 4133925, + "p99": 4136807 + }, + "samples": [ + 4101898, + 4103161, + 4109324, + 4110755, + 4113907, + 4114024, + 4114099, + 4114591, + 4114753, + 4115561, + 4115821, + 4115900, + 4116451, + 4116697, + 4116872, + 4118033, + 4118800, + 4119289, + 4121173, + 4121610, + 4126000, + 4126538, + 4126702, + 4126878, + 4127384, + 4127738, + 4131557, + 4133925, + 4134073, + 4136807 + ] + }, + "objective": 12551421.084503174, + "total": { + "percentiles": { + "mean": 5816640.933333334, + "median": 5799877.5, + "minimum": 5766724, + "p90": 5855670, + "p99": 5991796 + }, + "samples": [ + 5766724, + 5774887, + 5775071, + 5782089, + 5782459, + 5784036, + 5785112, + 5785576, + 5786156, + 5789904, + 5790242, + 5795412, + 5797694, + 5798673, + 5799789, + 5799966, + 5804235, + 5820084, + 5829994, + 5835900, + 5835902, + 5838215, + 5838892, + 5845116, + 5845528, + 5849420, + 5850738, + 5855670, + 5863948, + 5991796 + ] + }, + "workload": { + "dim": 768, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 18077977999153830296, + "initialization": { + "percentiles": { + "mean": 2329945.7333333334, + "median": 2330105.0, + "minimum": 2294280, + "p90": 2356371, + "p99": 2383549 + }, + "samples": [ + 2294280, + 2303697, + 2305339, + 2305669, + 2309027, + 2309896, + 2310610, + 2312125, + 2314649, + 2317666, + 2318330, + 2319183, + 2326239, + 2326995, + 2329769, + 2330441, + 2330678, + 2332067, + 2335466, + 2335967, + 2336688, + 2337062, + 2337522, + 2340404, + 2348113, + 2351492, + 2355929, + 2356371, + 2383149, + 2383549 + ] + }, + "lloyds": { + "percentiles": { + "mean": 5515752.433333334, + "median": 5515242.5, + "minimum": 5500971, + "p90": 5527831, + "p99": 5535827 + }, + "samples": [ + 5500971, + 5501024, + 5501642, + 5503907, + 5505187, + 5508187, + 5508363, + 5509669, + 5511690, + 5511768, + 5512275, + 5513724, + 5513764, + 5514634, + 5514980, + 5515505, + 5517262, + 5517558, + 5517804, + 5519914, + 5520844, + 5520869, + 5521251, + 5521312, + 5524630, + 5525226, + 5525997, + 5527831, + 5528958, + 5535827 + ] + }, + "objective": 16794756.62135315, + "total": { + "percentiles": { + "mean": 7845699.4, + "median": 7842755.5, + "minimum": 7799468, + "p90": 7884204, + "p99": 7908776 + }, + "samples": [ + 7799468, + 7807312, + 7814518, + 7818638, + 7819355, + 7823620, + 7824949, + 7826419, + 7829872, + 7833818, + 7834427, + 7836757, + 7838805, + 7841168, + 7841977, + 7843534, + 7844344, + 7847156, + 7850074, + 7854247, + 7854868, + 7856837, + 7859637, + 7863520, + 7868028, + 7868756, + 7871436, + 7884204, + 7904462, + 7908776 + ] + }, + "workload": { + "dim": 1024, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + }, + { + "center_hash": 571235301484887008, + "initialization": { + "percentiles": { + "mean": 7326393.8, + "median": 7293668.5, + "minimum": 7176322, + "p90": 7485590, + "p99": 7580349 + }, + "samples": [ + 7176322, + 7219695, + 7225947, + 7235262, + 7235270, + 7243231, + 7256728, + 7263396, + 7265813, + 7269106, + 7271179, + 7276415, + 7286493, + 7291970, + 7293155, + 7294182, + 7304200, + 7306860, + 7326126, + 7332088, + 7334595, + 7342484, + 7343923, + 7423923, + 7428231, + 7448687, + 7478711, + 7485590, + 7551883, + 7580349 + ] + }, + "lloyds": { + "percentiles": { + "mean": 16668619.166666666, + "median": 16666081.5, + "minimum": 16586869, + "p90": 16728774, + "p99": 16804942 + }, + "samples": [ + 16586869, + 16593323, + 16628608, + 16630320, + 16636152, + 16637893, + 16639131, + 16639207, + 16641850, + 16648074, + 16650442, + 16651373, + 16653559, + 16658798, + 16663668, + 16668495, + 16669581, + 16673570, + 16679692, + 16683027, + 16685549, + 16685646, + 16688670, + 16692085, + 16701854, + 16702197, + 16706041, + 16728774, + 16729185, + 16804942 + ] + }, + "objective": 50767967.460754395, + "total": { + "percentiles": { + "mean": 23995014.166666668, + "median": 23970544.5, + "minimum": 23818173, + "p90": 24134334, + "p99": 24254082 + }, + "samples": [ + 23818173, + 23862431, + 23863879, + 23865080, + 23878494, + 23901290, + 23921465, + 23928781, + 23929482, + 23937867, + 23940761, + 23945398, + 23959443, + 23964037, + 23968241, + 23972848, + 23974201, + 23977772, + 23998946, + 24000224, + 24012419, + 24071670, + 24074367, + 24096913, + 24115912, + 24117919, + 24130087, + 24134334, + 24233909, + 24254082 + ] + }, + "workload": { + "dim": 3072, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 1 + } + } + ] + } + }, + { + "input": { + "content": { + "center_counts": [ + 256 + ], + "dimensions": [ + 4, + 32, + 128 + ], + "implementation": "quantization", + "max_iterations": 10, + "measurements": 30, + "num_points": 50000, + "phase": "all", + "seed": 42, + "thread_counts": [ + 2, + 4, + 8 + ] + }, + "type": "kmeans-comparison" + }, + "results": { + "implementation": "quantization", + "phase": "all", + "workloads": [ + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 17185.666666666668, + "median": 14013.0, + "minimum": 13397, + "p90": 19924, + "p99": 68087 + }, + "samples": [ + 13397, + 13516, + 13534, + 13594, + 13707, + 13722, + 13723, + 13728, + 13797, + 13834, + 13836, + 13879, + 13888, + 13935, + 13954, + 14072, + 14162, + 14201, + 14447, + 14587, + 14851, + 15378, + 18487, + 19297, + 19556, + 19623, + 19785, + 19924, + 23069, + 68087 + ] + }, + "lloyds": { + "percentiles": { + "mean": 37465.433333333334, + "median": 28925.0, + "minimum": 26879, + "p90": 49681, + "p99": 166068 + }, + "samples": [ + 26879, + 26915, + 27123, + 27176, + 27373, + 27407, + 27495, + 27560, + 27610, + 27879, + 27938, + 27954, + 28495, + 28548, + 28683, + 29167, + 29368, + 29586, + 29831, + 29929, + 31676, + 31959, + 36260, + 42056, + 45224, + 45344, + 47519, + 49681, + 65260, + 166068 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 54652.4, + "median": 43143.5, + "minimum": 40641, + "p90": 69606, + "p99": 179467 + }, + "samples": [ + 40641, + 40768, + 41013, + 41203, + 41395, + 41503, + 41533, + 41570, + 41677, + 41952, + 42224, + 42484, + 42563, + 42891, + 43121, + 43166, + 44034, + 44781, + 45632, + 46672, + 47167, + 47339, + 56047, + 59793, + 60543, + 67144, + 68294, + 69606, + 133349, + 179467 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 41495.4, + "median": 41224.5, + "minimum": 37700, + "p90": 46087, + "p99": 46960 + }, + "samples": [ + 37700, + 38128, + 38331, + 38417, + 38614, + 38676, + 39201, + 39264, + 40150, + 40170, + 40589, + 40931, + 41016, + 41109, + 41115, + 41334, + 41365, + 41436, + 41579, + 41626, + 42107, + 42238, + 42554, + 43097, + 44498, + 44913, + 45396, + 46087, + 46261, + 46960 + ] + }, + "lloyds": { + "percentiles": { + "mean": 161288.46666666667, + "median": 160462.5, + "minimum": 154615, + "p90": 170258, + "p99": 178819 + }, + "samples": [ + 154615, + 155986, + 156001, + 156057, + 156588, + 156610, + 157120, + 157166, + 157633, + 158327, + 159756, + 159834, + 160130, + 160313, + 160416, + 160509, + 160541, + 160615, + 160921, + 160936, + 161311, + 162343, + 162770, + 163300, + 163830, + 165600, + 169222, + 170258, + 171127, + 178819 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 202784.93333333332, + "median": 201897.5, + "minimum": 193688, + "p90": 211275, + "p99": 220399 + }, + "samples": [ + 193688, + 195797, + 196052, + 196172, + 196242, + 198828, + 199165, + 199259, + 199274, + 199396, + 199440, + 200675, + 200872, + 201204, + 201651, + 202144, + 202256, + 202502, + 203514, + 204136, + 204215, + 204812, + 205424, + 206096, + 207896, + 209227, + 210659, + 211275, + 211278, + 220399 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 242522.76666666666, + "median": 239483.5, + "minimum": 222652, + "p90": 260651, + "p99": 263946 + }, + "samples": [ + 222652, + 227578, + 230223, + 230527, + 231677, + 233676, + 234155, + 234824, + 236595, + 236876, + 238192, + 238392, + 238467, + 238844, + 239300, + 239667, + 241920, + 245330, + 246065, + 246474, + 247164, + 248418, + 250077, + 250728, + 252380, + 253921, + 254804, + 260651, + 262160, + 263946 + ] + }, + "lloyds": { + "percentiles": { + "mean": 713255.9333333333, + "median": 711696.5, + "minimum": 703226, + "p90": 721753, + "p99": 738898 + }, + "samples": [ + 703226, + 703694, + 704042, + 706484, + 706671, + 706799, + 706846, + 707345, + 707951, + 708028, + 708181, + 708491, + 708653, + 709135, + 711522, + 711871, + 711979, + 712429, + 716167, + 716988, + 717146, + 718053, + 718171, + 718312, + 718445, + 718494, + 719390, + 721753, + 732514, + 738898 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 955779.9666666667, + "median": 955051.5, + "minimum": 935082, + "p90": 973082, + "p99": 983915 + }, + "samples": [ + 935082, + 937375, + 938349, + 941502, + 942072, + 945615, + 945632, + 946418, + 946685, + 948668, + 951172, + 952171, + 953865, + 954720, + 954909, + 955194, + 956816, + 956853, + 958455, + 959180, + 961289, + 962103, + 964586, + 964694, + 967339, + 968900, + 969468, + 973082, + 977290, + 983915 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 2 + } + }, + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 14830.5, + "median": 14634.0, + "minimum": 13422, + "p90": 16136, + "p99": 19770 + }, + "samples": [ + 13422, + 13524, + 13698, + 13775, + 13834, + 13924, + 13935, + 13968, + 14121, + 14156, + 14190, + 14272, + 14456, + 14581, + 14613, + 14655, + 14663, + 14767, + 14775, + 14981, + 15064, + 15096, + 15356, + 15526, + 15593, + 15753, + 16103, + 16136, + 16208, + 19770 + ] + }, + "lloyds": { + "percentiles": { + "mean": 30321.133333333335, + "median": 29077.5, + "minimum": 27903, + "p90": 34193, + "p99": 42565 + }, + "samples": [ + 27903, + 28091, + 28109, + 28213, + 28284, + 28316, + 28323, + 28413, + 28505, + 28592, + 28757, + 28819, + 28841, + 28853, + 28894, + 29261, + 30024, + 30142, + 30308, + 30378, + 30701, + 30792, + 31280, + 31344, + 31746, + 32561, + 32644, + 34193, + 34782, + 42565 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 45152.833333333336, + "median": 44184.5, + "minimum": 42100, + "p90": 50376, + "p99": 58702 + }, + "samples": [ + 42100, + 42338, + 42405, + 42680, + 42877, + 42914, + 42930, + 43173, + 43310, + 43349, + 43383, + 43447, + 43487, + 43550, + 44143, + 44226, + 44297, + 44346, + 44491, + 44945, + 45043, + 45216, + 45313, + 46068, + 46844, + 48316, + 49914, + 50376, + 50402, + 58702 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 41797.333333333336, + "median": 41086.0, + "minimum": 37551, + "p90": 46474, + "p99": 49521 + }, + "samples": [ + 37551, + 37668, + 38970, + 39283, + 39388, + 39422, + 40022, + 40267, + 40362, + 40630, + 40804, + 40841, + 40865, + 40920, + 41077, + 41095, + 41568, + 41675, + 41854, + 41858, + 41885, + 42359, + 42831, + 43397, + 43668, + 44429, + 44546, + 46474, + 48690, + 49521 + ] + }, + "lloyds": { + "percentiles": { + "mean": 192288.93333333332, + "median": 165915.5, + "minimum": 155857, + "p90": 338068, + "p99": 378457 + }, + "samples": [ + 155857, + 156582, + 157433, + 157676, + 157905, + 158905, + 159975, + 160051, + 160148, + 160667, + 161103, + 162115, + 162680, + 163555, + 165018, + 166813, + 167152, + 168868, + 173437, + 176994, + 179234, + 180229, + 182317, + 182996, + 194447, + 197313, + 335709, + 338068, + 346964, + 378457 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 234087.6, + "median": 206355.5, + "minimum": 195280, + "p90": 384400, + "p99": 419263 + }, + "samples": [ + 195280, + 196822, + 197701, + 198827, + 199771, + 200338, + 200349, + 201129, + 201148, + 202746, + 202962, + 203027, + 203824, + 204151, + 206124, + 206587, + 207892, + 211700, + 215323, + 217836, + 218518, + 222340, + 223899, + 226394, + 241743, + 243970, + 379745, + 384400, + 388819, + 419263 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 252115.93333333332, + "median": 250807.5, + "minimum": 215024, + "p90": 271767, + "p99": 310375 + }, + "samples": [ + 215024, + 225289, + 234875, + 237623, + 238325, + 239624, + 239922, + 240313, + 240314, + 242584, + 244038, + 244040, + 245314, + 247590, + 249988, + 251627, + 251739, + 252526, + 252649, + 253449, + 253569, + 254404, + 256727, + 258998, + 263945, + 269965, + 271256, + 271767, + 305619, + 310375 + ] + }, + "lloyds": { + "percentiles": { + "mean": 720034.1, + "median": 713639.5, + "minimum": 693751, + "p90": 730099, + "p99": 879119 + }, + "samples": [ + 693751, + 698405, + 698903, + 701029, + 701532, + 701604, + 703989, + 704111, + 704333, + 704705, + 706426, + 707538, + 710642, + 711687, + 712878, + 714401, + 714881, + 716589, + 717579, + 719104, + 719222, + 721685, + 722695, + 723206, + 726978, + 729448, + 729579, + 730099, + 774905, + 879119 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 972151.4, + "median": 965208.0, + "minimum": 916055, + "p90": 996945, + "p99": 1184739 + }, + "samples": [ + 916055, + 929402, + 936029, + 938529, + 944648, + 944752, + 947321, + 948744, + 949757, + 954182, + 956516, + 956903, + 958193, + 960633, + 965136, + 965280, + 966537, + 968806, + 969277, + 970022, + 970963, + 972862, + 973489, + 979935, + 981207, + 981525, + 990872, + 996945, + 1085283, + 1184739 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 4 + } + }, + { + "center_hash": 12536658597732962046, + "initialization": { + "percentiles": { + "mean": 14667.066666666668, + "median": 14135.0, + "minimum": 13180, + "p90": 17515, + "p99": 19109 + }, + "samples": [ + 13180, + 13246, + 13360, + 13404, + 13408, + 13470, + 13472, + 13610, + 13641, + 13676, + 13881, + 13897, + 13906, + 14050, + 14121, + 14149, + 14177, + 14287, + 14303, + 14304, + 14640, + 14673, + 14963, + 15279, + 16158, + 16954, + 17224, + 17515, + 17955, + 19109 + ] + }, + "lloyds": { + "percentiles": { + "mean": 30603.733333333334, + "median": 29008.5, + "minimum": 27730, + "p90": 37408, + "p99": 42911 + }, + "samples": [ + 27730, + 27958, + 28004, + 28041, + 28041, + 28338, + 28378, + 28467, + 28551, + 28707, + 28833, + 28840, + 28869, + 28910, + 28946, + 29071, + 29099, + 29263, + 29354, + 29623, + 29840, + 30449, + 30542, + 30852, + 31753, + 32887, + 35725, + 37408, + 42722, + 42911 + ] + }, + "objective": 4127.379323303059, + "total": { + "percentiles": { + "mean": 45271.8, + "median": 43415.5, + "minimum": 41288, + "p90": 56518, + "p99": 58881 + }, + "samples": [ + 41288, + 41600, + 41781, + 41809, + 41901, + 42181, + 42280, + 42350, + 42374, + 42624, + 42666, + 43031, + 43138, + 43225, + 43321, + 43510, + 43722, + 43928, + 44261, + 44571, + 44692, + 45866, + 46356, + 47027, + 47851, + 48977, + 49336, + 56518, + 57089, + 58881 + ] + }, + "workload": { + "dim": 4, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + }, + { + "center_hash": 6622839165321859206, + "initialization": { + "percentiles": { + "mean": 41054.2, + "median": 41028.0, + "minimum": 36883, + "p90": 44385, + "p99": 51314 + }, + "samples": [ + 36883, + 37332, + 37927, + 38437, + 38499, + 38503, + 38522, + 38671, + 38715, + 39253, + 39340, + 39495, + 40265, + 40457, + 40758, + 41298, + 41392, + 41502, + 41597, + 41621, + 41786, + 42446, + 42548, + 42947, + 43249, + 43438, + 44025, + 44385, + 45021, + 51314 + ] + }, + "lloyds": { + "percentiles": { + "mean": 160811.73333333334, + "median": 158885.5, + "minimum": 153092, + "p90": 169693, + "p99": 170019 + }, + "samples": [ + 153092, + 154501, + 155113, + 155246, + 155408, + 155664, + 155941, + 156125, + 156874, + 157335, + 157480, + 157533, + 158013, + 158194, + 158444, + 159327, + 159549, + 160550, + 162143, + 163108, + 163182, + 164333, + 164928, + 165414, + 168769, + 168921, + 169692, + 169693, + 169761, + 170019 + ] + }, + "objective": 394236.77257442474, + "total": { + "percentiles": { + "mean": 201867.2, + "median": 201424.5, + "minimum": 192434, + "p90": 210779, + "p99": 211154 + }, + "samples": [ + 192434, + 193458, + 193614, + 193847, + 194741, + 194918, + 196517, + 197477, + 197850, + 198903, + 199155, + 199884, + 200327, + 200428, + 201216, + 201633, + 201897, + 202337, + 205194, + 205393, + 205556, + 205817, + 207013, + 207209, + 207440, + 208410, + 210424, + 210779, + 210991, + 211154 + ] + }, + "workload": { + "dim": 32, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + }, + { + "center_hash": 14735922192017860116, + "initialization": { + "percentiles": { + "mean": 299173.63333333336, + "median": 261206.5, + "minimum": 235709, + "p90": 372930, + "p99": 1076387 + }, + "samples": [ + 235709, + 236347, + 236625, + 239721, + 246784, + 247345, + 247517, + 249082, + 252236, + 252437, + 252816, + 252827, + 253825, + 257898, + 258788, + 263625, + 267377, + 267604, + 267690, + 268953, + 269407, + 276253, + 283904, + 299980, + 301729, + 310009, + 347251, + 372930, + 382153, + 1076387 + ] + }, + "lloyds": { + "percentiles": { + "mean": 766883.0666666667, + "median": 728626.5, + "minimum": 705553, + "p90": 933792, + "p99": 999121 + }, + "samples": [ + 705553, + 707225, + 708154, + 711401, + 714584, + 714608, + 716129, + 717032, + 718768, + 721786, + 722170, + 722798, + 723455, + 725608, + 728529, + 728724, + 732473, + 732483, + 733897, + 737420, + 755079, + 761810, + 771230, + 779823, + 854825, + 864363, + 894288, + 933792, + 969364, + 999121 + ] + }, + "objective": 1962732.4958267212, + "total": { + "percentiles": { + "mean": 1066058.1333333333, + "median": 986318.5, + "minimum": 942180, + "p90": 1223191, + "p99": 1847618 + }, + "samples": [ + 942180, + 956754, + 957880, + 959146, + 962914, + 967436, + 973126, + 973374, + 975875, + 976180, + 978779, + 982394, + 984409, + 985300, + 986134, + 986503, + 989478, + 998132, + 1016379, + 1033466, + 1063541, + 1079804, + 1131969, + 1137233, + 1146727, + 1191691, + 1202078, + 1223191, + 1372053, + 1847618 + ] + }, + "workload": { + "dim": 128, + "max_iterations": 10, + "num_centers": 256, + "num_points": 50000, + "threads": 8 + } + } + ] + } + } +] diff --git a/issue939-benchmark-results/stage-comparison.svg b/issue939-benchmark-results/stage-comparison.svg new file mode 100644 index 000000000..c2cad7d19 --- /dev/null +++ b/issue939-benchmark-results/stage-comparison.svg @@ -0,0 +1,30 @@ + + K-means phase latency by dimension + Disk and quantization median initialization and Lloyd latency for one thread. + + K-means phase latency by dimension (1 thread) + + Disk + Quantization + + + + K-means++ initialization + + 048 s + + + + Lloyd iterations + + 0918 s + + + + + + + 43212838476810243072 + Dimensions + + diff --git a/issue939-benchmark-results/thread-comparison.svg b/issue939-benchmark-results/thread-comparison.svg new file mode 100644 index 000000000..803be728c --- /dev/null +++ b/issue939-benchmark-results/thread-comparison.svg @@ -0,0 +1,33 @@ + + Total K-means latency by configured thread count + Disk and quantization median total latency for dimensions 4, 32, and 128. + + Total K-means latency by configured threads + + Disk + Quantization + + + + Dimension 4 + + 0350700 ms + + + Dimension 32 + + 0450900 ms + + + Dimension 128 + + 06501300 ms + + + + + + 1248 + Configured threads + +