From d9c8ac1de0d81e4b1f73ef7fa54c050b73a1371b Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 28 Sep 2026 11:55:14 +0800 Subject: [PATCH] perf: adopt Lance full-snapshot prefilter optimization --- Cargo.lock | 42 ++++++++-------- Cargo.toml | 22 ++++----- tests/c_api_test.rs | 118 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cf44e4..3193fec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2452,7 +2452,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-array", "rand 0.9.2", @@ -3655,7 +3655,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arc-swap", "arrow", @@ -3727,7 +3727,7 @@ dependencies = [ [[package]] name = "lance-arrow" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-array", "arrow-buffer", @@ -3749,7 +3749,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-array", "arrow-buffer", @@ -3763,7 +3763,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-array", "arrow-schema", @@ -3773,7 +3773,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrayref", "crunchy", @@ -3816,7 +3816,7 @@ dependencies = [ [[package]] name = "lance-core" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-array", "arrow-buffer", @@ -3854,7 +3854,7 @@ dependencies = [ [[package]] name = "lance-datafusion" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow", "arrow-array", @@ -3886,7 +3886,7 @@ dependencies = [ [[package]] name = "lance-datagen" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow", "arrow-array", @@ -3904,7 +3904,7 @@ dependencies = [ [[package]] name = "lance-derive" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "proc-macro2", "quote", @@ -3914,7 +3914,7 @@ dependencies = [ [[package]] name = "lance-encoding" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-arith", "arrow-array", @@ -3948,7 +3948,7 @@ dependencies = [ [[package]] name = "lance-file" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-arith", "arrow-array", @@ -3980,7 +3980,7 @@ dependencies = [ [[package]] name = "lance-geo" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "datafusion", "geo-traits", @@ -3995,7 +3995,7 @@ dependencies = [ [[package]] name = "lance-index" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arc-swap", "arrow", @@ -4063,7 +4063,7 @@ dependencies = [ [[package]] name = "lance-index-core" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-array", "arrow-schema", @@ -4086,7 +4086,7 @@ dependencies = [ [[package]] name = "lance-io" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow", "arrow-array", @@ -4126,7 +4126,7 @@ dependencies = [ [[package]] name = "lance-linalg" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-array", "arrow-schema", @@ -4141,7 +4141,7 @@ dependencies = [ [[package]] name = "lance-namespace" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow", "async-trait", @@ -4168,7 +4168,7 @@ dependencies = [ [[package]] name = "lance-select" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow-array", "arrow-buffer", @@ -4183,7 +4183,7 @@ dependencies = [ [[package]] name = "lance-table" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "arrow", "arrow-array", @@ -4222,7 +4222,7 @@ dependencies = [ [[package]] name = "lance-tokenizer" version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=356acb0d333c96e970f6f84b97314fc5bc4193f7#356acb0d333c96e970f6f84b97314fc5bc4193f7" +source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 489a609..5b43998 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,14 +22,14 @@ crate-type = ["cdylib", "staticlib", "rlib"] # `lance_index::progress::IndexBuildProgress`; already in the graph # transitively via lance-index. async-trait = "0.1" -lance = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7", features = ["substrait"] } -lance-core = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" } -lance-file = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" } -lance-index = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" } -lance-io = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" } -lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" } -lance-table = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" } -lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7", features = ["substrait"] } +lance = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19", features = ["substrait"] } +lance-core = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } +lance-file = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } +lance-index = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } +lance-io = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } +lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } +lance-table = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } +lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19", features = ["substrait"] } datafusion = { version = "54.0.0", default-features = false } arrow = { version = "58.0.0", features = ["prettyprint", "ffi"] } arrow-array = "58.0.0" @@ -51,9 +51,9 @@ snafu = "0.9" uuid = { version = "1", features = ["v4"] } [dev-dependencies] -lance = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7", features = ["substrait"] } -lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" } -lance-file = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" } +lance = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19", features = ["substrait"] } +lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } +lance-file = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } tokio = { version = "1", features = ["rt-multi-thread", "macros"] } arrow-array = "58.0.0" arrow-schema = "58.0.0" diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs index 0524210..c02cf45 100644 --- a/tests/c_api_test.rs +++ b/tests/c_api_test.rs @@ -7717,6 +7717,124 @@ fn test_scanner_nearest_filter_postfilter() { unsafe { lance_dataset_close(ds) }; } +#[test] +fn test_scanner_nearest_full_snapshot_prefilter_statistics() { + for stable_row_ids in [false, true] { + let (_tmp, uri) = create_multi_fragment_vector_dataset(2, 32, 8, stable_row_ids); + let uri = c_str(&uri); + let dataset = unsafe { lance_dataset_open(uri.as_ptr(), ptr::null(), 0) }; + assert!(!dataset.is_null()); + let column = c_str("embedding"); + let params = LanceVectorIndexParams { + index_type: LanceVectorIndexType::IvfFlat, + metric: LanceMetricType::L2, + num_partitions: 1, + num_sub_vectors: 0, + num_bits: 0, + max_iterations: 2, + hnsw_m: 0, + hnsw_ef_construction: 0, + sample_rate: 0, + }; + assert_eq!( + unsafe { + lance_dataset_create_vector_index( + dataset, + column.as_ptr(), + ptr::null(), + ¶ms, + false, + ) + }, + 0 + ); + let mut fragments = [0_u64; 2]; + assert_eq!( + unsafe { lance_dataset_fragment_ids(dataset, fragments.as_mut_ptr()) }, + 0 + ); + + let mut reference_rows_scanned = None; + for scope in [None, Some(fragments.as_slice()), Some(&fragments[..1])] { + let scanner = unsafe { lance_scanner_new(dataset, ptr::null(), ptr::null()) }; + assert!(!scanner.is_null()); + if let Some(scope) = scope { + assert_eq!( + unsafe { lance_scanner_set_fragment_ids(scanner, scope.as_ptr(), scope.len()) }, + 0 + ); + } + let query = [60.0_f32; 8]; + assert_eq!( + unsafe { + lance_scanner_nearest( + scanner, + column.as_ptr(), + query.as_ptr().cast(), + query.len(), + LanceDataType::Float32 as i32, + 5, + ) + }, + 0 + ); + assert_eq!(unsafe { lance_scanner_set_prefilter(scanner, true) }, 0); + let mut captured = CapturedScanStatistics::default(); + assert_eq!( + unsafe { + lance_scanner_set_statistics_callback( + scanner, + Some(capture_scan_statistics), + (&mut captured as *mut CapturedScanStatistics).cast(), + ) + }, + 0 + ); + let batches = scan_all_rows_from_scanner(scanner); + let mut ids = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + ids.sort_unstable(); + assert_eq!(captured.calls, 1); + let count = |name: &str| { + captured + .metrics + .iter() + .filter(|(key, kind, _)| key == name && *kind == LanceScanMetricKind::Count) + .map(|(_, _, value)| *value) + .sum::() + }; + if scope.is_some_and(|scope| scope.len() == 1) { + assert_eq!(ids, vec![27, 28, 29, 30, 31]); + assert_eq!(count("prefilter_input_rows"), 32); + assert_eq!(count("prefilter_row_ids"), 32); + } else { + assert_eq!(ids, vec![58, 59, 60, 61, 62]); + assert_eq!(count("prefilter_input_rows"), 0); + // Equal results alone would miss a redundant scan of every row ID. + let rows_scanned = count("rows_scanned"); + if let Some(reference) = reference_rows_scanned { + assert_eq!(rows_scanned, reference); + } else { + reference_rows_scanned = Some(rows_scanned); + } + } + unsafe { lance_scanner_close(scanner) }; + } + unsafe { lance_dataset_close(dataset) }; + } +} + #[test] fn test_scanner_nearest_prefilter_with_fragment_ids_next() { let (_tmp, uri) = create_multi_fragment_vector_dataset(2, 32, 8, false);