From 770829bfc36f2a59ab7604b0dd211ec046dcdcdc Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 25 Sep 2026 20:15:42 +0300 Subject: [PATCH 01/36] feat(cli): --show-default-cparams, with a public LevelParameters --show-default-cparams was refused as unknown. It now prints, before compressing, the parameters the level selects for each input, in the reference command's layout (zstdcli.c, printDefaultCParams): name and size, then windowLog .. strategy with upstream's strategy names. It is refused when decompressing and ignored by -b, --train and -l, as there. The selection needed a public entry point: LevelParameters::for_level, the equivalent of ZSTD_getCParams, is the same port the encoder resolves a dictionary-less frame with. The bench-internals facade now goes through it, and the separate zero-means-unknown wrapper is gone. Checked against the system zstd v1.5.7: 216 cases (nine file sizes including an empty file, twelve levels from --fast=5 to -22, with and without a dictionary) print byte-identical output. Part of #128 --- zstd/src/bin/structured-zstd/main.rs | 104 +++++++++++++++++++++++++- zstd/src/bin/structured-zstd/tests.rs | 63 ++++++++++++++++ zstd/src/encoding/cparams.rs | 18 ----- zstd/src/encoding/mod.rs | 2 +- zstd/src/encoding/parameters.rs | 70 +++++++++++++++++ zstd/src/lib.rs | 6 +- 6 files changed, 240 insertions(+), 23 deletions(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 3fe8d6d0b..362f25fba 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -15,8 +15,8 @@ use std::io::{self, BufReader, ErrorKind, IsTerminal, Read, Write}; use std::path::{Path, PathBuf}; use structured_zstd::encoding::{ - CompressionContext, CompressionLevel, CompressionParameters, LiteralCompressionMode, Strategy, - StreamingEncoder, + CompressionContext, CompressionLevel, CompressionParameters, LevelParameters, + LiteralCompressionMode, Strategy, StreamingEncoder, }; /// Error type for the tool: a boxed message, which is all a command-line @@ -212,6 +212,9 @@ struct Options { /// with the window sized to the input, so the whole reference is /// reachable. patch_from: Option, + /// Print the parameters the level selects for each input before + /// compressing it (`--show-default-cparams`). + show_default_cparams: bool, /// Which dictionary trainer `--train*` runs. trainer: Trainer, /// The trainer's tuning from `--train-fastcover=...` / `--train-cover=...`. @@ -926,6 +929,7 @@ fn parse_args_into( advanced: AdvancedParams::default(), literals: LiteralCompressionMode::Auto, patch_from: None, + show_default_cparams: false, trainer: Trainer::FastCover, trainer_params: TrainerParams::default(), }; @@ -1014,6 +1018,7 @@ fn parse_args_into( "no-pass-through" => opts.pass_through = Some(false), "exclude-compressed" => opts.exclude_compressed = true, "ignore-read-errors" => opts.ignore_read_errors = true, + "show-default-cparams" => opts.show_default_cparams = true, "progress" => opts.progress = Progress::Always, "no-progress" => opts.progress = Progress::Never, "version" => { @@ -1665,6 +1670,87 @@ the trainer tuning, and -M/--memory below the enforced ceiling when decoding. A new output file keeps its source's permissions. "; +/// Upstream's names for the nine strategies, in ordinal order from 1 +/// (`zstdcli.c`, `ZSTD_strategyMap`). +const STRATEGY_NAMES: [&str; 9] = [ + "ZSTD_fast", + "ZSTD_dfast", + "ZSTD_greedy", + "ZSTD_lazy", + "ZSTD_lazy2", + "ZSTD_btlazy2", + "ZSTD_btopt", + "ZSTD_btultra", + "ZSTD_btultra2", +]; + +/// Write what `--show-default-cparams` reports for one input: the parameters +/// `level` selects for it, in the reference command's layout +/// (`zstdcli.c`, `printDefaultCParams`). +/// +/// `size` is the input's length when it has one (`None` for stdin or anything +/// not a regular file). A length of zero is printed as such but sized as an +/// unknown source, because `ZSTD_getCParams`, which the reference calls here, +/// reads zero as "unknown". +fn write_default_cparams( + out: &mut impl Write, + name: &str, + size: Option, + dictionary_size: usize, + level: i32, +) -> io::Result<()> { + match size { + Some(bytes) => writeln!(out, "{name} ({bytes} bytes)")?, + None => writeln!(out, "{name} (src size unknown)")?, + } + let params = + LevelParameters::for_level(level, size.filter(|&bytes| bytes != 0), dictionary_size); + let ordinal = params.strategy.ordinal(); + // `ordinal` is 1..=9 by construction of `Strategy`. + let strategy = STRATEGY_NAMES[ordinal as usize - 1]; + writeln!(out, " - windowLog : {}", params.window_log)?; + writeln!(out, " - chainLog : {}", params.chain_log)?; + writeln!(out, " - hashLog : {}", params.hash_log)?; + writeln!(out, " - searchLog : {}", params.search_log)?; + writeln!(out, " - minMatch : {}", params.min_match)?; + writeln!(out, " - targetLength : {}", params.target_length)?; + writeln!(out, " - strategy : {strategy} ({ordinal})") +} + +/// Print `--show-default-cparams` for every input of a compressing run, on +/// stderr and whatever the verbosity, as the reference command does. +fn show_default_cparams(opts: &Options) -> Result<()> { + let dictionary_size = match dictionary_path(opts) { + Some(path) => fs::metadata(path) + .wrap_err_with(|| format!("failed to inspect dictionary file {}", path.display()))? + .len(), + None => 0, + }; + let dictionary_size = usize::try_from(dictionary_size) + .map_err(|_| eyre!("dictionary of {dictionary_size} bytes does not fit in memory"))?; + let mut err = io::stderr().lock(); + let stdin = [PathBuf::from("-")]; + let inputs: &[PathBuf] = if opts.inputs.is_empty() { + &stdin + } else { + &opts.inputs + }; + for input in inputs { + let (name, size) = if input == Path::new("-") { + (STDIN_MARK.to_string(), None) + } else { + let size = fs::metadata(input) + .ok() + .filter(fs::Metadata::is_file) + .map(|metadata| metadata.len()); + (input.display().to_string(), size) + }; + write_default_cparams(&mut err, &name, size, dictionary_size, opts.level) + .wrap_err("failed to write the default parameters")?; + } + Ok(()) +} + /// The file the run's dictionary comes from: `-D`, or the `--patch-from` /// reference, which is a dictionary by another name. The command line refuses /// both at once, so at most one is set. @@ -2047,6 +2133,20 @@ fn run_selected(mut opts: Options) -> Result { return list_files(&opts); } + // Reached only by the streaming modes, as the reference command's check is + // (benchmark, training and listing have left by now and ignore the flag). + // Decompression has no parameters to show; testing is not decompression + // there, so it is not refused, and prints nothing since it compresses + // nothing. + if opts.show_default_cparams { + if opts.mode == Mode::Decompress { + bail!("error : can't use --show-default-cparams in decompression mode"); + } + if opts.mode == Mode::Compress { + show_default_cparams(&opts)?; + } + } + // A destination named outright belongs to the whole run, whatever it reads: // stdin, an explicit `-`, or files. The dictionary is what the frame being // written will need to be read back, so an `-o` pointing at it destroys the diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 64463e147..b14630b69 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -3352,6 +3352,69 @@ fn an_unreadable_directory_named_alone_fails_the_run() { } } +/// `--show-default-cparams` prints the reference command's layout: the name and +/// size, then one line per parameter, the strategy by upstream's name and +/// ordinal. Level 3 on an unknown-size source is the `clevels.h` row +/// `{21, 16, 17, 1, 5, 0, ZSTD_dfast}` unadjusted. +#[test] +fn show_default_cparams_prints_the_reference_layout() { + let mut out = Vec::new(); + write_default_cparams(&mut out, STDIN_MARK, None, 0, 3).unwrap(); + assert_eq!( + String::from_utf8(out).unwrap(), + "/*stdin*\\ (src size unknown)\n \ + - windowLog : 21\n \ + - chainLog : 16\n \ + - hashLog : 17\n \ + - searchLog : 1\n \ + - minMatch : 5\n \ + - targetLength : 0\n \ + - strategy : ZSTD_dfast (2)\n" + ); +} + +/// An empty file is reported with its size, zero, but sized as an unknown +/// source, since the reference reads a zero size as unknown here; a real size +/// moves the selection (level 11 on 4 KiB is the optimal parser). +#[test] +fn show_default_cparams_sizes_an_empty_file_as_unknown() { + let mut empty = Vec::new(); + write_default_cparams(&mut empty, "empty", Some(0), 0, 11).unwrap(); + let mut unknown = Vec::new(); + write_default_cparams(&mut unknown, "empty", None, 0, 11).unwrap(); + let empty = String::from_utf8(empty).unwrap(); + let unknown = String::from_utf8(unknown).unwrap(); + assert!(empty.starts_with("empty (0 bytes)\n"), "{empty}"); + assert_eq!( + empty.lines().skip(1).collect::>(), + unknown.lines().skip(1).collect::>(), + "same parameters as an unknown size" + ); + + let mut small = Vec::new(); + write_default_cparams(&mut small, "small", Some(4096), 0, 11).unwrap(); + let small = String::from_utf8(small).unwrap(); + assert!( + small.contains(" - strategy : ZSTD_btopt (7)\n"), + "{small}" + ); + assert!(small.contains(" - windowLog : 12\n"), "{small}"); +} + +/// Decompression has no parameters to show, so the flag is refused there, as +/// the reference command refuses it. +#[test] +fn show_default_cparams_is_refused_when_decompressing() { + let scratch = Scratch::new("cparamsd"); + let frame = scratch.file("f.zst", &frame_of(b"payload")); + let mut opts = parse(&["-d", "-q", "--show-default-cparams", "f"]).unwrap(); + opts.inputs = vec![frame]; + let err = run(opts) + .expect_err("decompression with --show-default-cparams is refused") + .to_string(); + assert!(err.contains("decompression mode"), "{err}"); +} + /// Decompression reports how many bytes came out, which is what `-t` and the /// summaries print; a corrupted checksum is ignored under `--no-check`. #[test] diff --git a/zstd/src/encoding/cparams.rs b/zstd/src/encoding/cparams.rs index b250ce942..e87f5df63 100644 --- a/zstd/src/encoding/cparams.rs +++ b/zstd/src/encoding/cparams.rs @@ -372,24 +372,6 @@ fn get_cparams_mode( adjust_cparams(cp, src_size_hint, dict_size, create_cdict) } -/// Public `ZSTD_getCParams` entry: maps `src_size_hint == 0` to UNKNOWN, -/// matching upstream exactly. The C-reference comparison surface (`zz_cparams` -/// validates it byte-for-byte against C `ZSTD_getCParams`); the encoder sizes -/// its own tables from [`default_cparams`] + [`create_cdict_table_logs`]. -#[cfg(feature = "bench-internals")] -pub(crate) fn get_cparams_public( - compression_level: i32, - src_size_hint: u64, - dict_size: usize, -) -> CParams { - let src = if src_size_hint == 0 { - CONTENTSIZE_UNKNOWN - } else { - src_size_hint - }; - get_cparams(compression_level, src, dict_size) -} - /// The `(hash_log, chain_log)` a dictionary's prepared match-finder tables get /// under `ZSTD_cpm_createCDict` — the single source for the CDict table /// geometry (mirrors `ZSTD_adjustCParams_internal` with an unknown source, so diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index 374eb2d22..8031c68f6 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -107,7 +107,7 @@ pub use levels::config::{ }; pub use match_generator::MatchGeneratorDriver; pub use parameters::{ - Bounds, CParameter, CompressionParameters, CompressionParametersBuilder, + Bounds, CParameter, CompressionParameters, CompressionParametersBuilder, LevelParameters, LiteralCompressionMode, ParameterError, Strategy, }; pub use streaming_encoder::{CompressionContext, StreamingEncoder}; diff --git a/zstd/src/encoding/parameters.rs b/zstd/src/encoding/parameters.rs index d32dbe3cd..cc4f7616c 100644 --- a/zstd/src/encoding/parameters.rs +++ b/zstd/src/encoding/parameters.rs @@ -242,6 +242,76 @@ impl CParameter { } } +/// The match-finder parameters a numeric level selects for a source of a given +/// size, the drop-in equivalent of C zstd's `ZSTD_getCParams`. +/// +/// The size matters: the reference's level table has a row per source-size +/// tier, and the tier changes the strategy as well as the table widths, so one +/// level can run a different match-finder on a small input than on a large +/// one. This is the selection the encoder itself makes for a frame without a +/// dictionary; knobs set through [`CompressionParameters`] override it. +/// +/// # Examples +/// +/// ``` +/// use structured_zstd::encoding::{LevelParameters, Strategy}; +/// +/// // Level 11 is a lazy2 level on a large or unknown-size source... +/// let large = LevelParameters::for_level(11, None, 0); +/// assert_eq!(large.strategy, Strategy::Lazy2); +/// +/// // ...and the optimal parser on one of 16 KiB or less, whose window is +/// // also cut down to the source. +/// let small = LevelParameters::for_level(11, Some(4096), 0); +/// assert_eq!(small.strategy, Strategy::Btopt); +/// assert_eq!(small.window_log, 12); +/// ``` +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct LevelParameters { + /// Back-reference window, `log2`. C `windowLog`. + pub window_log: u32, + /// Chain / binary-tree table size, `log2`. C `chainLog`. + pub chain_log: u32, + /// Hash table size, `log2`. C `hashLog`. + pub hash_log: u32, + /// Search attempts per position, `log2`. C `searchLog`. + pub search_log: u32, + /// Match-finder hash width, in bytes. C `minMatch`. + pub min_match: u32, + /// Length that ends a search early; on a negative level, the step size + /// minus one. C `targetLength`. + pub target_length: u32, + /// The match-finder strategy. C `strategy`. + pub strategy: Strategy, +} + +impl LevelParameters { + /// Parameters for `level` compressing a source of `source_size` bytes with + /// a `dictionary_size`-byte dictionary (`0` for none). + /// + /// `level` is on the reference's scale: `0` is the default level, levels + /// above [`CompressionLevel::MAX_LEVEL`] clamp to it, and a negative level + /// is an acceleration factor. `None` is a source of unknown size, which is + /// sized as a large one; `Some(0)` is a source that is really empty. + /// C `ZSTD_getCParams` spells unknown as `0`, so a caller porting from it + /// maps `0` to `None`. + pub fn for_level(level: i32, source_size: Option, dictionary_size: usize) -> Self { + let size = source_size.unwrap_or(crate::encoding::cparams::CONTENTSIZE_UNKNOWN); + let cp = crate::encoding::cparams::get_cparams(level, size, dictionary_size); + Self { + window_log: cp.window_log, + chain_log: cp.chain_log, + hash_log: cp.hash_log, + search_log: cp.search_log, + min_match: cp.min_match, + target_length: cp.target_length, + // Every row of the level table names one of the nine strategies. + strategy: Strategy::from_ordinal(cp.strategy) + .expect("the level table only holds strategies 1..=9"), + } + } +} + /// Error returned by [`CompressionParametersBuilder::build`] when a knob /// is set outside its [`CParameter::bounds`]. #[derive(Copy, Clone, Debug, PartialEq, Eq)] diff --git a/zstd/src/lib.rs b/zstd/src/lib.rs index d35b2256a..deefb9446 100644 --- a/zstd/src/lib.rs +++ b/zstd/src/lib.rs @@ -154,7 +154,9 @@ pub mod testing { src: u64, dict: usize, ) -> (u32, u32, u32, u32, u32, u32, u32) { - let cp = crate::encoding::cparams::get_cparams_public(level, src, dict); + // `ZSTD_getCParams` spells an unknown source size as 0. + let cp = + crate::encoding::LevelParameters::for_level(level, (src != 0).then_some(src), dict); ( cp.window_log, cp.chain_log, @@ -162,7 +164,7 @@ pub mod testing { cp.search_log, cp.min_match, cp.target_length, - cp.strategy, + cp.strategy.ordinal(), ) } From 7804b25039e4843ce46b327596744c6a109a062a Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 25 Sep 2026 21:41:43 +0300 Subject: [PATCH 02/36] fix(encoding): gate the post-split pass on the strategy The post-split block pass ran on Level(16..=22) with a window of at least 128 KiB, keyed on the level number. Upstream decides it from the effective cParams (zstd_compress.c, ZSTD_resolveBlockSplitterMode: strategy >= btopt && windowLog >= 17), so two frames were compressed worse than they should be: - a parameter set whose knobs move a lower level onto the optimal band (level 3 carrying every level-22 knob came out 2.5% larger than level 22 with the same knobs; zstd --max at the default level lost 1.8% against its own level 22 on z000033); - levels 13-15 wherever the source size or a dictionary's CDict tier selects btopt / btultra with a window of 2^17 or more. The gate now reads the strategy the frame runs. Compressed bytes against the previous build (identical at every other level on all three shapes): z000033[..200000] L13 90378 -> 85017 (upstream 85167) L14 90341 -> 85110 (upstream 83172) L15 90272 -> 85061 (upstream 82986) z000033 + dict_tests/dictionary L13 457425 -> 446171 (upstream 462835) L14 455233 -> 444172 (upstream 438399) L15 454769 -> 443924 (upstream 437877) z000033 (1 MB, no dict) unchanged at L1-L22 The pass is not free. runner1 (x86, no PMU: task-clock), prebuilt bench-profile binaries interleaved with zstd 1.5.7, three rounds of perf stat -r 5, ranges not overlapping: case before after upstream L13 z000033[..200000] 26.3-26.7 34.3-34.7 26.8-27.8 ms L15 z000033[..200000] 28.7-29.0 36.6-37.5 34.6-35.7 ms L13 1 MB + dict 117.4-119.4 149.1-150.9 99.4-102.5 ms L12 (control, never splits) 19.1-20.0 19.1-19.8 18.8-19.9 ms The matcher costs the same in both builds; the added time is the split itself (sub-block size estimates and trial Huffman / FSE table builds), which levels 16-22 already pay. That cost is its own target. Carries the regression test a_fully_specified_parameter_set_ignores_the_ base_level (fails before, passes after). The sequence-capture tool, which cannot follow a post-split frame, now also counts the frame's blocks against its matcher calls, since a level below 16 can post-split. Part of #128 --- zstd/src/encoding/frame_compressor/tests.rs | 2 +- zstd/src/encoding/levels/fastest.rs | 30 ++++++++++++------ zstd/src/encoding/sequence_capture.rs | 34 +++++++++++++------- zstd/src/tests/parameters_test.rs | 35 +++++++++++++++++++++ 4 files changed, 80 insertions(+), 21 deletions(-) diff --git a/zstd/src/encoding/frame_compressor/tests.rs b/zstd/src/encoding/frame_compressor/tests.rs index 4ccfdb2b2..f1d2227d0 100644 --- a/zstd/src/encoding/frame_compressor/tests.rs +++ b/zstd/src/encoding/frame_compressor/tests.rs @@ -1992,7 +1992,7 @@ fn frame_emit_info_decompressed_ranges_match_decoded_output() { let data = emit_info_fixture_data(); // Cover both the single-block-per-chunk path (Default) and the - // Level(16..=22) post-split path (multiple physical partitions per + // optimal-band post-split path (multiple physical partitions per // input chunk), since lsm-tree compresses at zstd:22 and post-split // is the riskiest capture site (per-partition `src_size`). for level in [ diff --git a/zstd/src/encoding/levels/fastest.rs b/zstd/src/encoding/levels/fastest.rs index 173df13b4..d79c7d7c8 100644 --- a/zstd/src/encoding/levels/fastest.rs +++ b/zstd/src/encoding/levels/fastest.rs @@ -11,6 +11,7 @@ use crate::{ compression_level_allows_raw_fast_path, }, match_generator::MatchGeneratorDriver, + strategy::StrategyTag, }, }; use alloc::vec::Vec; @@ -168,11 +169,9 @@ pub(crate) fn compress_block_encoded( // consume it: when no sink collects checksums (the common case), and when // the block is headed for the post-split helper, which emits several // physical blocks and records a checksum per partition of its own. + let post_split = post_split_enabled(state.strategy_tag, window_size); #[cfg(all(feature = "lsm", feature = "hash"))] - let post_split_path = rle_byte_opt.is_none() - && !raw_fast_path - && matches!(compression_level, CompressionLevel::Level(16..=22)) - && state.matcher.window_size() >= (1 << 17); + let post_split_path = rle_byte_opt.is_none() && !raw_fast_path && post_split; #[cfg(all(feature = "lsm", feature = "hash"))] let precomputed_checksum = block_checksums .as_ref() @@ -222,9 +221,7 @@ pub(crate) fn compress_block_encoded( } else { // Compress as a standard compressed block uncompressed_data.commit(&mut state.matcher); - if matches!(compression_level, CompressionLevel::Level(16..=22)) - && state.matcher.window_size() >= (1 << 17) - { + if post_split { // This helper may emit multiple physical blocks (compressed or raw) // into `output`; the decompressed-size and (if requested) checksum // sidecars are pushed per physical block from inside the partition @@ -363,8 +360,9 @@ pub(crate) fn compress_block_encoded( /// branch selection and shares the heavy `compress_block` machinery; the /// only differences are how the block is acquired (borrowed slice, no /// copy) and that raw/RLE bodies are emitted straight from `block`. The -/// `Level(16..=22)` post-split branch is unreachable here (the borrowed -/// path is gated to Fast levels), so it is omitted. +/// post-split branch is unreachable here (it needs an optimal-band +/// strategy, and the borrowed path is gated to the fast one), so it is +/// omitted. #[allow(clippy::too_many_arguments)] pub(crate) fn compress_block_encoded_borrowed( state: &mut CompressState, @@ -568,6 +566,20 @@ pub(crate) fn compress_block_encoded_borrowed( } } +/// Whether a compressed block goes through the post-split pass, which may cut +/// it into several blocks along its sequences. Decided by the strategy the +/// frame runs, not by its level, as upstream zstd decides it +/// (`zstd_compress.c`, `ZSTD_resolveBlockSplitterMode`: `strategy >= btopt && +/// windowLog >= 17`), so a parameter set that moves the strategy moves the +/// pass with it. +#[inline] +fn post_split_enabled(strategy_tag: StrategyTag, window_size: u64) -> bool { + matches!( + strategy_tag, + StrategyTag::BtOpt | StrategyTag::BtUltra | StrategyTag::BtUltra2 + ) && window_size >= 1 << 17 +} + /// Whether this block may go out raw without being searched. /// /// The classifier answers from the block's own bytes, which cannot see a repeat diff --git a/zstd/src/encoding/sequence_capture.rs b/zstd/src/encoding/sequence_capture.rs index 96c90978e..b27ad8ee7 100644 --- a/zstd/src/encoding/sequence_capture.rs +++ b/zstd/src/encoding/sequence_capture.rs @@ -339,7 +339,8 @@ fn compress_and_collect_sequences_impl( // maps to exactly ONE physical on-wire block, so // `CapturingMatcher::current_block` tracks correctly. // - // * Post-split (`Level(16..=22)` + window >= 1<<17, dispatched + // * Post-split (a btopt / btultra / btultra2 frame with a window of at + // least 1<<17, dispatched // from `levels/fastest.rs::compress_block_encoded` via // `compress_block_with_post_split`): a SINGLE matcher call's // output is split into multiple physical blocks by @@ -347,11 +348,10 @@ fn compress_and_collect_sequences_impl( // → N blocks → `current_block` only increments once, // `block_tail_lengths.len()` is short by `N - 1`. // - // Reject `Level(n >= 16)` only. Covers `Level(16..=22)` and - // clamped `Level(>22)` (match_generator.rs:412-415 lands on - // Level 22 params for n > 22). `Level(11..=15)` is allowed - // because pre-split produces a separate matcher call per - // physical block (PR #149 review #24 + #27 + #30). + // Reject `Level(n >= 16)` up front: every such level runs the optimal + // band on a large source, and levels above 22 clamp to 22. A lower level + // the source size or a dictionary moves onto that band is caught after + // compression, by counting the frame's blocks. let post_split = matches!(level, CompressionLevel::Level(n) if n >= 16); assert!( !post_split, @@ -460,10 +460,22 @@ fn compress_and_collect_sequences_impl( // or RLE block is present so the broken precondition surfaces // immediately instead of being misread as a real divergence // (PR #149 review #25). - let raw_or_rle = detect_raw_or_rle_blocks_in_frame(&output).expect( + let (physical_blocks, raw_or_rle) = detect_raw_or_rle_blocks_in_frame(&output).expect( "sequence_capture: failed to parse emitted frame header — refusing to \ return a possibly-misaligned capture without raw-block detection", ); + // The level guard above is a prediction; this is the fact. The post-split + // pass follows the strategy the frame runs, which the source size or a + // dictionary can raise onto the optimal band below level 16, so the frame + // itself is checked for more blocks than the matcher was asked for. + assert_eq!( + physical_blocks, + block_tail_lengths.len(), + "compress_and_collect_sequences does not support post-split levels: the \ + frame holds {physical_blocks} blocks for {} matcher calls, so the per-call \ + block counter cannot line up with it.", + block_tail_lengths.len(), + ); assert!( raw_or_rle.is_empty(), "compress_and_collect_sequences: emitted frame contains {} raw/RLE block(s) at \ @@ -480,14 +492,14 @@ fn compress_and_collect_sequences_impl( } } -/// Walk the emitted Zstandard frame and return the on-wire indices -/// of any Raw_Block or RLE_Block entries (RFC 8878 §3.1.1.2.2). The +/// Walk the emitted Zstandard frame and return its block count and the +/// on-wire indices of any Raw_Block or RLE_Block entries (RFC 8878 §3.1.1.2.2). The /// capture's matcher hook cannot observe the encoder's late /// raw-fallback decision; this parser gives us a way to fail-fast /// when that decision happens. Returns `Err` on malformed frames so /// the caller can panic with a clearer diagnostic than a silent /// short read. -fn detect_raw_or_rle_blocks_in_frame(frame: &[u8]) -> Result, &'static str> { +fn detect_raw_or_rle_blocks_in_frame(frame: &[u8]) -> Result<(usize, Vec), &'static str> { const ZSTD_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD]; if frame.len() < 6 || frame[..4] != ZSTD_MAGIC { return Err("frame missing zstd magic"); @@ -575,7 +587,7 @@ fn detect_raw_or_rle_blocks_in_frame(frame: &[u8]) -> Result, &'stati if content_checksum_flag == 1 && cursor.checked_add(4).is_none_or(|end| end > frame.len()) { return Err("truncated content checksum"); } - Ok(raw_or_rle) + Ok((block_idx, raw_or_rle)) } #[cfg(test)] diff --git a/zstd/src/tests/parameters_test.rs b/zstd/src/tests/parameters_test.rs index 60ee9b559..576da97e7 100644 --- a/zstd/src/tests/parameters_test.rs +++ b/zstd/src/tests/parameters_test.rs @@ -88,6 +88,41 @@ fn empty_override_is_byte_identical_to_level() { } } +/// Once every match-finder knob is set, the base level has nothing left to +/// decide: the frame is the same whichever level the parameters start from, as +/// it is upstream, where the block splitter follows the effective cParams +/// (`ZSTD_resolveBlockSplitterMode`: `strategy >= btopt && windowLog >= 17`). +/// A dfast base level carrying btultra2 knobs used to skip the post-split pass +/// a btultra2 level runs, because that pass was keyed on the level number. +#[test] +fn a_fully_specified_parameter_set_ignores_the_base_level() { + let data = &include_bytes!("../../decodecorpus_files/z000033")[..512 * 1024]; + let frame_from = |level: i32| { + let params = CompressionParameters::builder(CompressionLevel::Level(level)) + .window_log(20) + .chain_log(21) + .hash_log(21) + .search_log(9) + .min_match(3) + .target_length(999) + .strategy(Strategy::Btultra2) + .build() + .unwrap(); + compress_with_parameters(data, ¶ms) + }; + let native = frame_from(22); + for level in [3, 13] { + let frame = frame_from(level); + assert_eq!( + frame.len(), + native.len(), + "level {level} base with btultra2 knobs diverged from level 22", + ); + assert_eq!(frame, native); + } + assert_eq!(decode(&native), data); +} + /// Custom parameters must produce valid (decodable) frames that /// reproduce the input. #[test] From cdd2152bfd79af3419eb81fe19808c1c54f0dc58 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 25 Sep 2026 21:48:55 +0300 Subject: [PATCH 03/36] feat(cli): --max, every knob at its hardest end The reference command's --max (zstdcli.c, setMaxCompression) replaces the compression parameters wholesale: the largest window, chain, hash and search logs, the smallest minMatch, the longest targetLength, btultra2, and the long-distance matcher at its widest with its hash rate derived. It also unlocks the ultra levels and long-distance matching. A --zstd= list before it is overwritten, one after it adjusts the maximum, as there. Refused on a 32-bit target, as there. One departure: the window stops at 27, the widest this build decodes, where the reference goes to 31; a larger one would write frames this tool cannot open. z000033, against zstd 1.5.7 on the M1: input ours upstream file 426636 B, 2.06 s 426527 B, 4.31 s stdin 426632 B, 1.52 s, 11.5 GB 426527 B, 5.71 s, 17.2 GB peak Part of #128 --- zstd/src/bin/structured-zstd/main.rs | 60 ++++++++++++++++++++- zstd/src/bin/structured-zstd/tests.rs | 77 +++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 362f25fba..ba1f72702 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -457,7 +457,7 @@ fn check_window_log(log: u32) -> Result<()> { let bounds = CParameter::WindowLog.bounds(); let decodable = structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE.ilog2(); - let upper = bounds.upper_bound.min(i64::from(decodable)); + let upper = i64::from(max_window_log()); if i64::from(log) < bounds.lower_bound || i64::from(log) > upper { bail!( "window log {log} is outside the supported range {}..={upper} \ @@ -469,6 +469,50 @@ fn check_window_log(log: u32) -> Result<()> { Ok(()) } +/// The largest window this build both writes and reads back: the encoder's +/// ceiling or the decoder's, whichever is lower (see [`check_window_log`]). +fn max_window_log() -> u32 { + use structured_zstd::encoding::CParameter; + + let encodable = u32::try_from(CParameter::WindowLog.bounds().upper_bound) + .expect("the window-log bound is a small positive number"); + let decodable = structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE.ilog2(); + encodable.min(decodable) +} + +/// Every knob at the end of its range that compresses hardest, as the +/// reference command's `--max` sets them (`zstdcli.c`, `setMaxCompression`). +/// One departure: the window stops at [`max_window_log`] rather than at 31, +/// since a larger one would write frames this build refuses to decode. The +/// long-distance hash rate is left to derive from the rest, which is what the +/// reference's 0 there asks for. +fn max_compression_params() -> AdvancedParams { + use structured_zstd::encoding::CParameter; + + let upper = |parameter: CParameter| { + u32::try_from(parameter.bounds().upper_bound) + .expect("every compression-parameter bound is a small positive number") + }; + let lower = |parameter: CParameter| { + u32::try_from(parameter.bounds().lower_bound) + .expect("every compression-parameter bound is a small positive number") + }; + AdvancedParams { + window_log: Some(max_window_log()), + chain_log: Some(upper(CParameter::ChainLog)), + hash_log: Some(upper(CParameter::HashLog)), + search_log: Some(upper(CParameter::SearchLog)), + min_match: Some(lower(CParameter::MinMatch)), + target_length: Some(upper(CParameter::TargetLength)), + strategy: Some(Strategy::Btultra2), + ldm_hash_log: Some(upper(CParameter::LdmHashLog)), + // The reference's heuristic value, not a bound. + ldm_min_match: Some(16), + ldm_bucket_size_log: Some(upper(CParameter::LdmBucketSizeLog)), + ldm_hash_rate_log: None, + } +} + /// Validate the parameter list of `--adapt=min=N,max=N`. /// /// The bounds have no effect here — the level does not vary — but a command @@ -1003,6 +1047,18 @@ fn parse_args_into( "keep" => opts.keep = true, "rm" => opts.remove_source = true, "ultra" => ultra = true, + // Replaces every knob at once, as the reference command's does, + // so a `--zstd=` before it is overwritten and one after it + // adjusts the maximum. Its tables at their widest do not fit a + // 32-bit address space, which the reference refuses the same way. + "max" => { + if usize::BITS < 64 { + bail!("--max is incompatible with 32-bit mode"); + } + ultra = true; + opts.long = true; + opts.advanced = max_compression_params(); + } "quiet" => *verbosity -= 1, "verbose" => *verbosity += 1, // The wire-format switches: the checksum, the @@ -1614,6 +1670,8 @@ Advanced options: Advanced compression options: --ultra Enable levels beyond 19, up to 22; requires more memory. + --max Compress with every parameter at its maximum; the window stops at 27, + the widest this build reads back. Requires a lot of memory. --fast[=#] Use to very fast compression levels. [Default: 1] --long[=#] Enable long distance matching with window log #. [Default: 27] Available from level 16 up (or with --zstd=strat=7..9), where diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index b14630b69..577b692ab 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -4194,6 +4194,83 @@ fn advanced_parameters_reach_the_frame() { ); } +/// `--max` sets every knob to its hardest end, as the reference's +/// `setMaxCompression` does, with the window stopped where this build still +/// decodes. It unlocks the ultra levels and long-distance matching, replaces a +/// `--zstd=` list given before it, and is adjusted by one given after it. +#[test] +fn max_sets_every_knob_to_its_hardest_end() { + let opts = parse(&["--max", "f"]).unwrap(); + assert!(opts.long, "--max enables long-distance matching"); + assert_eq!( + opts.advanced, + AdvancedParams { + window_log: Some(27), + chain_log: Some(30), + hash_log: Some(30), + search_log: Some(30), + min_match: Some(3), + target_length: Some(131_072), + strategy: Some(Strategy::Btultra2), + ldm_hash_log: Some(30), + ldm_min_match: Some(16), + ldm_bucket_size_log: Some(8), + ldm_hash_rate_log: None, + } + ); + assert_eq!( + parse(&["--max", "-22", "f"]).unwrap().level, + 22, + "--max unlocks the ultra levels" + ); + assert!( + parse(&["-3", "--max", "f"]).is_ok(), + "the strategy it sets carries long-distance matching at any level" + ); + + let before = parse(&["--zstd=wlog=20,hlog=18", "--max", "f"]).unwrap(); + assert_eq!( + before.advanced, + max_compression_params(), + "an earlier list is replaced" + ); + let after = parse(&["--max", "--zstd=wlog=20", "f"]).unwrap(); + assert_eq!( + after.advanced.window_log, + Some(20), + "a later list adjusts it" + ); + assert_eq!( + after.advanced.chain_log, + Some(30), + "and leaves the rest at the maximum" + ); +} + +/// A `--max` frame over a known-size input is down-sized to the input, so it +/// compresses without the widest tables and decodes back to the input. +#[test] +fn a_max_frame_round_trips() { + let opts = parse(&["--max", "f"]).unwrap(); + // Small: at a search depth of 2^30 a debug build walks every candidate. + let payload: Vec = (0..16 * 1024u32) + .map(|i| b'a' + (i.wrapping_mul(2_654_435_761) >> 28) as u8) + .collect(); + let mut frame = Vec::new(); + compress_stream( + payload.as_slice(), + &mut frame, + &FrameSettings { + pledged_size: Some(payload.len() as u64), + ..FrameSettings::from_options(&opts) + }, + &mut no_dict(), + ) + .unwrap(); + assert!(frame.len() < payload.len(), "the payload is compressible"); + assert_eq!(decoded(&frame).unwrap(), payload); +} + /// `--long` below level 16 is refused because the matcher does not run there, /// unless `--zstd=strat=` moves the level onto a parser where it does. #[test] From 4ccdb7a6c03fe5f45bf063ffa79289d059b043de Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 25 Sep 2026 21:56:54 +0300 Subject: [PATCH 04/36] feat(cli): -B# cuts benchmark inputs into independent frames -B was parsed and dropped. The reference's benchmark (benchzstd.c, BMK_benchMemAdvancedNoAlloc) builds a block table: every input is a frame of its own, cut into -B pieces when the size is at least 32, each compressed independently. Ours compressed all inputs as one stream, so a multi-file benchmark measured a different thing and -B did nothing. - The benchmark now compresses that same block table, one frame per piece, and checks the round trip after measuring. - Frames carry no content checksum, as the reference's benchmark sets none (BMK_initCCtx); the command's --check default does not apply. - The -q header reports the block size asked for, as the reference's. - -B is read like readU32FromChar (a count with K / M); -B0 is none. - The -M accounting sizes the frames buffer as the bound of each piece and the encoder by the widest piece. z000033, -b1 -e5 -B16K -i1, M1, bytes (ours / zstd 1.5.7): L1 512442 / 512274, L2 497291 / 497176, L3 507883 / 491220, L4 476138 / 476388, L5 475162 / 475423. Part of #128 --- zstd/src/bin/structured-zstd/main.rs | 209 +++++++++++++++++++------- zstd/src/bin/structured-zstd/tests.rs | 49 ++++++ 2 files changed, 204 insertions(+), 54 deletions(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index ba1f72702..70d267ff7 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -142,6 +142,10 @@ struct Options { /// Measure each input on its own (`-S`) instead of as one stream, so the /// reported ratio and throughput describe a file rather than a mixture. bench_separately: bool, + /// `-B#`: the benchmark cuts every input into independent frames of this + /// many bytes (below [`MIN_BENCH_BLOCK_SIZE`] it cuts nothing, as the + /// reference's benchmark ignores such a size). + block_size: Option, /// Long-distance matching (`--long`), enabled on the encoder via the /// compression-parameters API. long: bool, @@ -285,6 +289,11 @@ const DEFAULT_MAX_DICT: usize = 112_640; /// Least time `-b` measures each level for (upstream `BMK_TIMETEST_DEFAULT_S`). const DEFAULT_BENCH_SECONDS: f64 = 3.0; +/// Smallest `-B` block the benchmark cuts its inputs into; a smaller one leaves +/// each input whole (`benchzstd.c`, `BMK_benchMemAdvancedNoAlloc`: +/// `adv->blockSize >= 32`). +const MIN_BENCH_BLOCK_SIZE: u64 = 32; + /// Window log a bare `--long` selects, as upstream documents (128 MiB). const DEFAULT_LONG_WINDOW_LOG: u32 = 27; @@ -945,6 +954,7 @@ fn parse_args_into( bench_end: default_level, bench_secs: DEFAULT_BENCH_SECONDS, bench_separately: false, + block_size: None, long: false, long_window_log: None, memory_limit: None, @@ -1312,23 +1322,30 @@ fn parse_args_into( 'v' => *verbosity += 1, 'C' => opts.checksum = true, 'r' => opts.recursive = true, - 'B' | 'T' => { - // `-B[N]` job / block size, `-T[N]` thread count. Both - // steer how the work is done, not what comes out: we use a - // fixed block size and run single-threaded. Upstream - // accepts them, so a script that passes them must not fail - // here — but the VALUE is still parsed: ignoring what a - // flag does is not a reason to ignore what it says, and a - // typo is a broken command line either way. A size takes a - // size suffix; a thread count is a plain count, the way - // `--threads=` reads it. + 'B' => { + // `-B[N]` cuts the benchmark's inputs into independent + // frames and a training sample into several samples. When + // compressing it is the job size of a multi-threaded run, + // which this build does not have, so it is kept and has no + // effect there. Read as the reference reads it + // (`readU32FromChar`): a count with an optional `K` / `M`. + let rest: String = chars[ci + 1..].iter().collect(); + let (size, tail) = read_leading_u32(&rest).wrap_err("invalid -B value")?; + if !tail.is_empty() { + bail!("invalid -B value `{rest}`"); + } + opts.block_size = (size != 0).then_some(u64::from(size)); + ci = chars.len(); + continue; + } + 'T' => { + // `-T[N]` thread count: single-threaded here, so it steers + // nothing, but the value is still parsed the way + // `--threads=` reads it, since a typo is a broken command + // line either way. let rest: String = chars[ci + 1..].iter().collect(); if !rest.is_empty() { - if c == 'B' { - parse_size(&rest).wrap_err("invalid -B value")?; - } else { - rest.parse::().wrap_err("invalid -T thread count")?; - } + rest.parse::().wrap_err("invalid -T thread count")?; } ci = chars.len(); continue; @@ -1681,6 +1698,7 @@ Advanced compression options: --zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#[,lhlog=#,lmml=#,lblog=#,lhrlog=#] Override the level's compression parameters knob by knob. --exclude-compressed Only compress files that are not already compressed. + --show-default-cparams Print the parameters the level selects for each input. --stream-size=# Specify size of streaming input from STDIN. --size-hint=# Optimize compression parameters for streaming input of approximately size #. @@ -1712,13 +1730,15 @@ Benchmark options: -b# Perform benchmarking with compression level #. [Default: 3] -e# Test all compression levels up to #; starting level is `-b#`. [Default: 1] -i# Set the minimum evaluation to time # seconds. [Default: 3] + -B# Cut file into independent chunks of size #. [Default: No chunking] -S Output one benchmark result per input file. [Default: Consolidated result] -D dictionary Benchmark using dictionary Environment: ZSTD_CLEVEL sets the default compression level; ZSTD_NBTHREADS is read and validated. Accepted for compatibility, with no effect here: -T#/--threads=#, --single-thread, ---auto-threads, -B#, --block-size=#, --adapt, --zstd=ovlog=#, --[no-]sparse, +--auto-threads, -B# and --block-size=# when compressing (the job size of a +multi-threaded run), --adapt, --zstd=ovlog=#, --[no-]sparse, --[no-]asyncio, --[no-]mmap-dict, --[no-]row-match-finder (compression runs single-threaded). @@ -2733,17 +2753,27 @@ fn run_benchmark(opts: &Options, dict: Option>) -> Result<()> { if let Some(limit) = opts.memory_limit { // With `-S` only one input is in memory at a time, so the largest file // is what has to fit rather than their sum. + let subject: &[u64] = if opts.bench_separately { + let at = sizes + .iter() + .position(|&size| size == largest) + .expect("the largest size is one of the sizes"); + &sizes[at..=at] + } else { + &sizes + }; let inputs = if opts.bench_separately { largest } else { sum }; - // Three buffers exist at once: the input, the frame it compresses to, + let chunks = bench_chunk_lengths(subject, opts.block_size); + // Three buffers exist at once: the input, the frames it compresses to, // and the decoded copy. Each is allocated at the size named here and // never grows past it, so this is what the run actually holds rather - // than a lower bound on it. The frame's is `compress_bound`, which is - // the input plus the framing an incompressible input still pays — the - // case a ceiling has to survive. - let frame = usize::try_from(inputs) - .map(structured_zstd::encoding::compress_bound) - .map(|bound| bound as u64) - .ok(); + // than a lower bound on it. The frames' is `compress_bound` of each, + // which is the input plus the framing an incompressible input still + // pays — the case a ceiling has to survive. + let frame = bench_frames_bound(&chunks); + // The encoder is sized by the frame it builds, and the frames are the + // chunks, so the widest chunk is the source it is weighed against. + let widest_chunk = chunks.iter().copied().max().unwrap_or(0); // Beside them stands the match finder every compression pass builds, // whose tables are the largest thing at the higher levels — hundreds of // MiB where the buffers are tens. It is sized by the level, by the @@ -2779,13 +2809,13 @@ fn run_benchmark(opts: &Options, dict: Option>) -> Result<()> { Some(parameters) => { structured_zstd::encoding::estimated_compression_workspace_bytes_for_parameters( ¶meters, - Some(inputs), + Some(widest_chunk), dictionary, ) } None => structured_zstd::encoding::estimated_compression_workspace_bytes_for_run( compression_level, - Some(inputs), + Some(widest_chunk), None, false, dictionary, @@ -2828,7 +2858,13 @@ fn run_benchmark(opts: &Options, dict: Option>) -> Result<()> { for (input, size) in opts.inputs.iter().zip(&sizes) { let data = read_inputs_bounded(std::slice::from_ref(input), std::slice::from_ref(size))?; - benchmark_one(opts, codecs, &input.display().to_string(), &data)?; + benchmark_one( + opts, + codecs, + &input.display().to_string(), + &data, + std::slice::from_ref(size), + )?; } return Ok(()); } @@ -2840,7 +2876,7 @@ fn run_benchmark(opts: &Options, dict: Option>) -> Result<()> { [only] => only.display().to_string(), many => format!(" {} files", many.len()), }; - benchmark_one(opts, codecs, &label, &data) + benchmark_one(opts, codecs, &label, &data, &sizes) } /// Read every input into one buffer, taking no more room — and no more bytes — @@ -2887,10 +2923,46 @@ fn read_inputs_bounded(inputs: &[PathBuf], sizes: &[u64]) -> Result> { Ok(data) } -/// Measure one benchmark subject: the whole input as one stream, or a single -/// file under `-S`. Split out so the two modes differ only in what they hand -/// over, not in how the measurement is taken. -fn benchmark_one(opts: &Options, codecs: &mut Codecs, label: &str, data: &[u8]) -> Result<()> { +/// The lengths of the independent frames a benchmark compresses its inputs as, +/// in order: every input a frame of its own, cut into `block_size` pieces when +/// one of at least [`MIN_BENCH_BLOCK_SIZE`] is given. An empty input yields no +/// frame. This is the reference's block table (`benchzstd.c`, +/// `BMK_benchMemAdvancedNoAlloc`), so ratio and speed describe the same frames. +fn bench_chunk_lengths(file_sizes: &[u64], block_size: Option) -> Vec { + let block = block_size.filter(|&size| size >= MIN_BENCH_BLOCK_SIZE); + let mut chunks = Vec::new(); + for &size in file_sizes { + let piece = block.unwrap_or(size.max(1)); + let mut left = size; + while left > 0 { + let take = left.min(piece); + chunks.push(take); + left -= take; + } + } + chunks +} + +/// The room the frames of `chunks` can take at most: `compress_bound` of each. +/// `None` when that is more than this machine can address. +fn bench_frames_bound(chunks: &[u64]) -> Option { + chunks.iter().try_fold(0u64, |total, &chunk| { + let bound = structured_zstd::encoding::compress_bound(usize::try_from(chunk).ok()?); + total.checked_add(u64::try_from(bound).ok()?) + }) +} + +/// Measure one benchmark subject: every input together, or a single file under +/// `-S`. Split out so the two modes differ only in what they hand over, not in +/// how the measurement is taken. `file_sizes` are the lengths of the inputs +/// `data` holds, in order; each is compressed as frames of its own. +fn benchmark_one( + opts: &Options, + codecs: &mut Codecs, + label: &str, + data: &[u8], + file_sizes: &[u64], +) -> Result<()> { use std::time::Instant; if data.is_empty() { @@ -2908,42 +2980,66 @@ fn benchmark_one(opts: &Options, codecs: &mut Codecs, label: &str, data: &[u8]) ); if opts.verbosity == 1 { // The reference command's machine-readable header, for scripts that - // drive `-b -q`. + // drive `-b -q`; the block size is the one asked for, as it prints it. println!( - "bench {UPSTREAM_VERSION} : input {} bytes, {} seconds, 0 KB blocks", + "bench {UPSTREAM_VERSION} : input {} bytes, {} seconds, {} KB blocks", data.len(), - opts.bench_secs as u64 + opts.bench_secs as u64, + opts.block_size.unwrap_or(0) >> 10 ); } + let chunks = bench_chunk_lengths(file_sizes, opts.block_size); + debug_assert_eq!( + chunks.iter().sum::(), + data.len() as u64, + "the frames cover the input exactly" + ); // The two buffers the measurement fills, sized once from what they will - // hold: the frame can be no larger than `compress_bound` says, and the - // decoded copy is exactly the input again. That keeps them the size the `-M` - // ceiling counted them at instead of the doubled capacity a growing `Vec` - // ends up with — and it keeps the growth out of the timed sections, which - // would otherwise be reported as compression and decompression speed. - let mut compressed = Vec::with_capacity(structured_zstd::encoding::compress_bound(data.len())); + // hold: the frames can be no larger than `compress_bound` of each says, + // and the decoded copy is exactly the input again. That keeps them the size + // the `-M` ceiling counted them at instead of the doubled capacity a + // growing `Vec` ends up with — and it keeps the growth out of the timed + // sections, which would otherwise be reported as compression and + // decompression speed. + let frames_bound = bench_frames_bound(&chunks) + .and_then(|bound| usize::try_from(bound).ok()) + .ok_or_else(|| eyre!("-b: {label} is more than this machine can hold compressed"))?; + let mut compressed = Vec::with_capacity(frames_bound); let mut decoded = Vec::with_capacity(data.len()); for level in opts.bench_start..=opts.bench_end { validate_level(level)?; + let settings = FrameSettings { + level, + size_hint: None, + // The reference's benchmark compresses with the library's frame + // defaults (`BMK_initCCtx` sets no checksum), not the command's, so + // its sizes and decoding speeds carry no content checksum. + checksum: false, + ..FrameSettings::from_options(opts) + }; let mut best_compress = f64::MAX; let start = Instant::now(); loop { compressed.clear(); let t = Instant::now(); - compress_stream( - data, - &mut compressed, - &FrameSettings { - level, - // The benchmark holds the whole input, so the length is - // exact and there is no estimate to fall back on. - pledged_size: Some(data.len() as u64), - size_hint: None, - ..FrameSettings::from_options(opts) - }, - codecs, - )?; + let mut rest = data; + for &chunk in &chunks { + // Each piece is a frame of its own, as the reference's + // benchmark compresses every block independently. The length is + // exact, so it is pledged rather than estimated. + let (piece, tail) = rest.split_at(chunk as usize); + rest = tail; + compress_stream( + piece, + &mut compressed, + &FrameSettings { + pledged_size: Some(chunk), + ..settings + }, + codecs, + )?; + } best_compress = best_compress.min(t.elapsed().as_secs_f64()); if start.elapsed().as_secs_f64() >= opts.bench_secs { break; @@ -2966,6 +3062,11 @@ fn benchmark_one(opts: &Options, codecs: &mut Codecs, label: &str, data: &[u8]) break; } } + // A speed measured on output that is not the input measures nothing; + // the reference's benchmark checks the round trip the same way. + if decoded != data { + bail!("-b: level {level} did not decode {label} back to its input"); + } let c_speed = if best_compress > 0.0 { mb / best_compress diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 577b692ab..a5ef1c916 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -1879,6 +1879,55 @@ fn benchmark_flags_parse_level_range() { assert_eq!(opts.bench_end, opts.bench_start); } +/// `-B#` is read the way the reference reads it (a count with `K` / `M`), and +/// zero is no block size at all. +#[test] +fn block_size_is_read_like_the_reference_reads_it() { + assert_eq!( + parse(&["-b", "-B64K", "f"]).unwrap().block_size, + Some(64 << 10) + ); + assert_eq!( + parse(&["-b", "-B1MiB", "f"]).unwrap().block_size, + Some(1 << 20) + ); + assert_eq!( + parse(&["-b", "-B4096", "f"]).unwrap().block_size, + Some(4096) + ); + assert_eq!(parse(&["-b", "-B0", "f"]).unwrap().block_size, None); + assert_eq!(parse(&["-b", "-B", "f"]).unwrap().block_size, None); + assert!(parse(&["-b", "-B1G", "f"]).is_err(), "no G multiplier"); + assert!(parse(&["-b", "-Bx", "f"]).is_err()); +} + +/// The benchmark compresses every input as frames of its own, cut into `-B` +/// pieces from 32 bytes up, as the reference's block table does; an empty +/// input yields no frame, and a smaller `-B` cuts nothing. +#[test] +fn benchmark_frames_follow_the_inputs_and_the_block_size() { + assert_eq!(bench_chunk_lengths(&[100, 50], None), vec![100, 50]); + assert_eq!( + bench_chunk_lengths(&[100, 50], Some(40)), + vec![40, 40, 20, 40, 10] + ); + assert_eq!( + bench_chunk_lengths(&[100, 0, 50], Some(64)), + vec![64, 36, 50] + ); + assert_eq!( + bench_chunk_lengths(&[100], Some(31)), + vec![100], + "below 32 is ignored" + ); + assert_eq!(bench_chunk_lengths(&[100], Some(32)), vec![32, 32, 32, 4]); + assert_eq!( + bench_frames_bound(&[40, 40]), + Some(2 * structured_zstd::encoding::compress_bound(40) as u64), + "each frame pays its own framing" + ); +} + #[test] fn dash_is_a_stdin_input() { let opts = parse(&["-d", "-"]).unwrap(); From be020b22e096fb201e2fde953899ecdcae4d2511 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 25 Sep 2026 22:18:42 +0300 Subject: [PATCH 05/36] feat(dictionary): the legacy trainer, and --train-legacy --train-legacy was refused: the reference's original trainer (zdict.c, ZDICT_trainFromBuffer_legacy) had no counterpart here. Library (dict-builder): - dictionary::suffix_array: SA-IS suffix-array construction over u32 positions, alloc only. A suffix array is unique for its text, so it is the array the reference gets from divsufsort. - dictionary::legacy: the trainer itself, following zdict.c: the per-position suffix neighbourhood walk (ZDICT_analyzePos), candidate merging (ZDICT_tryMerge / insertDictItem), selectivity as samples >> s with a floor of 4, the 2000 MiB corpus cap applied a whole sample at a time, and the best-last content layout. - create_legacy_dict_from_slice(samples, sample_sizes, ...) trains and finalizes. It takes the sample sizes because the trainer counts them. CLI: - --train-legacy, --train-legacy=s=# / selectivity=#, and -s#. - The legacy trainer's samples are loaded as dibio.c loads them: files in DiB_shuffle order, each one sample of at most 128 KiB, or cut whole into -B# samples; empty files left out; fewer than five refused; at most 2 GiB, or -M when smaller. Verified against libzstd 1.5.7: - ffi-bench/tests/legacy_trainer_ffi.rs: log lines, the dict_tests unit files and z000033 in 4 KiB samples, selectivity 0/4/9/12, sizes 4 KiB, 16 KiB and 112640 B: the content equals the reference dictionary's, byte for byte, in all 36 cases (only the entropy header differs). - CLI against zstd 1.5.7 on the same file list: identical content with and without -B#, -s#, =s=# and --maxdict. With -r the two walk a directory in different orders (readdir versus sorted), so the shuffle starts from a different list. Training the 16 MB decodecorpus set with -B4096 takes 2.05 s against the reference's 1.24 s on the M1. Part of #128 --- README.md | 25 +- ffi-bench/Cargo.toml | 5 + ffi-bench/tests/legacy_trainer_ffi.rs | 134 ++++++ zstd/src/bin/structured-zstd/main.rs | 299 ++++++++++-- zstd/src/bin/structured-zstd/tests.rs | 117 ++++- zstd/src/dictionary/legacy.rs | 548 ++++++++++++++++++++++ zstd/src/dictionary/legacy/tests.rs | 96 ++++ zstd/src/dictionary/mod.rs | 87 ++++ zstd/src/dictionary/suffix_array.rs | 207 ++++++++ zstd/src/dictionary/suffix_array/tests.rs | 57 +++ zstd/src/encoding/sequence_capture.rs | 12 +- zstd/src/lib.rs | 14 + 12 files changed, 1536 insertions(+), 65 deletions(-) create mode 100644 ffi-bench/tests/legacy_trainer_ffi.rs create mode 100644 zstd/src/dictionary/legacy.rs create mode 100644 zstd/src/dictionary/legacy/tests.rs create mode 100644 zstd/src/dictionary/suffix_array.rs create mode 100644 zstd/src/dictionary/suffix_array/tests.rs diff --git a/README.md b/README.md index ac4c9e734..38c536436 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ - **Production-grade decoder** — complete [RFC 8878](https://www.rfc-editor.org/rfc/rfc8878) implementation: dictionary-backed streams, raw / RLE / compressed blocks, the full frame format, optional content checksums, runtime-dispatched SIMD kernels (SSE2 / BMI2 / AVX2 / NEON, opt-in AVX-512). - **Full-range encoder** — every C-zstd level (`-131072..=22`) produces valid frames decodable by this crate and by upstream C zstd; named presets, per-knob parameter overrides, long-distance matching, streaming via `std::io::Write`. -- **Dictionaries end to end** — compress and decompress with the same dictionary format C zstd consumes; reusable parsed handles; pure-Rust COVER / FastCOVER training behind the `dict-builder` feature. +- **Dictionaries end to end** — compress and decompress with the same dictionary format C zstd consumes; reusable parsed handles; pure-Rust COVER / FastCOVER / legacy training behind the `dict-builder` feature. - **Wire-compatible both ways** — frames interoperate with C zstd in either direction; interop is enforced in CI against the reference implementation. - **`no_std` ready** — the decoder builds with `--no-default-features` for embedded and sandboxed targets. - **WebAssembly / npm** — the same codec as an npm package with automatic SIMD selection; no native addons, no postinstall scripts. @@ -88,9 +88,15 @@ sound archive. `--exclude-compressed` skips inputs whose extension names an already-compressed format. -Flags that only steer how the work is done (`-T`, `-B`, `--adapt`, ...) are -accepted and ignored; their values are still validated, so a typo is an error -rather than silence. +Flags that only steer how the work is done (`-T`, `--adapt`, `-B` when +compressing, ...) are accepted and ignored; their values are still validated, +so a typo is an error rather than silence. `-b` measures every input as frames +of its own, cut into `-B#` pieces when a size is given, as upstream's +benchmark does. +`--max` sets every compression parameter to its hardest end, as upstream's +does, with the window stopped at 27 (the widest this build decodes), and +`--show-default-cparams` prints what the level selects for each input in +upstream's layout. `--target-compressed-block-size` does take effect: it bounds what goes into a block, so blocks flush sooner. `--long` means `--long=27`, as upstream documents, and is capped there: a larger window would produce frames this @@ -113,8 +119,10 @@ with FastCOVER, the algorithm upstream also defaults to (a knob set to zero keeps its default, as upstream reads it), and a bare `--train-cover` trains with the COVER trainer. Its tuning, `--train-cover=...`, is refused rather than misread: the reference-side parameters name knobs this trainer does not -have. `--train-legacy` names an algorithm this build does not have and is -refused. `-D` takes either a dictionary produced by `--train` or any file at +have. `--train-legacy[=s=#]` (or `-s#`) runs upstream's original trainer, which +counts samples: they are loaded as upstream loads them (each file one sample of +up to 128 KiB, or cut into `-B#` pieces), and for the same file list the +dictionary carries the same content as upstream's. `-D` takes either a dictionary produced by `--train` or any file at all, which is then used as raw content the way upstream does; such a dictionary has no ID, so the same bytes must be supplied when decoding. @@ -248,6 +256,9 @@ in pure Rust: - COVER (`create_raw_dict_from_source`) and FastCOVER (`create_fastcover_raw_dict_from_source`) raw dictionaries - `finalize_raw_dict` to produce the full zstd dictionary format - `create_fastcover_dict_from_source` for train + finalize in one call +- `create_legacy_dict_from_slice`: upstream's original suffix-array trainer + (`ZDICT_trainFromBuffer_legacy`), whose content matches upstream's byte for + byte on the same samples ## Feature flags @@ -261,7 +272,7 @@ in pure Rust: | `kernel-simd128` | ✅ | WebAssembly SIMD kernel (needs `-C target-feature=+simd128`) | | `kernel-vbmi2` | ❌ | AVX-512 decode kernel (see note below) | | `kernel-scalar` | ✅ | Marker for the always-compiled scalar fallback | -| `dict-builder` | ❌ | Pure-Rust COVER / FastCOVER dictionary training | +| `dict-builder` | ❌ | Pure-Rust COVER / FastCOVER / legacy dictionary training | | `lsm` | ❌ | [Storage-format extensions](#storage-format-extensions) | Each flag gates its tier wherever that tier exists. `kernel-sse`, diff --git a/ffi-bench/Cargo.toml b/ffi-bench/Cargo.toml index 760683e44..8f5107b43 100644 --- a/ffi-bench/Cargo.toml +++ b/ffi-bench/Cargo.toml @@ -135,6 +135,11 @@ name = "dictionary_ffi" path = "tests/dictionary_ffi.rs" required-features = ["bench-internals"] +[[test]] +name = "legacy_trainer_ffi" +path = "tests/legacy_trainer_ffi.rs" +required-features = ["bench-internals"] + [[test]] name = "encode_corpus_ffi" path = "tests/encode_corpus_ffi.rs" diff --git a/ffi-bench/tests/legacy_trainer_ffi.rs b/ffi-bench/tests/legacy_trainer_ffi.rs new file mode 100644 index 000000000..1d9af99a2 --- /dev/null +++ b/ffi-bench/tests/legacy_trainer_ffi.rs @@ -0,0 +1,134 @@ +//! The legacy trainer against the reference's `ZDICT_trainFromBuffer_legacy`: +//! both walk the same suffix array with the same selection rules, so the +//! content they choose has to be the same bytes. Only the header in front of it +//! differs, since the entropy tables are built by each side's own finalizer. +#![cfg(all(feature = "bench-internals", feature = "dict-builder"))] + +use structured_zstd::testing::legacy_dict_content; +use zstd::zstd_safe::zstd_sys; + +/// Every sample back to back, and the length of each. +type Samples = (Vec, Vec); + +/// The reference's dictionary content for the same corpus: its dictionary with +/// the header (magic, id, entropy tables, repeat offsets) cut off. +fn reference_content( + samples: &[u8], + sizes: &[usize], + dict_size: usize, + selectivity: u32, +) -> Option> { + let mut dict = vec![0u8; dict_size]; + let params = zstd_sys::ZDICT_legacy_params_t { + selectivityLevel: selectivity, + zParams: zstd_sys::ZDICT_params_t { + compressionLevel: 0, + notificationLevel: 0, + dictID: 0, + }, + }; + // SAFETY: every buffer is valid for the length passed with it, and + // `sizes` holds `sizes.len()` entries summing to `samples.len()`. + let written = unsafe { + zstd_sys::ZDICT_trainFromBuffer_legacy( + dict.as_mut_ptr().cast(), + dict.len(), + samples.as_ptr().cast(), + sizes.as_ptr(), + sizes.len() as u32, + params, + ) + }; + // SAFETY: plain query on a return code. + if written == 0 || unsafe { zstd_sys::ZDICT_isError(written) } != 0 { + return None; + } + dict.truncate(written); + // SAFETY: `dict` holds `written` bytes of the dictionary just built. + let header = unsafe { zstd_sys::ZDICT_getDictHeaderSize(dict.as_ptr().cast(), dict.len()) }; + // SAFETY: plain query on a return code. + assert_eq!(unsafe { zstd_sys::ZDICT_isError(header) }, 0); + Some(dict[header..].to_vec()) +} + +/// Log lines of a few shapes, one sample each. +fn log_lines(count: u32) -> Samples { + const SHAPES: [&str; 4] = [ + "ts={i} level=INFO msg=\"flush memtable\" tenant=demo table=orders region=eu-west\n", + "ts={i} level=WARN msg=\"slow compaction\" tenant=demo table=users region=us-east\n", + "ts={i} level=INFO msg=\"rotate segment\" tenant=acme table=orders region=eu-west\n", + "ts={i} level=ERROR msg=\"write stalled\" tenant=acme table=events region=ap-south\n", + ]; + let mut samples = Vec::new(); + let mut sizes = Vec::new(); + for i in 0..count { + let line = SHAPES[(i % 4) as usize].replace("{i}", &format!("{:08}", i * 7919)); + sizes.push(line.len()); + samples.extend_from_slice(line.as_bytes()); + } + (samples, sizes) +} + +/// The systemd unit files under `dict_tests/files`, one sample each. +fn unit_files() -> Samples { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../zstd/dict_tests/files"); + let mut names: Vec<_> = std::fs::read_dir(dir) + .expect("the fixture directory exists") + .map(|entry| entry.expect("readable entry").path()) + .filter(|path| path.extension().is_none_or(|ext| ext != "zst")) + .collect(); + names.sort(); + let mut samples = Vec::new(); + let mut sizes = Vec::new(); + for name in names { + let bytes = std::fs::read(&name).expect("readable fixture"); + if bytes.is_empty() { + continue; + } + sizes.push(bytes.len()); + samples.extend_from_slice(&bytes); + } + (samples, sizes) +} + +/// The decodecorpus file cut into 4 KiB samples, as `--train -B4096` cuts it. +fn corpus_blocks() -> Samples { + let bytes = include_bytes!("../../zstd/decodecorpus_files/z000033").to_vec(); + let sizes = bytes.chunks(4096).map(<[u8]>::len).collect(); + (bytes, sizes) +} + +/// Same corpus, same selectivity, same size: the same content, byte for byte. +#[test] +fn the_legacy_trainer_selects_the_references_content() { + let fixtures: [(&str, Samples); 3] = [ + ("log lines", log_lines(3000)), + ("unit files", unit_files()), + ("decodecorpus blocks", corpus_blocks()), + ]; + for (name, (samples, sizes)) in &fixtures { + for selectivity in [0u32, 4, 9, 12] { + for dict_size in [4096usize, 16 * 1024, 112_640] { + let ours = legacy_dict_content(samples, sizes, dict_size, selectivity); + let theirs = reference_content(samples, sizes, dict_size, selectivity); + let (Some(ours), Some(theirs)) = (ours, theirs) else { + panic!("{name} s={selectivity} size={dict_size}: one side trained nothing"); + }; + // The reference places its header in front of the content and + // lets it overwrite the front when both do not fit, so its + // content is ours or a tail of it. + assert!( + theirs.len() <= ours.len() && ours.ends_with(&theirs), + "{name} s={selectivity} size={dict_size}: {} content bytes against the \ + reference's {}", + ours.len(), + theirs.len(), + ); + assert!( + theirs.len() * 10 >= ours.len() * 9, + "{name} s={selectivity} size={dict_size}: the header should cost a sliver" + ); + } + } + } +} diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 70d267ff7..162bf5ede 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -223,6 +223,9 @@ struct Options { trainer: Trainer, /// The trainer's tuning from `--train-fastcover=...` / `--train-cover=...`. trainer_params: TrainerParams, + /// The legacy trainer's selectivity from `-s#` or `--train-legacy=s=#`; + /// zero is its default. + selectivity: u32, } /// The dictionary trainers `--train` selects between. @@ -232,6 +235,8 @@ enum Trainer { FastCover, /// `--train-cover`: the segment-scoring COVER trainer. Cover, + /// `--train-legacy`: the reference's original suffix-array trainer. + Legacy, } /// Tuning from `--train-fastcover=k=#,d=#,f=#,steps=#,split=#,accel=#` and @@ -986,6 +991,7 @@ fn parse_args_into( show_default_cparams: false, trainer: Trainer::FastCover, trainer_params: TrainerParams::default(), + selectivity: 0, }; let mut ultra = false; // `-e`, when typed. Without it the benchmark ends where it starts, so no @@ -1033,11 +1039,9 @@ fn parse_args_into( opts.trainer = Trainer::Cover; opts.trainer_params = TrainerParams::default(); } - // The legacy trainer produces a different dictionary. Accepting - // the flag and running another trainer would hand back one the - // caller did not ask for, with nothing to say so. "train-legacy" => { - bail!("--{long} is not implemented; --train-cover and --train-fastcover are") + select_mode(&mut opts, Mode::Train); + opts.trainer = Trainer::Legacy; } // `-c` and `-o` name competing destinations, so each clears the // other and the later one on the command line wins, as upstream @@ -1227,6 +1231,23 @@ fn parse_args_into( select_mode(&mut opts, Mode::Train); opts.trainer = Trainer::FastCover; opts.trainer_params = parse_trainer_params(v, true)?; + } else if let Some(v) = long.strip_prefix("train-legacy=") { + // `s=#` or `selectivity=#`, as `parseLegacyParameters` + // reads it. + select_mode(&mut opts, Mode::Train); + opts.trainer = Trainer::Legacy; + let value = v + .strip_prefix("selectivity=") + .or_else(|| v.strip_prefix("s=")) + .ok_or_else(|| { + eyre!("--train-legacy takes `s=#` or `selectivity=#`, got `{v}`") + })?; + let (selectivity, tail) = read_leading_u32(value) + .wrap_err("invalid --train-legacy selectivity")?; + if !tail.is_empty() { + bail!("invalid --train-legacy selectivity `{value}`"); + } + opts.selectivity = selectivity; } else if let Some(reference) = option_value(long, "patch-from", arg_os, &mut iter)? { @@ -1318,6 +1339,19 @@ fn parse_args_into( 'k' => opts.keep = true, // `-S` measures each input on its own. 'S' => opts.bench_separately = true, + 's' => { + // `-s#`: the legacy trainer's selectivity, read like the + // reference reads it (`readU32FromChar`). + let rest: String = chars[ci + 1..].iter().collect(); + let (selectivity, tail) = + read_leading_u32(&rest).wrap_err("invalid -s selectivity")?; + if !tail.is_empty() { + bail!("invalid -s selectivity `{rest}`"); + } + opts.selectivity = selectivity; + ci = chars.len(); + continue; + } 'q' => *verbosity -= 1, 'v' => *verbosity += 1, 'C' => opts.checksum = true, @@ -1722,6 +1756,10 @@ Dictionary builder: --train-cover Use the cover algorithm (takes no tuning here). --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] Use the fast cover algorithm (with optional arguments). + + --train-legacy[=s=#] Use the legacy algorithm with selectivity #. [Default: 9] + -B# With --train-legacy, cut each file into samples of size #; + otherwise each file is one sample of up to 128 KiB. -o NAME Use NAME as dictionary name. [Default: dictionary] --maxdict=# Limit dictionary to specified size #. [Default: 112640] --dictID=# Force dictionary ID to #. [Default: Random] @@ -1743,8 +1781,9 @@ multi-threaded run), --adapt, --zstd=ovlog=#, --[no-]sparse, single-threaded). Rejected rather than ignored, because they would change the result: --format= -other than zstd, --rsyncable (needs worker threads), --train-legacy, shrink in -the trainer tuning, and -M/--memory below the enforced ceiling when decoding. +other than zstd, --rsyncable (needs worker threads), shrink in the trainer +tuning, and -M/--memory below the enforced ceiling when decoding. +--train-cover and --train-fastcover read whole files, so -B does not cut them. A new output file keeps its source's permissions. "; @@ -3158,13 +3197,14 @@ fn bench_display_name(label: &str) -> String { } } -/// `--train`: build a FastCOVER dictionary from the concatenated sample files -/// and write it to `-o` (default `dictionary`). Mirrors upstream -/// `zstd --train FILEs -o dict --maxdict=N [--dictID=N]`. +/// `--train`: build a dictionary from the sample files with the selected +/// trainer (FastCOVER, COVER or legacy) and write it to `-o` (default +/// `dictionary`). Mirrors upstream `zstd --train FILEs -o dict --maxdict=N +/// [--dictID=N]`. fn train_dictionary(opts: &Options) -> Result<()> { use structured_zstd::dictionary::{ - FinalizeOptions, create_fastcover_dict_from_slice, create_raw_dict_from_slice, - finalize_raw_dict, + FinalizeOptions, create_fastcover_dict_from_slice, create_legacy_dict_from_slice, + create_raw_dict_from_slice, finalize_raw_dict, }; if opts.inputs.iter().any(|input| input == Path::new("-")) { @@ -3201,9 +3241,17 @@ fn train_dictionary(opts: &Options) -> Result<()> { // Whether the trainer takes the tuning it was given is a question about the // command line alone, so it is answered before any sample is touched: a run // bound to be refused does not first read a corpus that may be large. - // `Some` holds the FastCOVER options, `None` stands for COVER. - let fastcover = match opts.trainer { - Trainer::FastCover => Some(fastcover_options(&opts.trainer_params)?), + enum Plan { + FastCover(structured_zstd::dictionary::FastCoverOptions), + Cover, + Legacy, + } + let plan = match opts.trainer { + Trainer::FastCover => Plan::FastCover(fastcover_options(&opts.trainer_params)?), + // The legacy trainer is tuned by selectivity alone; a cover tuning list + // given before `--train-legacy` names a trainer that no longer runs, + // as it does in the reference. + Trainer::Legacy => Plan::Legacy, Trainer::Cover => { // The COVER trainer here scores segments by k-mer frequency, as the // reference's does, but is not parameterised the same way: `k`, @@ -3216,7 +3264,7 @@ fn train_dictionary(opts: &Options) -> Result<()> { use --train-fastcover=... for a tunable trainer" ); } - None + Plan::Cover } }; let output = opts @@ -3268,37 +3316,29 @@ fn train_dictionary(opts: &Options) -> Result<()> { } } - // Each sample is opened once, and what the dictionary may carry is taken - // from that same open file rather than from its path afterwards. A path - // answers about whatever it names at the moment it is asked, and training - // takes long enough for a sample to be replaced while it runs: asking again - // at the end could describe a file whose bytes are not the ones now inside - // the dictionary, and grant its permissions to theirs. - let mut corpus = Vec::new(); - let mut samples = Vec::with_capacity(opts.inputs.len()); - for input in &opts.inputs { - let mut file = File::open(input) - .wrap_err_with(|| format!("failed to open training sample {}", input.display()))?; - let metadata = file - .metadata() - .wrap_err_with(|| format!("failed to inspect {}", input.display()))?; - if !metadata.is_file() { - bail!( - "--train needs regular files: {} is not one", - input.display() - ); - } - file.read_to_end(&mut corpus) - .wrap_err_with(|| format!("failed to read training sample {}", input.display()))?; - samples.push(metadata); - } - let finalize = FinalizeOptions { dict_id: opts.dict_id, }; let mut dict = Vec::new(); - match fastcover { - Some(options) => { + let sources = match plan { + Plan::Legacy => { + // The legacy trainer counts samples, so they are loaded as the + // reference's command loads them: shuffled, capped per file, and + // cut by `-B`. The same files then yield the same content. + let set = load_training_samples(&opts.inputs, opts.block_size, opts.memory_limit)?; + create_legacy_dict_from_slice( + &set.corpus, + &set.sizes, + &mut dict, + opts.max_dict, + opts.selectivity, + finalize, + ) + .map_err(|err| eyre!("dictionary training failed: {err}"))?; + set.sources + } + Plan::FastCover(options) => { + let (corpus, sources) = read_whole_samples(&opts.inputs)?; // From the slice, not through a reader: the corpus is the largest // thing this run holds, and the reader path buffers it a second // time inside. @@ -3310,8 +3350,10 @@ fn train_dictionary(opts: &Options) -> Result<()> { finalize, ) .map_err(|err| eyre!("dictionary training failed: {err}"))?; + sources } - None => { + Plan::Cover => { + let (corpus, sources) = read_whole_samples(&opts.inputs)?; // From the slice, as FastCOVER is: the reader path would buffer // the whole corpus a second time, and `corpus` has to stay alive // for the finalizing pass below anyway. @@ -3326,8 +3368,9 @@ fn train_dictionary(opts: &Options) -> Result<()> { // was asked for, so the best content survives the cut. dict = finalize_raw_dict(raw.as_slice(), corpus.as_slice(), opts.max_dict, finalize) .map_err(|err| eyre!("dictionary training failed: {err}"))?; + sources } - } + }; // A trained dictionary is an output file like any other, so it is written // through a temporary that is renamed into place: an interrupted run @@ -3340,7 +3383,7 @@ fn train_dictionary(opts: &Options) -> Result<()> { .and_then(|()| sink.flush()) .wrap_err_with(|| format!("failed to write dictionary {}", output.display()))?; } else { - place_trained_dictionary(&output, &dict, &samples)?; + place_trained_dictionary(&output, &dict, &sources)?; } display!( opts.verbosity, @@ -3353,6 +3396,172 @@ fn train_dictionary(opts: &Options) -> Result<()> { Ok(()) } +/// Read every sample file whole into one corpus, for the trainers that do not +/// tell samples apart; `-B` has nothing to cut for them, and they get every +/// byte, as the reference's do when `-B` cuts files into samples. +/// +/// Each sample is opened once, and what the dictionary may carry is taken from +/// that same open file rather than from its path afterwards. A path answers +/// about whatever it names at the moment it is asked, and training takes long +/// enough for a sample to be replaced while it runs: asking again at the end +/// could describe a file whose bytes are not the ones now inside the +/// dictionary, and grant its permissions to theirs. +fn read_whole_samples(inputs: &[PathBuf]) -> Result<(Vec, Vec)> { + let mut corpus = Vec::new(); + let mut sources = Vec::with_capacity(inputs.len()); + for input in inputs { + let mut file = File::open(input) + .wrap_err_with(|| format!("failed to open training sample {}", input.display()))?; + let metadata = file + .metadata() + .wrap_err_with(|| format!("failed to inspect {}", input.display()))?; + if !metadata.is_file() { + bail!( + "--train needs regular files: {} is not one", + input.display() + ); + } + file.read_to_end(&mut corpus) + .wrap_err_with(|| format!("failed to read training sample {}", input.display()))?; + sources.push(metadata); + } + Ok((corpus, sources)) +} + +/// Most bytes one file contributes as a sample when `-B` does not cut it +/// (`dibio.c`, `SAMPLESIZE_MAX`). +const TRAINING_SAMPLE_MAX: u64 = 128 << 10; + +/// Most training data loaded at all (`dibio.c`, `MAX_SAMPLES_SIZE`). +const TRAINING_DATA_MAX: u64 = 2 << 30; + +/// Fewest samples a trainer is given (`dibio.c`: "nb of samples too low"). +const TRAINING_SAMPLES_MIN: usize = 5; + +/// The samples a trainer is handed, loaded the way the reference's command +/// loads them (`dibio.c`, `DiB_trainFromFiles`). +struct TrainingSet { + /// Every sample back to back. + corpus: Vec, + /// The length of each sample in `corpus`. + sizes: Vec, + /// What each file read was, taken from the open file. + sources: Vec, +} + +/// Reorder the sample files the way the reference does before loading +/// (`DiB_shuffle`), so a sample set too large to load keeps a spread of files +/// rather than the first ones, and the corpus is laid out as there. +fn shuffle_training_files(files: &mut [T]) { + let mut seed: u32 = 0xFD2F_B528; + let mut next = || { + seed = (seed.wrapping_mul(2_654_435_761) ^ 2_246_822_519).rotate_left(13); + seed >> 5 + }; + for i in (1..files.len()).rev() { + let j = (next() % (i as u32 + 1)) as usize; + files.swap(i, j); + } +} + +/// Load the training samples as the reference's command does: files in its +/// shuffled order, each one sample of at most [`TRAINING_SAMPLE_MAX`] bytes, or +/// cut whole into `block_size` samples when `-B` gives one; empty files left +/// out; at most [`TRAINING_DATA_MAX`] bytes, or `memory_limit` when smaller. +fn load_training_samples( + inputs: &[PathBuf], + block_size: Option, + memory_limit: Option, +) -> Result { + let mut order: Vec<&PathBuf> = inputs.iter().collect(); + shuffle_training_files(&mut order); + + // What would be loaded without a limit, and as how many samples. + let mut wanted = 0u64; + let mut samples = 0u64; + for input in &order { + let size = fs::metadata(input) + .wrap_err_with(|| format!("failed to inspect {}", input.display()))? + .len(); + if size == 0 { + continue; + } + match block_size { + Some(block) => { + samples += size.div_ceil(block); + wanted += size; + } + None => { + samples += 1; + wanted += size.min(TRAINING_SAMPLE_MAX); + } + } + } + if samples < TRAINING_SAMPLES_MIN as u64 { + bail!( + "{samples} training sample(s) is too few; provide one file per sample, or \ + split files into fixed-size samples with -B#" + ); + } + let budget = wanted + .min(TRAINING_DATA_MAX) + .min(memory_limit.unwrap_or(u64::MAX)); + let budget = usize::try_from(budget) + .map_err(|_| eyre!("{budget} bytes of samples is more than this machine can hold"))?; + + let mut set = TrainingSet { + corpus: Vec::with_capacity(budget), + sizes: Vec::new(), + sources: Vec::new(), + }; + 'files: for input in order { + if set.sizes.len() as u64 >= samples { + break; + } + let file = File::open(input) + .wrap_err_with(|| format!("failed to open training sample {}", input.display()))?; + let metadata = file + .metadata() + .wrap_err_with(|| format!("failed to inspect {}", input.display()))?; + let size = metadata.len(); + if size == 0 { + continue; + } + let mut reader = file; + let mut taken = 0u64; + loop { + let piece = match block_size { + Some(block) => (size - taken).min(block), + None => size.min(TRAINING_SAMPLE_MAX), + }; + if set.corpus.len() as u64 + piece > budget as u64 { + if taken == 0 { + break 'files; + } + break; + } + let before = set.corpus.len(); + (&mut reader) + .take(piece) + .read_to_end(&mut set.corpus) + .wrap_err_with(|| format!("failed to read training sample {}", input.display()))?; + if (set.corpus.len() - before) as u64 != piece { + bail!( + "{} changed while it was being read; run again", + input.display() + ); + } + set.sizes.push(piece as usize); + taken += piece; + if block_size.is_none() || taken >= size || set.sizes.len() as u64 >= samples { + break; + } + } + set.sources.push(metadata); + } + Ok(set) +} + /// Write a trained dictionary to the regular file `output` through a /// temporary renamed into place, no more readable than the strictest of the /// `samples` (their metadata) it was trained on. diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index a5ef1c916..693ebd088 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -1724,19 +1724,119 @@ fn training_refuses_to_overwrite_without_force() { } /// The trainer flags name algorithms, and the algorithm decides what the -/// dictionary contains. FastCOVER and COVER are here; the legacy trainer is -/// not, so its flag has to say no rather than run another trainer under its -/// name. +/// dictionary contains: `--train-legacy` selects the legacy trainer, and its +/// selectivity comes from `=s=#`, `=selectivity=#` or `-s#`, read as the +/// reference reads them. #[test] -fn the_legacy_trainer_is_refused_not_substituted() { +fn the_legacy_trainer_flags_select_it_and_its_selectivity() { assert_eq!(parse(&["--train", "s1"]).unwrap().mode, Mode::Train); assert_eq!( parse(&["--train-fastcover", "s1"]).unwrap().mode, Mode::Train ); assert_eq!(parse(&["--train-cover", "s1"]).unwrap().mode, Mode::Train); - assert!(parse(&["--train-legacy", "s1"]).is_err()); - assert!(parse(&["--train-legacy=s=8", "s1"]).is_err()); + let legacy = parse(&["--train-legacy", "s1"]).unwrap(); + assert_eq!( + (legacy.mode, legacy.trainer), + (Mode::Train, Trainer::Legacy) + ); + assert_eq!(legacy.selectivity, 0, "the trainer's own default"); + assert_eq!(parse(&["--train-legacy=s=8", "s1"]).unwrap().selectivity, 8); + assert_eq!( + parse(&["--train-legacy=selectivity=12", "s1"]) + .unwrap() + .selectivity, + 12 + ); + assert_eq!( + parse(&["--train-legacy", "-s5", "s1"]).unwrap().selectivity, + 5 + ); + assert!(parse(&["--train-legacy=k=8", "s1"]).is_err(), "no such key"); + assert!( + parse(&["--train-legacy=s=8x", "s1"]).is_err(), + "trailing junk" + ); + assert!(parse(&["--train-legacy", "-sx", "s1"]).is_err()); +} + +/// `--train-legacy` trains end to end: sample files in, a dictionary out that +/// the decoder accepts, made of what the samples repeat. +#[test] +fn legacy_training_writes_a_dictionary() { + let dir = Scratch::new("legacy-train"); + let mut names = Vec::new(); + for i in 0..40u32 { + let body = format!( + "[Unit]\nDescription=worker {i}\nAfter=network-online.target\n\n[Service]\n\ + ExecStart=/usr/bin/worker --id {i} --config /etc/worker/worker.toml\n\ + Restart=on-failure\n" + ); + names.push(dir.file(&format!("w{i}.service"), body.as_bytes())); + } + let output = dir.path().join("legacy.dict"); + let mut args = vec!["--train-legacy".to_string(), "-o".to_string()]; + args.push(output.display().to_string()); + args.extend(names.iter().map(|name| name.display().to_string())); + let argv: Vec<&str> = args.iter().map(String::as_str).collect(); + run(parse(&argv).unwrap()).unwrap(); + + let dict = fs::read(&output).unwrap(); + structured_zstd::decoding::Dictionary::decode_dict(&dict).expect("a valid dictionary"); + let text = String::from_utf8_lossy(&dict); + assert!(text.contains("After=network-online.target")); + assert!(text.contains("Restart=on-failure")); +} + +/// The legacy trainer is handed its samples as the reference's command hands +/// them over: each file one sample of at most 128 KiB, or cut whole into `-B` +/// pieces; empty files left out; fewer than five samples refused. +#[test] +fn training_samples_are_loaded_the_way_the_reference_loads_them() { + let dir = Scratch::new("training-load"); + let big = dir.file("big", &vec![b'x'; 300 << 10]); + let small = dir.file("small", b"0123456789"); + let empty = dir.file("empty", b""); + let inputs = vec![big.clone(), small.clone(), empty.clone()]; + + assert!( + load_training_samples(&inputs, None, None).is_err(), + "two samples are too few" + ); + + let cut = load_training_samples(&inputs, Some(64 << 10), None).unwrap(); + let mut sizes = cut.sizes.clone(); + sizes.sort_unstable(); + assert_eq!( + sizes, + vec![10, 44 << 10, 64 << 10, 64 << 10, 64 << 10, 64 << 10] + ); + assert_eq!(cut.corpus.len(), (300 << 10) + 10, "whole files once cut"); + + let mut many = Vec::new(); + for i in 0..5 { + many.push(dir.file(&format!("big{i}"), &vec![b'y'; 200 << 10])); + } + let capped = load_training_samples(&many, None, None).unwrap(); + assert_eq!( + capped.sizes, + vec![128 << 10; 5], + "each file capped at 128 KiB" + ); + let limited = load_training_samples(&many, None, Some(300 << 10)).unwrap(); + assert_eq!(limited.sizes.len(), 2, "-M bounds what is loaded"); +} + +/// The files are taken in the reference's shuffled order (`DiB_shuffle`), so +/// the same file list lays the corpus out the same way; the legacy dictionary +/// trained from a fixed list then carries the reference's content byte for +/// byte. The order is what `dibio.c`'s own `DiB_shuffle`, compiled as is, +/// makes of eight entries. +#[test] +fn training_files_are_shuffled_like_the_reference_shuffles_them() { + let mut order: Vec = (0..8).collect(); + shuffle_training_files(&mut order); + assert_eq!(order, vec![4, 5, 2, 0, 6, 1, 7, 3]); } /// A window is a promise about how much memory decoding will need, so it is @@ -4464,7 +4564,10 @@ fn trainer_parameters_parse_and_build_options() { let opts = parse(&["--train-cover", "s1"]).unwrap(); assert_eq!(opts.trainer, Trainer::Cover); assert!(opts.trainer_params.is_default()); - assert!(parse(&["--train-legacy", "s1"]).is_err()); + assert_eq!( + parse(&["--train-legacy", "s1"]).unwrap().trainer, + Trainer::Legacy + ); } /// An empty `--filelist` is nothing to do for the modes that stream, but the diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs new file mode 100644 index 000000000..a39c30e08 --- /dev/null +++ b/zstd/src/dictionary/legacy.rs @@ -0,0 +1,548 @@ +//! The reference's original dictionary trainer (`zdict.c`, +//! `ZDICT_trainFromBuffer_legacy`), which `zstd --train-legacy` runs. +//! +//! It walks the suffix array of the whole corpus and, for every position not +//! yet covered, measures how many other positions share a prefix of at least +//! [`MIN_MATCH_LENGTH`] bytes with it. A prefix repeated often enough becomes a +//! candidate segment, scored by the bytes it would save; overlapping candidates +//! merge, and the best of them, up to the requested size, are the dictionary. +//! Selectivity sets how often "often enough" is: a candidate needs at least +//! `samples >> selectivity` repetitions, and never fewer than [`MIN_RATIO`]. + +use alloc::vec; +use alloc::vec::Vec; + +use super::suffix_array::suffix_array; + +/// Fewest repetitions that make a prefix a candidate (`MINRATIO`). +const MIN_RATIO: u32 = 4; +/// Shortest prefix counted as a repetition (`MINMATCHLENGTH`). +const MIN_MATCH_LENGTH: usize = 7; +/// Longest repetition length told apart when scoring (`LLIMIT`). +const LENGTH_LIMIT: usize = 64; +/// Smallest candidate table (`DICTLISTSIZE_DEFAULT`). +const DICT_LIST_SIZE_DEFAULT: usize = 10_000; +/// Bytes of noise after the corpus, so a comparison running off its end stops +/// on a mismatch (`NOISELENGTH`). +const NOISE_LENGTH: usize = 32; +/// Most corpus the trainer reads; whole samples beyond it are dropped +/// (`ZDICT_MAX_SAMPLES_SIZE`). +const MAX_SAMPLES_SIZE: usize = 2000 << 20; +/// Smallest dictionary content the trainer returns (`ZDICT_CONTENTSIZE_MIN`). +const CONTENT_SIZE_MIN: usize = 128; +/// Smallest corpus worth training on (`ZDICT_MIN_SAMPLES_SIZE`). +pub(crate) const MIN_SAMPLES_SIZE: usize = CONTENT_SIZE_MIN * MIN_RATIO as usize; +/// Selectivity the reference uses when none is given (`g_selectivity_default`). +pub const DEFAULT_SELECTIVITY: u32 = 9; + +/// Smallest dictionary the trainer is asked for (`ZDICT_DICTSIZE_MIN`). +pub(crate) const DICT_SIZE_MIN: usize = 256; + +/// What was too small for the legacy trainer to produce a dictionary. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TooSmall { + /// The requested size is below [`DICT_SIZE_MIN`]. + Dictionary, + /// The corpus is smaller than [`MIN_SAMPLES_SIZE`]. + Corpus, + /// The corpus repeats too little to fill [`CONTENT_SIZE_MIN`] bytes. + Content, +} + +/// A candidate segment (`dictItem`). In the table, entry 0 is a header whose +/// `pos` counts the entries in use, the header included, and whose `savings` +/// is the largest value, so the sorted insert stops on it. +#[derive(Debug, Clone, Copy, Default)] +struct DictItem { + pos: u32, + length: u32, + savings: u32, +} + +/// The corpus followed by its noise band, read through bounds-checked +/// accessors that treat anything past the band as a mismatch. +struct Corpus { + bytes: Vec, +} + +impl Corpus { + fn new(samples: &[u8]) -> Self { + let mut bytes = Vec::with_capacity(samples.len() + NOISE_LENGTH); + bytes.extend_from_slice(samples); + // The reference's noise generator (`ZDICT_fillNoise`). + let mut acc: u32 = 2_654_435_761; + for _ in 0..NOISE_LENGTH { + acc = acc.wrapping_mul(2_246_822_519); + bytes.push((acc >> 21) as u8); + } + Self { bytes } + } + + #[inline] + fn byte(&self, at: usize) -> Option { + self.bytes.get(at).copied() + } + + #[inline] + fn read16(&self, at: usize) -> Option { + let pair = self.bytes.get(at..at + 2)?; + Some(u16::from_le_bytes([pair[0], pair[1]])) + } + + #[inline] + fn read64(&self, at: usize) -> Option { + let word = self.bytes.get(at..at + 8)?; + Some(u64::from_le_bytes(word.try_into().expect("eight bytes"))) + } + + /// Bytes `a` and `b` have in common (`ZDICT_count`). + fn common(&self, a: usize, b: usize) -> usize { + let (Some(left), Some(right)) = (self.bytes.get(a..), self.bytes.get(b..)) else { + return 0; + }; + left.iter().zip(right).take_while(|(x, y)| x == y).count() + } +} + +/// The suffix array with one extra slot on each side, both pointing into the +/// noise band, as the reference lays it out (`suffix0[0]` and +/// `suffix[bufferSize]`): a walk off either end compares against noise and +/// stops there. +struct Suffixes { + padded: Vec, + noise: u32, +} + +impl Suffixes { + fn new(sa: Vec, len: u32) -> Self { + let mut padded = Vec::with_capacity(sa.len() + 2); + padded.push(len); + padded.extend_from_slice(&sa); + padded.push(len); + Self { padded, noise: len } + } + + /// `suffix[at]`, for `at` in `-1..=len`; any rank further out is the noise + /// band as well, which a walk can only reach on a corpus whose tail matches + /// the noise. + #[inline] + fn at(&self, at: i64) -> usize { + usize::try_from(at + 1) + .ok() + .and_then(|slot| self.padded.get(slot)) + .copied() + .unwrap_or(self.noise) as usize + } +} + +/// Train dictionary content from `samples`, the concatenation of samples whose +/// lengths are `sample_sizes`, keeping at most `dict_size` bytes. +/// +/// The segments are laid out best last, so the most valuable content sits +/// nearest the data and is reached with the smallest offsets. `selectivity` of +/// zero is [`DEFAULT_SELECTIVITY`]. +pub(crate) fn train_legacy_raw( + samples: &[u8], + sample_sizes: &[usize], + dict_size: usize, + selectivity: u32, +) -> Result, TooSmall> { + let total: usize = sample_sizes.iter().sum(); + debug_assert_eq!(total, samples.len(), "the sizes describe the corpus"); + if dict_size < DICT_SIZE_MIN { + return Err(TooSmall::Dictionary); + } + if total < MIN_SAMPLES_SIZE { + return Err(TooSmall::Corpus); + } + let selectivity = if selectivity == 0 { + DEFAULT_SELECTIVITY + } else { + selectivity + }; + let nb_samples = sample_sizes.len(); + let min_rep = if selectivity > 30 { + MIN_RATIO + } else { + // A repetition count past u32 is more than a u32-indexed corpus has + // positions, so nothing could qualify. + u32::try_from(nb_samples >> selectivity).map_err(|_| TooSmall::Content)? + }; + let list_size = DICT_LIST_SIZE_DEFAULT.max(nb_samples).max(dict_size / 16); + let mut list = vec![DictItem::default(); list_size]; + list[0] = DictItem { + pos: 1, + length: 0, + savings: u32::MAX, + }; + + // Whole samples past the size limit are left out, last first. + let mut used = total; + let mut kept = nb_samples; + while used > MAX_SAMPLES_SIZE { + kept -= 1; + used -= sample_sizes[kept]; + } + let corpus = Corpus::new(&samples[..used]); + find_segments(&mut list, &corpus, used, min_rep); + + let segments = list[0].pos as usize; + let content_size: usize = list[1..segments].iter().map(|d| d.length as usize).sum(); + if content_size < CONTENT_SIZE_MIN { + return Err(TooSmall::Content); + } + // Keep the best segments that fit, in rank order. + let mut fit = 1; + let mut size = 0usize; + while fit < segments { + let next = size + list[fit].length as usize; + if next > dict_size { + break; + } + size = next; + fit += 1; + } + // Filled from the back, as the reference fills its buffer: rank 1 last. + let mut content = vec![0u8; size]; + let mut end = size; + for item in &list[1..fit] { + let length = item.length as usize; + let start = end - length; + let from = item.pos as usize; + content[start..end].copy_from_slice(&corpus.bytes[from..from + length]); + end = start; + } + debug_assert_eq!(end, 0, "the kept segments fill the content exactly"); + Ok(content) +} + +/// `ZDICT_trainBuffer_legacy`: walk every uncovered position of the corpus in +/// text order and insert the segment its suffix neighbourhood yields. +fn find_segments(list: &mut [DictItem], corpus: &Corpus, len: usize, min_rep: u32) { + let min_ratio = min_rep.max(MIN_RATIO); + let sa = suffix_array(&corpus.bytes[..len]); + let mut rank = vec![0u32; len]; + for (at, &pos) in sa.iter().enumerate() { + rank[pos as usize] = at as u32; + } + let suffixes = Suffixes::new(sa, len as u32); + // Slack past the corpus, as the reference allocates it: a covered run may + // be marked into the noise band. + let mut done = vec![false; len + 16]; + let max_size = list.len() as u32; + + let mut cursor = 0usize; + while cursor < len { + if done[cursor] { + cursor += 1; + continue; + } + let solution = analyze_position( + &mut done, + &suffixes, + i64::from(rank[cursor]), + corpus, + min_ratio, + ); + if solution.length == 0 { + cursor += 1; + continue; + } + insert_item(list, max_size, solution, corpus); + cursor += solution.length as usize; + } +} + +/// Mark `at` covered, if it lies inside the marks. +#[inline] +fn mark(done: &mut [bool], at: usize) { + if let Some(slot) = done.get_mut(at) { + *slot = true; + } +} + +/// `ZDICT_analyzePos`: the segment the suffix of rank `start` leads to, or an +/// empty one. Every position the analysis settles is marked in `done`. +fn analyze_position( + done: &mut [bool], + suffixes: &Suffixes, + mut start: i64, + corpus: &Corpus, + min_ratio: u32, +) -> DictItem { + let mut pos = suffixes.at(start); + let mut end = start; + let empty = DictItem::default(); + mark(done, pos); + + // A run of one repeated pair is skipped whole: it compresses without a + // dictionary. + let repeats = |a: usize, b: usize| matches!((corpus.read16(a), corpus.read16(b)), (Some(x), Some(y)) if x == y); + if repeats(pos, pos + 2) || repeats(pos + 1, pos + 3) || repeats(pos + 2, pos + 4) { + let pattern = corpus.read16(pos + 4); + let mut pattern_end = 6usize; + while pattern.is_some() && corpus.read16(pos + pattern_end) == pattern { + pattern_end += 2; + } + if corpus.byte(pos + pattern_end).is_some() + && corpus.byte(pos + pattern_end) == corpus.byte(pos + pattern_end - 1) + { + pattern_end += 1; + } + for u in 1..pattern_end { + mark(done, pos + u); + } + return empty; + } + + // The neighbours sharing at least the minimum length, forward then back. + loop { + end += 1; + if corpus.common(pos, suffixes.at(end)) < MIN_MATCH_LENGTH { + break; + } + } + while corpus.common(pos, suffixes.at(start - 1)) >= MIN_MATCH_LENGTH { + start -= 1; + } + + if ((end - start) as u32) < min_ratio { + for id in start..end { + mark(done, suffixes.at(id)); + } + return empty; + } + + // Lengthen the shared prefix one byte at a time while the most common + // continuation still repeats often enough. + let mut refined_start = start; + let mut refined_end = end; + let mut mml = MIN_MATCH_LENGTH; + loop { + let mut current_char = 0u8; + let mut current_count = 0u32; + let mut current_id = refined_start; + let mut selected_count = 0u32; + let mut selected_id = current_id; + for id in refined_start..refined_end { + let c = corpus.byte(suffixes.at(id) + mml).unwrap_or(0); + if c != current_char { + if current_count > selected_count { + selected_count = current_count; + selected_id = current_id; + } + current_id = id; + current_char = c; + current_count = 0; + } + current_count += 1; + } + if current_count > selected_count { + selected_count = current_count; + selected_id = current_id; + } + if selected_count < min_ratio { + break; + } + refined_start = selected_id; + refined_end = refined_start + i64::from(selected_count); + mml += 1; + } + + // Measure the refined segment's neighbourhood. + start = refined_start; + pos = suffixes.at(refined_start); + end = start; + let mut lengths = [0u32; LENGTH_LIMIT]; + loop { + end += 1; + let length = corpus.common(pos, suffixes.at(end)).min(LENGTH_LIMIT - 1); + lengths[length] += 1; + if length < MIN_MATCH_LENGTH { + break; + } + } + { + let mut length = MIN_MATCH_LENGTH; + while length >= MIN_MATCH_LENGTH && start > 0 { + length = corpus + .common(pos, suffixes.at(start - 1)) + .min(LENGTH_LIMIT - 1); + lengths[length] += 1; + if length >= MIN_MATCH_LENGTH { + start -= 1; + } + } + } + + // The longest length still shared by enough neighbours. + let mut cumulative = [0u32; LENGTH_LIMIT]; + cumulative[LENGTH_LIMIT - 1] = lengths[LENGTH_LIMIT - 1]; + for i in (0..LENGTH_LIMIT - 1).rev() { + cumulative[i] = cumulative[i + 1] + lengths[i]; + } + let mut max_length = LENGTH_LIMIT - 1; + while max_length >= MIN_MATCH_LENGTH && cumulative[max_length] < min_ratio { + max_length -= 1; + } + // Do not end inside a run of the last byte. + { + let last = corpus.byte(pos + max_length - 1); + let mut l = max_length; + while l >= 2 && corpus.byte(pos + l - 2) == last { + l -= 1; + } + max_length = l; + } + if max_length < MIN_MATCH_LENGTH { + return empty; + } + + let mut savings = [0u32; LENGTH_LIMIT]; + for i in MIN_MATCH_LENGTH..=max_length { + savings[i] = savings[i - 1].wrapping_add(lengths[i].wrapping_mul((i - 3) as u32)); + } + let solution = DictItem { + pos: pos as u32, + length: max_length as u32, + savings: savings[max_length], + }; + + for id in start..end { + let tested = suffixes.at(id); + let length = if tested == pos { + max_length + } else { + corpus.common(pos, tested).min(max_length) + }; + for p in tested..tested + length { + mark(done, p); + } + } + solution +} + +/// Whether the `length` bytes at `a` equal those at `b` (`isIncluded`). +fn is_included(corpus: &Corpus, a: usize, b: usize, length: usize) -> bool { + match ( + corpus.bytes.get(a..a + length), + corpus.bytes.get(b..b + length), + ) { + (Some(x), Some(y)) => x == y, + _ => false, + } +} + +/// Move entry `at` towards the front while its savings beat its predecessor's. +fn promote(list: &mut [DictItem], mut at: usize) -> usize { + let item = list[at]; + while at > 1 && list[at - 1].savings < item.savings { + list[at] = list[at - 1]; + at -= 1; + } + list[at] = item; + at +} + +/// `ZDICT_tryMerge`: fold `elt` into an entry it overlaps, skipping entry +/// `skip`. Returns the merged entry's index, or 0 when nothing merged. +fn try_merge(list: &mut [DictItem], elt: DictItem, skip: usize, corpus: &Corpus) -> usize { + let size = list[0].pos as usize; + let elt_end = elt.pos + elt.length; + + // An existing entry starts inside `elt`: extend it backwards. + for u in 1..size { + if u == skip { + continue; + } + if list[u].pos > elt.pos && list[u].pos <= elt_end { + let added = list[u].pos - elt.pos; + list[u].length += added; + list[u].pos = elt.pos; + list[u].savings = list[u] + .savings + .wrapping_add(elt.savings.wrapping_mul(added) / elt.length); + list[u].savings = list[u].savings.wrapping_add(elt.length / 8); + return promote(list, u); + } + } + + // `elt` starts inside an existing entry, or right after a copy of it. + for u in 1..size { + if u == skip { + continue; + } + if list[u].pos + list[u].length >= elt.pos && list[u].pos < elt.pos { + let added = elt_end as i64 - i64::from(list[u].pos + list[u].length); + list[u].savings = list[u].savings.wrapping_add(elt.length / 8); + if added > 0 { + list[u].length += added as u32; + list[u].savings = list[u] + .savings + .wrapping_add(elt.savings.wrapping_mul(added as u32) / elt.length); + } + return promote(list, u); + } + let head = corpus.read64(list[u].pos as usize); + if head.is_some() + && head == corpus.read64(elt.pos as usize + 1) + && is_included( + corpus, + list[u].pos as usize, + elt.pos as usize + 1, + list[u].length as usize, + ) + { + // The reference takes this product at pointer width, where it does + // not wrap, unlike the two above. + let added = (i64::from(elt.length) - i64::from(list[u].length)).max(1) as u64; + list[u].pos = elt.pos; + list[u].savings = list[u] + .savings + .wrapping_add((u64::from(elt.savings) * added / u64::from(elt.length)) as u32); + list[u].length = elt.length.min(list[u].length + 1); + return u; + } + } + 0 +} + +/// `ZDICT_removeDictItem`. +fn remove_item(list: &mut [DictItem], id: usize) { + if id == 0 { + return; + } + let max = list[0].pos as usize; + for u in id..max - 1 { + list[u] = list[u + 1]; + } + list[0].pos -= 1; +} + +/// `ZDICT_insertDictItem`: merge `elt` into the table if it overlaps an entry, +/// and keep merging while the merged entry overlaps another; otherwise insert +/// it in savings order, dropping the last entry when the table is full. +fn insert_item(list: &mut [DictItem], max_size: u32, elt: DictItem, corpus: &Corpus) { + let mut merge_id = try_merge(list, elt, 0, corpus); + if merge_id != 0 { + loop { + let merged = try_merge(list, list[merge_id], merge_id, corpus); + if merged == 0 { + return; + } + remove_item(list, merge_id); + merge_id = merged; + } + } + let next = list[0].pos.min(max_size - 1) as usize; + let mut current = next - 1; + while list[current].savings < elt.savings { + list[current + 1] = list[current]; + current -= 1; + } + list[current + 1] = elt; + list[0].pos = next as u32 + 1; +} + +#[cfg(test)] +mod tests; diff --git a/zstd/src/dictionary/legacy/tests.rs b/zstd/src/dictionary/legacy/tests.rs new file mode 100644 index 000000000..8037736be --- /dev/null +++ b/zstd/src/dictionary/legacy/tests.rs @@ -0,0 +1,96 @@ +use super::*; +use alloc::format; +use alloc::string::String; + +/// `count` log lines of a few shapes, each line a sample. +fn log_samples(count: u32) -> (Vec, Vec) { + const SHAPES: [&str; 4] = [ + "ts={i} level=INFO msg=\"flush memtable\" tenant=demo table=orders region=eu-west\n", + "ts={i} level=WARN msg=\"slow compaction\" tenant=demo table=users region=us-east\n", + "ts={i} level=INFO msg=\"rotate segment\" tenant=acme table=orders region=eu-west\n", + "ts={i} level=ERROR msg=\"write stalled\" tenant=acme table=events region=ap-south\n", + ]; + let mut samples = Vec::new(); + let mut sizes = Vec::new(); + for i in 0..count { + let line = SHAPES[(i % 4) as usize].replace("{i}", &format!("{:08}", i * 7919)); + sizes.push(line.len()); + samples.extend_from_slice(line.as_bytes()); + } + (samples, sizes) +} + +/// The trainer keeps what the corpus repeats: every segment it returns is a +/// run of the corpus that occurs at least `MIN_RATIO` times, and the content +/// stays within the size asked for. +#[test] +fn the_content_is_made_of_repeated_corpus_runs() { + let (samples, sizes) = log_samples(400); + let content = train_legacy_raw(&samples, &sizes, 4096, 0).unwrap(); + assert!(content.len() >= CONTENT_SIZE_MIN && content.len() <= 4096); + let text = String::from_utf8_lossy(&content); + for field in [ + "tenant=demo table=orders", + "msg=\"flush memtable\"", + "region=eu-west", + ] { + assert!( + text.contains(field), + "{field} is repeated in every fourth sample" + ); + } + // A timestamp is unique per sample, so no whole one survives. + assert!(!text.contains(&format!("ts={:08}", 3 * 7919))); +} + +/// A size below the reference's minimum, a corpus under 512 bytes, and a corpus +/// with nothing repeated are refused, as `ZDICT_trainFromBuffer_legacy` refuses +/// them. +#[test] +fn too_little_to_train_on_is_refused() { + let (samples, sizes) = log_samples(400); + assert_eq!( + train_legacy_raw(&samples, &sizes, DICT_SIZE_MIN - 1, 0), + Err(TooSmall::Dictionary) + ); + let (small, small_sizes) = log_samples(5); + assert!(small.len() < MIN_SAMPLES_SIZE); + assert_eq!( + train_legacy_raw(&small, &small_sizes, 4096, 0), + Err(TooSmall::Corpus) + ); + let mut state = 0x9E37_79B9u32; + let noise: Vec = (0..8192) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + assert_eq!( + train_legacy_raw(&noise, &[noise.len()], 4096, 0), + Err(TooSmall::Content) + ); +} + +/// Selectivity decides how often a segment must repeat: `samples >> s`, never +/// below four. With many samples a low selectivity demands more repetitions +/// than a rare shape has, so it keeps less than a high one. +#[test] +fn selectivity_sets_how_often_a_segment_must_repeat() { + let (mut samples, mut sizes) = log_samples(4096); + // A shape present in 40 samples: kept when 40 repetitions suffice. + for i in 0..40 { + let line = format!("rare shape {i:04} backup completed for volume vol-archive-7\n"); + sizes.push(line.len()); + samples.extend_from_slice(line.as_bytes()); + } + let strict = train_legacy_raw(&samples, &sizes, 64 * 1024, 6).unwrap(); + let loose = train_legacy_raw(&samples, &sizes, 64 * 1024, 9).unwrap(); + let has_rare = |content: &[u8]| String::from_utf8_lossy(content).contains("vol-archive-7"); + // 4136 >> 6 = 64 repetitions needed: the rare shape has 40. + assert!(!has_rare(&strict)); + // 4136 >> 9 = 8 repetitions needed. + assert!(has_rare(&loose)); +} diff --git a/zstd/src/dictionary/mod.rs b/zstd/src/dictionary/mod.rs index ef1ccde54..9339c37a2 100644 --- a/zstd/src/dictionary/mod.rs +++ b/zstd/src/dictionary/mod.rs @@ -25,7 +25,9 @@ mod cover; mod fastcover; mod frequency; +mod legacy; mod reservoir; +mod suffix_array; use crate::bit_io::BitWriter; use crate::blocks::sequence_section::{ @@ -43,6 +45,7 @@ pub use fastcover::{ DEFAULT_D_CANDIDATES, DEFAULT_F_CANDIDATES, DEFAULT_K_CANDIDATES, FastCoverParams, FastCoverTuned, }; +pub use legacy::DEFAULT_SELECTIVITY; use std::{ boxed::Box, collections::{BinaryHeap, HashMap}, @@ -700,6 +703,90 @@ pub fn create_fastcover_dict_from_slice( Ok(tuned) } +/// Train and finalize a dictionary with the reference's original trainer, the +/// one `zstd --train-legacy` runs (`ZDICT_trainFromBuffer_legacy`). +/// +/// `samples` is every sample back to back and `sample_sizes` their lengths: +/// the trainer searches the corpus as a whole, and the number of samples sets +/// how often a segment has to repeat to be kept, `samples >> selectivity` +/// times and at least 4. A higher `selectivity` keeps more, rarer segments; +/// zero is [`DEFAULT_SELECTIVITY`]. Corpus past 2000 MiB is dropped a whole +/// sample at a time from the end. +/// +/// # Errors +/// +/// `InvalidInput` when `dict_size` is below 256 bytes, when the corpus is under +/// 512 bytes, when it repeats too little to yield 128 bytes of content, or when +/// `sample_sizes` does not add up to `samples.len()`. +/// +/// # Examples +/// +/// ``` +/// use structured_zstd::dictionary::{create_legacy_dict_from_slice, FinalizeOptions}; +/// +/// let mut samples = Vec::new(); +/// let mut sizes = Vec::new(); +/// for i in 0..200u32 { +/// let line = format!("tenant=demo table=orders key={i} region=eu status=shipped\n"); +/// sizes.push(line.len()); +/// samples.extend_from_slice(line.as_bytes()); +/// } +/// let mut dict = Vec::new(); +/// create_legacy_dict_from_slice(&samples, &sizes, &mut dict, 4096, 0, FinalizeOptions::default()) +/// .unwrap(); +/// assert!(dict.starts_with(&[0x37, 0xA4, 0x30, 0xEC])); +/// ``` +pub fn create_legacy_dict_from_slice( + samples: &[u8], + sample_sizes: &[usize], + output: &mut W, + dict_size: usize, + selectivity: u32, + finalize: FinalizeOptions, +) -> io::Result<()> { + let described = sample_sizes + .iter() + .try_fold(0usize, |total, &size| total.checked_add(size)); + if described != Some(samples.len()) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "the sample sizes do not add up to the corpus", + )); + } + let content = + legacy::train_legacy_raw(samples, sample_sizes, dict_size, selectivity).map_err(|err| { + let reason = match err { + legacy::TooSmall::Dictionary => format!( + "a legacy dictionary must be at least {} bytes", + legacy::DICT_SIZE_MIN + ), + legacy::TooSmall::Corpus => format!( + "the samples total {} bytes; the legacy trainer needs at least {}", + samples.len(), + legacy::MIN_SAMPLES_SIZE + ), + legacy::TooSmall::Content => { + "the samples repeat too little to yield dictionary content".into() + } + }; + io::Error::new(io::ErrorKind::InvalidInput, reason) + })?; + let finalized = finalize_raw_dict(content.as_slice(), samples, dict_size, finalize)?; + output.write_all(finalized.as_slice()) +} + +/// The legacy trainer's content alone, for the reference comparison in +/// `ffi-bench`. +#[cfg(feature = "bench-internals")] +pub(crate) fn legacy_dict_content( + samples: &[u8], + sample_sizes: &[usize], + dict_size: usize, + selectivity: u32, +) -> Option> { + legacy::train_legacy_raw(samples, sample_sizes, dict_size, selectivity).ok() +} + /// Build a finalized FastCOVER dictionary, attach it to a fastest-level /// frame compressor, and compress a fresh payload. Returns /// `(finalized_dictionary, compressed_frame, original_payload)` so a diff --git a/zstd/src/dictionary/suffix_array.rs b/zstd/src/dictionary/suffix_array.rs new file mode 100644 index 000000000..3de21ce48 --- /dev/null +++ b/zstd/src/dictionary/suffix_array.rs @@ -0,0 +1,207 @@ +//! Suffix-array construction by induced sorting (SA-IS, Nong, Zhang and Chan, +//! "Two Efficient Algorithms for Linear Time Suffix Array Construction"). +//! +//! A suffix array is unique for its text: the positions of every suffix in +//! lexicographic order, a suffix that is a prefix of another ordered first. +//! So any correct construction yields the array the reference trainer gets +//! from `divsufsort`, which is what makes the legacy trainer built on it +//! reproduce the reference's choices. + +use alloc::vec; +use alloc::vec::Vec; + +/// Marks an empty slot of the array under construction. +const EMPTY: u32 = u32::MAX; + +/// Below this length a comparison sort is cheaper than induction. +const NAIVE_THRESHOLD: usize = 10; + +/// A symbol of the text being sorted: the bytes at the top level, the ranks of +/// the reduced string in the recursion. +trait Symbol: Copy + Ord { + fn index(self) -> usize; +} + +impl Symbol for u8 { + #[inline] + fn index(self) -> usize { + usize::from(self) + } +} + +impl Symbol for u32 { + #[inline] + fn index(self) -> usize { + self as usize + } +} + +/// The suffix array of `text`: `sa[i]` is the start of the `i`-th smallest +/// suffix. +/// +/// # Panics +/// +/// If `text` is `u32::MAX` bytes or longer, since positions are held as `u32` +/// with one value kept free as the empty mark. +pub(crate) fn suffix_array(text: &[u8]) -> Vec { + assert!( + text.len() < EMPTY as usize, + "a suffix array of {} bytes does not fit u32 positions", + text.len() + ); + sa_is(text, usize::from(u8::MAX)) +} + +/// SA-IS over `s`, whose symbols all lie in `0..=upper`. The text is taken to +/// end in a virtual sentinel smaller than every symbol. +fn sa_is(s: &[T], upper: usize) -> Vec { + let n = s.len(); + match n { + 0 => return Vec::new(), + 1 => return vec![0], + 2 => return if s[0] < s[1] { vec![0, 1] } else { vec![1, 0] }, + _ => {} + } + if n < NAIVE_THRESHOLD { + let mut sa: Vec = (0..n as u32).collect(); + sa.sort_unstable_by(|&a, &b| s[a as usize..].cmp(&s[b as usize..])); + return sa; + } + + // `ls[i]`: the suffix at `i` is S-type (smaller than the one after it). + // The last is L-type against the virtual sentinel. + let mut ls = vec![false; n]; + for i in (0..n - 1).rev() { + ls[i] = if s[i] == s[i + 1] { + ls[i + 1] + } else { + s[i] < s[i + 1] + }; + } + // Bucket starts: `sum_l[c]` where the L-type suffixes of `c` begin, + // `sum_s[c]` where its S-type ones do. + let mut sum_l = vec![0u32; upper + 1]; + let mut sum_s = vec![0u32; upper + 1]; + for i in 0..n { + if ls[i] { + // An S-type suffix is followed by a larger symbol, so its own is + // never the largest and the next bucket exists. + debug_assert!(s[i].index() < upper); + sum_l[s[i].index() + 1] += 1; + } else { + sum_s[s[i].index()] += 1; + } + } + for c in 0..=upper { + sum_s[c] += sum_l[c]; + if c < upper { + sum_l[c + 1] += sum_s[c]; + } + } + + let mut sa = vec![EMPTY; n]; + let mut buf = vec![0u32; upper + 1]; + let induce = |sa: &mut [u32], buf: &mut [u32], lms: &[u32]| { + sa.fill(EMPTY); + buf.copy_from_slice(&sum_s); + for &d in lms { + let d = d as usize; + if d == n { + continue; + } + let c = s[d].index(); + sa[buf[c] as usize] = d as u32; + buf[c] += 1; + } + buf.copy_from_slice(&sum_l); + let c = s[n - 1].index(); + sa[buf[c] as usize] = (n - 1) as u32; + buf[c] += 1; + for i in 0..n { + let v = sa[i]; + if v != EMPTY && v >= 1 && !ls[v as usize - 1] { + let c = s[v as usize - 1].index(); + sa[buf[c] as usize] = v - 1; + buf[c] += 1; + } + } + buf.copy_from_slice(&sum_l); + for i in (0..n).rev() { + let v = sa[i]; + if v != EMPTY && v >= 1 && ls[v as usize - 1] { + let c = s[v as usize - 1].index() + 1; + buf[c] -= 1; + sa[buf[c] as usize] = v - 1; + } + } + }; + + // The leftmost S-type positions, and each one's rank among them. + let mut lms_map = vec![EMPTY; n + 1]; + let mut lms = Vec::new(); + for i in 1..n { + if !ls[i - 1] && ls[i] { + lms_map[i] = lms.len() as u32; + lms.push(i as u32); + } + } + let m = lms.len(); + + induce(&mut sa, &mut buf, &lms); + + if m > 0 { + let mut sorted_lms: Vec = sa + .iter() + .copied() + .filter(|&v| v != EMPTY && lms_map[v as usize] != EMPTY) + .collect(); + // Name each LMS substring by rank, equal substrings sharing a name, and + // sort the string of names recursively. + let mut rec_s = vec![0u32; m]; + let mut rec_upper = 0u32; + rec_s[lms_map[sorted_lms[0] as usize] as usize] = 0; + for i in 1..m { + let mut l = sorted_lms[i - 1] as usize; + let mut r = sorted_lms[i] as usize; + let next = |p: usize| { + let rank = lms_map[p] as usize; + if rank + 1 < m { + lms[rank + 1] as usize + } else { + n + } + }; + let end_l = next(l); + let end_r = next(r); + let mut same = true; + if end_l - l != end_r - r { + same = false; + } else { + while l < end_l { + if s[l] != s[r] { + break; + } + l += 1; + r += 1; + } + if l == n || s[l] != s[r] { + same = false; + } + } + if !same { + rec_upper += 1; + } + rec_s[lms_map[sorted_lms[i] as usize] as usize] = rec_upper; + } + + let rec_sa = sa_is(&rec_s, rec_upper as usize); + for (slot, &rank) in sorted_lms.iter_mut().zip(&rec_sa) { + *slot = lms[rank as usize]; + } + induce(&mut sa, &mut buf, &sorted_lms); + } + sa +} + +#[cfg(test)] +mod tests; diff --git a/zstd/src/dictionary/suffix_array/tests.rs b/zstd/src/dictionary/suffix_array/tests.rs new file mode 100644 index 000000000..63469bbf9 --- /dev/null +++ b/zstd/src/dictionary/suffix_array/tests.rs @@ -0,0 +1,57 @@ +use super::*; + +/// The array a comparison sort gives: the definition, not an algorithm. +fn naive(text: &[u8]) -> Vec { + let mut sa: Vec = (0..text.len() as u32).collect(); + sa.sort_by(|&a, &b| text[a as usize..].cmp(&text[b as usize..])); + sa +} + +fn lcg_bytes(seed: u64, len: usize, alphabet: u8) -> Vec { + let mut state = seed; + (0..len) + .map(|_| { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + ((state >> 33) % u64::from(alphabet)) as u8 + }) + .collect() +} + +/// Induced sorting agrees with the definition on the shapes that exercise its +/// branches: tiny texts (the direct cases), runs of one byte (all L-type, no +/// LMS positions), periodic text (equal LMS substrings, so the recursion runs +/// on repeated names), small and full alphabets. +#[test] +fn induced_sorting_matches_the_definition() { + let fixed: [&[u8]; 9] = [ + b"", + b"a", + b"ab", + b"ba", + b"banana", + b"mississippi", + b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + b"abababababababababababababababab", + b"abracadabra abracadabra abracadabra", + ]; + for text in fixed { + assert_eq!(suffix_array(text), naive(text), "text {text:?}"); + } + for (seed, alphabet) in [(1u64, 2u8), (2, 3), (3, 4), (4, 26), (5, 255)] { + for len in [9usize, 10, 11, 40, 257, 4096] { + let text = lcg_bytes(seed, len, alphabet); + assert_eq!( + suffix_array(&text), + naive(&text), + "len {len}, alphabet {alphabet}" + ); + } + } + let mut periodic = Vec::new(); + for _ in 0..300 { + periodic.extend_from_slice(b"tenant=demo key="); + } + assert_eq!(suffix_array(&periodic), naive(&periodic)); +} diff --git a/zstd/src/encoding/sequence_capture.rs b/zstd/src/encoding/sequence_capture.rs index b27ad8ee7..3d570808b 100644 --- a/zstd/src/encoding/sequence_capture.rs +++ b/zstd/src/encoding/sequence_capture.rs @@ -17,7 +17,7 @@ //! and its consumer bench only produce the data, not the labels. //! //! Implementation goes through [`FrameCompressor::new_with_matcher`] + -//! a [`CapturingMatcher`] wrapper rather than driving the matcher in +//! a `CapturingMatcher` wrapper rather than driving the matcher in //! isolation, so the captured stream reflects block-splitter decisions, //! strategy-tag selection and per-level resets exactly as the //! production encoder would emit them. Capturing the matcher in @@ -251,11 +251,11 @@ impl Matcher for CapturingMatcher { /// `compressed.len() >= MAX_BLOCK_SIZE`. The capture would then /// contain phantom triples whose on-wire form has no sequences. To /// prevent silently misaligned output, this function parses the -/// emitted frame's block headers (RFC 8878 §3.1.1.2.2) via -/// [`detect_raw_or_rle_blocks_in_frame`] and panics with a clear -/// diagnostic if any Raw_Block or RLE_Block is present. Callers -/// see a hard failure instead of a misleading capture -/// (PR #149 review #25). +/// emitted frame's block headers (RFC 8878 §3.1.1.2.2) and panics with +/// a clear diagnostic if any Raw_Block or RLE_Block is present, or if +/// the frame holds more blocks than the matcher was asked for (a +/// post-split frame). Callers see a hard failure instead of a +/// misleading capture. pub fn compress_and_collect_sequences(input: &[u8], level: CompressionLevel) -> SequenceCapture { compress_and_collect_sequences_impl(input, level, None, None) } diff --git a/zstd/src/lib.rs b/zstd/src/lib.rs index deefb9446..d93fc0eb6 100644 --- a/zstd/src/lib.rs +++ b/zstd/src/lib.rs @@ -252,6 +252,20 @@ pub mod testing { crate::dictionary::dict_roundtrip_fixture() } + /// The content the legacy trainer selects, before any header: facade for + /// the `ffi-bench` test that compares it with the tail of the reference's + /// `ZDICT_trainFromBuffer_legacy` dictionary. `None` when the trainer + /// yields nothing. + #[cfg(feature = "dict-builder")] + pub fn legacy_dict_content( + samples: &[u8], + sample_sizes: &[usize], + dict_size: usize, + selectivity: u32, + ) -> Option> { + crate::dictionary::legacy_dict_content(samples, sample_sizes, dict_size, selectivity) + } + pub use crate::blocks::block::BlockType; /// First block's type (raw / rle / compressed) in a frame. Facade over the From 5a4ccef31fd78bb15d56780f2ca8f2eaa4e4e634 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 25 Sep 2026 22:33:37 +0300 Subject: [PATCH 06/36] perf(dictionary): word compares and unchecked induction in legacy The legacy trainer spent its time where the reference's does, in the suffix-neighbourhood walk and the suffix-array induction, but paid more for each step: - The shared-prefix count compared a byte at a time; it now compares a word at a time and locates the first differing byte from the XOR, as ZDICT_count does. - Covered runs are marked with one fill rather than a bounds-checked write per position. - The SA-IS induction sweeps index by the bucket layout's invariants instead of checking every access, and reject an empty slot and position 0 with one compare of v - 1 against n. Each invariant is a debug_assert, so the debug suite checks it on every construction. The dictionaries are byte-identical to the previous build's. Part of #128 --- zstd/src/dictionary/legacy.rs | 47 +++++++++++++++++++++-------- zstd/src/dictionary/suffix_array.rs | 47 +++++++++++++++++++++++------ 2 files changed, 72 insertions(+), 22 deletions(-) diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index a39c30e08..ad00eb953 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -95,12 +95,27 @@ impl Corpus { Some(u64::from_le_bytes(word.try_into().expect("eight bytes"))) } - /// Bytes `a` and `b` have in common (`ZDICT_count`). + /// Bytes `a` and `b` have in common (`ZDICT_count`), compared a word at + /// a time as the reference compares them. + #[inline] fn common(&self, a: usize, b: usize) -> usize { - let (Some(left), Some(right)) = (self.bytes.get(a..), self.bytes.get(b..)) else { + let bytes = self.bytes.as_slice(); + let Some(limit) = bytes.len().checked_sub(a.max(b)) else { return 0; }; - left.iter().zip(right).take_while(|(x, y)| x == y).count() + let word = |at: usize| u64::from_le_bytes(bytes[at..at + 8].try_into().expect("8 bytes")); + let mut n = 0; + while n + 8 <= limit { + let diff = word(a + n) ^ word(b + n); + if diff != 0 { + return n + (diff.trailing_zeros() / 8) as usize; + } + n += 8; + } + while n < limit && bytes[a + n] == bytes[b + n] { + n += 1; + } + n } } @@ -127,9 +142,11 @@ impl Suffixes { /// the noise. #[inline] fn at(&self, at: i64) -> usize { - usize::try_from(at + 1) - .ok() - .and_then(|slot| self.padded.get(slot)) + // A walk stops at rank -1 at the lowest, where the padding holds the + // noise position. + debug_assert!(at >= -1); + self.padded + .get((at + 1) as usize) .copied() .unwrap_or(self.noise) as usize } @@ -261,6 +278,16 @@ fn mark(done: &mut [bool], at: usize) { } } +/// Mark `len` positions from `at` covered, as far as the marks reach. Both +/// are bounded by the corpus length, so the sum cannot overflow. +#[inline] +fn mark_run(done: &mut [bool], at: usize, len: usize) { + let end = (at + len).min(done.len()); + if at < end { + done[at..end].fill(true); + } +} + /// `ZDICT_analyzePos`: the segment the suffix of rank `start` leads to, or an /// empty one. Every position the analysis settles is marked in `done`. fn analyze_position( @@ -289,9 +316,7 @@ fn analyze_position( { pattern_end += 1; } - for u in 1..pattern_end { - mark(done, pos + u); - } + mark_run(done, pos + 1, pattern_end - 1); return empty; } @@ -415,9 +440,7 @@ fn analyze_position( } else { corpus.common(pos, tested).min(max_length) }; - for p in tested..tested + length { - mark(done, p); - } + mark_run(done, tested, length); } solution } diff --git a/zstd/src/dictionary/suffix_array.rs b/zstd/src/dictionary/suffix_array.rs index 3de21ce48..e00f00aa8 100644 --- a/zstd/src/dictionary/suffix_array.rs +++ b/zstd/src/dictionary/suffix_array.rs @@ -117,21 +117,48 @@ fn sa_is(s: &[T], upper: usize) -> Vec { let c = s[n - 1].index(); sa[buf[c] as usize] = (n - 1) as u32; buf[c] += 1; + // The two induction sweeps are the whole cost of the construction, one + // random write per suffix. A slot holding the empty mark or position 0 + // has no predecessor to induce, and one unsigned compare of `v - 1` + // against `n` rejects both. Every index below is in bounds by the + // bucket layout: a position is below `n`, and each bucket's cursor + // stays inside the bucket its symbol's suffixes fill. for i in 0..n { - let v = sa[i]; - if v != EMPTY && v >= 1 && !ls[v as usize - 1] { - let c = s[v as usize - 1].index(); - sa[buf[c] as usize] = v - 1; - buf[c] += 1; + // SAFETY: `i < n == sa.len()`. + let p = unsafe { *sa.get_unchecked(i) }.wrapping_sub(1) as usize; + if p < n { + // SAFETY: `p < n == ls.len() == s.len()`. + if !unsafe { *ls.get_unchecked(p) } { + let c = unsafe { s.get_unchecked(p) }.index(); + debug_assert!(c < buf.len() && (buf[c] as usize) < n); + // SAFETY: `c <= upper` (symbols lie in `0..=upper` and + // `buf.len() == upper + 1`); `buf[c] < n` by the layout. + unsafe { + let slot = buf.get_unchecked_mut(c); + *sa.get_unchecked_mut(*slot as usize) = p as u32; + *slot += 1; + } + } } } buf.copy_from_slice(&sum_l); for i in (0..n).rev() { - let v = sa[i]; - if v != EMPTY && v >= 1 && ls[v as usize - 1] { - let c = s[v as usize - 1].index() + 1; - buf[c] -= 1; - sa[buf[c] as usize] = v - 1; + // SAFETY: `i < n == sa.len()`. + let p = unsafe { *sa.get_unchecked(i) }.wrapping_sub(1) as usize; + if p < n { + // SAFETY: `p < n == ls.len() == s.len()`. + if unsafe { *ls.get_unchecked(p) } { + let c = unsafe { s.get_unchecked(p) }.index() + 1; + debug_assert!(c < buf.len()); + // SAFETY: an S-type symbol is below `upper`, so + // `c <= upper`; the bucket's end cursor is above its start. + unsafe { + let slot = buf.get_unchecked_mut(c); + *slot -= 1; + debug_assert!((*slot as usize) < n); + *sa.get_unchecked_mut(*slot as usize) = p as u32; + } + } } } }; From 126128daa5e799c1ee70e0ba83514d2403ba6083 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 25 Sep 2026 22:41:44 +0300 Subject: [PATCH 07/36] perf(dictionary): SA-IS induction without the type array The induction sweeps follow Yuta Mori's sais.c induceSA: the predecessor's type is decided when a suffix is written, from the neighbouring symbol, and kept in the top bit of the entry, so a sweep no longer reads a separate type array at a random position per suffix; the bucket cursor stays in a register while the symbol does not change. runner1 (x86, task-clock), --train-legacy -B4096 over the 16 MB decodecorpus set, prebuilt bench-profile binaries interleaved with zstd 1.5.7, three rounds of perf stat -r 3: before 3804-3855 ms after 3503-3507 ms zstd 1.5.7 2318-2326 ms Dictionaries byte-identical to the previous build's. Positions now need 31 bits, well above the trainer's 2000 MiB corpus cap. Part of #128 --- zstd/src/dictionary/suffix_array.rs | 117 ++++++++++++++++++---------- 1 file changed, 78 insertions(+), 39 deletions(-) diff --git a/zstd/src/dictionary/suffix_array.rs b/zstd/src/dictionary/suffix_array.rs index e00f00aa8..149a2623d 100644 --- a/zstd/src/dictionary/suffix_array.rs +++ b/zstd/src/dictionary/suffix_array.rs @@ -41,12 +41,12 @@ impl Symbol for u32 { /// /// # Panics /// -/// If `text` is `u32::MAX` bytes or longer, since positions are held as `u32` -/// with one value kept free as the empty mark. +/// If `text` is 2 GiB or longer: positions are held as `u32` whose top bit +/// marks an entry during construction. pub(crate) fn suffix_array(text: &[u8]) -> Vec { assert!( - text.len() < EMPTY as usize, - "a suffix array of {} bytes does not fit u32 positions", + text.len() < 1 << 31, + "a suffix array of {} bytes does not fit 31-bit positions", text.len() ); sa_is(text, usize::from(u8::MAX)) @@ -99,66 +99,105 @@ fn sa_is(s: &[T], upper: usize) -> Vec { } } - let mut sa = vec![EMPTY; n]; + // Bucket ends: one past the last slot of each symbol's bucket. + let ends: Vec = (0..=upper) + .map(|c| if c < upper { sum_l[c + 1] } else { n as u32 }) + .collect(); + let mut sa = vec![0u32; n]; let mut buf = vec![0u32; upper + 1]; + // The induction sweeps are the whole cost of the construction: one random + // write per suffix. They follow Yuta Mori's `sais.c` (`induceSA`): whether + // a suffix's predecessor is to be induced in the sweep that reads it is + // decided when the suffix is written, from the symbol before it, which is + // next to the one just read, and kept as the complement of the position + // (`!j`, the top bit set). The L sweep complements every entry it reads, + // which turns exactly the entries whose predecessor is S-type into the + // live ones for the S sweep; the S sweep restores the rest. The bucket + // cursor stays in a register while the symbol does not change. + // + // Every index below is in bounds by the bucket layout: a position is below + // `n`, a symbol lies in `0..=upper`, and each cursor stays inside the + // bucket its symbol's suffixes fill. + let live = |v: u32| (v as i32) > 0; let induce = |sa: &mut [u32], buf: &mut [u32], lms: &[u32]| { - sa.fill(EMPTY); + sa.fill(0); buf.copy_from_slice(&sum_s); for &d in lms { let d = d as usize; if d == n { continue; } + // An LMS suffix's predecessor is L-type: live for the L sweep. let c = s[d].index(); sa[buf[c] as usize] = d as u32; buf[c] += 1; } + buf.copy_from_slice(&sum_l); - let c = s[n - 1].index(); - sa[buf[c] as usize] = (n - 1) as u32; - buf[c] += 1; - // The two induction sweeps are the whole cost of the construction, one - // random write per suffix. A slot holding the empty mark or position 0 - // has no predecessor to induce, and one unsigned compare of `v - 1` - // against `n` rejects both. Every index below is in bounds by the - // bucket layout: a position is below `n`, and each bucket's cursor - // stays inside the bucket its symbol's suffixes fill. + let last = n - 1; + let mut c1 = s[last].index(); + let mut b = buf[c1] as usize; + sa[b] = if s[last - 1] < s[last] { + !(last as u32) + } else { + last as u32 + }; + b += 1; for i in 0..n { // SAFETY: `i < n == sa.len()`. - let p = unsafe { *sa.get_unchecked(i) }.wrapping_sub(1) as usize; - if p < n { - // SAFETY: `p < n == ls.len() == s.len()`. - if !unsafe { *ls.get_unchecked(p) } { - let c = unsafe { s.get_unchecked(p) }.index(); - debug_assert!(c < buf.len() && (buf[c] as usize) < n); - // SAFETY: `c <= upper` (symbols lie in `0..=upper` and - // `buf.len() == upper + 1`); `buf[c] < n` by the layout. + let v = unsafe { *sa.get_unchecked(i) }; + unsafe { *sa.get_unchecked_mut(i) = !v }; + if live(v) { + let j = v as usize - 1; + // SAFETY: `j < n == s.len()`. + let c0 = unsafe { s.get_unchecked(j) }.index(); + if c0 != c1 { + debug_assert!(c0 < buf.len() && c1 < buf.len()); + // SAFETY: both symbols lie in `0..=upper`. unsafe { - let slot = buf.get_unchecked_mut(c); - *sa.get_unchecked_mut(*slot as usize) = p as u32; - *slot += 1; + *buf.get_unchecked_mut(c1) = b as u32; + b = *buf.get_unchecked(c0) as usize; } + c1 = c0; } + // `j` is L-type; its predecessor is live for this sweep when it + // is L-type as well, which here means not smaller. + let dead = j > 0 && unsafe { s.get_unchecked(j - 1) }.index() < c1; + debug_assert!(b < n); + // SAFETY: `b` stays inside bucket `c1`. + unsafe { *sa.get_unchecked_mut(b) = if dead { !(j as u32) } else { j as u32 } }; + b += 1; } } - buf.copy_from_slice(&sum_l); + + buf.copy_from_slice(&ends); + let mut c1 = 0usize; + let mut b = buf[0] as usize; for i in (0..n).rev() { // SAFETY: `i < n == sa.len()`. - let p = unsafe { *sa.get_unchecked(i) }.wrapping_sub(1) as usize; - if p < n { - // SAFETY: `p < n == ls.len() == s.len()`. - if unsafe { *ls.get_unchecked(p) } { - let c = unsafe { s.get_unchecked(p) }.index() + 1; - debug_assert!(c < buf.len()); - // SAFETY: an S-type symbol is below `upper`, so - // `c <= upper`; the bucket's end cursor is above its start. + let v = unsafe { *sa.get_unchecked(i) }; + if live(v) { + let j = v as usize - 1; + // SAFETY: `j < n == s.len()`. + let c0 = unsafe { s.get_unchecked(j) }.index(); + if c0 != c1 { + debug_assert!(c0 < buf.len() && c1 < buf.len()); + // SAFETY: both symbols lie in `0..=upper`. unsafe { - let slot = buf.get_unchecked_mut(c); - *slot -= 1; - debug_assert!((*slot as usize) < n); - *sa.get_unchecked_mut(*slot as usize) = p as u32; + *buf.get_unchecked_mut(c1) = b as u32; + b = *buf.get_unchecked(c0) as usize; } + c1 = c0; } + // `j` is S-type; its predecessor is live when it is S-type as + // well, which here means not larger. + let dead = j == 0 || unsafe { s.get_unchecked(j - 1) }.index() > c1; + debug_assert!(b > 0); + b -= 1; + // SAFETY: `b` stays inside bucket `c1`. + unsafe { *sa.get_unchecked_mut(b) = if dead { !(j as u32) } else { j as u32 } }; + } else { + unsafe { *sa.get_unchecked_mut(i) = !v }; } } }; From fde314fd13cf9cb1ccd842c1a8206727e41718ff Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 25 Sep 2026 22:48:43 +0300 Subject: [PATCH 08/36] perf(dictionary): name LMS substrings inside the suffix array The LMS naming follows sais.c. The sorted LMS positions are compacted to the front of the array, told apart by the type array rather than a rank map; each substring's end is found by a short forward scan; names go to m + p / 2 and are gathered in text order at the back. The per-position rank map (four bytes a position, read at random twice per substring) is gone. A substring reaching the end of the text is taken to be unique, which keeps the comparison inside the text by construction. runner1 (x86, task-clock), --train-legacy -B4096 over the 16 MB decodecorpus set, interleaved with zstd 1.5.7, three rounds of perf stat -r 3: before 3464-3561 ms after 3283-3320 ms zstd 1.5.7 2324-2373 ms Dictionaries byte-identical. The new test sorts 2000 short texts over two and three symbols against the definition; the previous build passes it as well. Part of #128 --- zstd/src/dictionary/suffix_array.rs | 109 +++++++++++----------- zstd/src/dictionary/suffix_array/tests.rs | 14 +++ 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/zstd/src/dictionary/suffix_array.rs b/zstd/src/dictionary/suffix_array.rs index 149a2623d..593311e42 100644 --- a/zstd/src/dictionary/suffix_array.rs +++ b/zstd/src/dictionary/suffix_array.rs @@ -202,68 +202,73 @@ fn sa_is(s: &[T], upper: usize) -> Vec { } }; - // The leftmost S-type positions, and each one's rank among them. - let mut lms_map = vec![EMPTY; n + 1]; - let mut lms = Vec::new(); - for i in 1..n { - if !ls[i - 1] && ls[i] { - lms_map[i] = lms.len() as u32; - lms.push(i as u32); - } - } + // The leftmost S-type positions, in text order. + let is_lms = |p: usize| p > 0 && p < n && ls[p] && !ls[p - 1]; + let lms: Vec = (1..n).filter(|&i| is_lms(i)).map(|i| i as u32).collect(); let m = lms.len(); induce(&mut sa, &mut buf, &lms); if m > 0 { - let mut sorted_lms: Vec = sa - .iter() - .copied() - .filter(|&v| v != EMPTY && lms_map[v as usize] != EMPTY) - .collect(); - // Name each LMS substring by rank, equal substrings sharing a name, and - // sort the string of names recursively. - let mut rec_s = vec![0u32; m]; - let mut rec_upper = 0u32; - rec_s[lms_map[sorted_lms[0] as usize] as usize] = 0; - for i in 1..m { - let mut l = sorted_lms[i - 1] as usize; - let mut r = sorted_lms[i] as usize; - let next = |p: usize| { - let rank = lms_map[p] as usize; - if rank + 1 < m { - lms[rank + 1] as usize - } else { - n - } - }; - let end_l = next(l); - let end_r = next(r); - let mut same = true; - if end_l - l != end_r - r { - same = false; - } else { - while l < end_l { - if s[l] != s[r] { - break; - } - l += 1; - r += 1; - } - if l == n || s[l] != s[r] { - same = false; - } + // Name each LMS substring by rank, equal substrings sharing a name, + // and sort the string of names recursively. Laid out in `sa` itself as + // `sais.c` lays it out: the sorted LMS positions compacted to the + // front, each one's name at `m + p / 2` (LMS positions are at least two + // apart, so the slots are distinct and lie past the first `m`), then + // the names gathered in text order at the back. A substring runs from + // its LMS position to the next one, or to the end of the text. + let mut k = 0; + for i in 0..n { + let v = sa[i] as usize; + if is_lms(v) { + sa[k] = v as u32; + k += 1; + } + } + debug_assert_eq!(k, m); + sa[m..].fill(EMPTY); + let substring_end = |p: usize| { + let mut end = p + 1; + while end < n && !is_lms(end) { + end += 1; } + end + }; + let mut name = 0u32; + let mut prev = sa[0] as usize; + let mut prev_end = substring_end(prev); + debug_assert!(m + prev / 2 < n); + sa[m + prev / 2] = 0; + for i in 1..m { + let cur = sa[i] as usize; + let cur_end = substring_end(cur); + // Equal when as long and equal symbol for symbol, the symbol after + // included. One that runs into the end of the text ends at the + // virtual sentinel, which nothing else reaches, so it is unique. + let same = cur_end - cur == prev_end - prev + && cur_end < n + && prev_end < n + && s[cur..=cur_end] == s[prev..=prev_end]; if !same { - rec_upper += 1; + name += 1; } - rec_s[lms_map[sorted_lms[i] as usize] as usize] = rec_upper; + debug_assert!(m + cur / 2 < n); + sa[m + cur / 2] = name; + prev = cur; + prev_end = cur_end; } - - let rec_sa = sa_is(&rec_s, rec_upper as usize); - for (slot, &rank) in sorted_lms.iter_mut().zip(&rec_sa) { - *slot = lms[rank as usize]; + let mut j = n; + for i in (m..n).rev() { + if sa[i] != EMPTY { + j -= 1; + sa[j] = sa[i]; + } } + debug_assert_eq!(j, n - m); + let rec_s = sa[n - m..].to_vec(); + + let rec_sa = sa_is(&rec_s, name as usize); + let sorted_lms: Vec = rec_sa.iter().map(|&rank| lms[rank as usize]).collect(); induce(&mut sa, &mut buf, &sorted_lms); } sa diff --git a/zstd/src/dictionary/suffix_array/tests.rs b/zstd/src/dictionary/suffix_array/tests.rs index 63469bbf9..cc15bccbf 100644 --- a/zstd/src/dictionary/suffix_array/tests.rs +++ b/zstd/src/dictionary/suffix_array/tests.rs @@ -55,3 +55,17 @@ fn induced_sorting_matches_the_definition() { } assert_eq!(suffix_array(&periodic), naive(&periodic)); } + +/// Short texts over two or three symbols, by the thousand: the shapes where an +/// LMS substring running into the end of the text has the length of another, +/// which naming has to tell apart without reading past the end, and where the +/// recursion goes several levels deep. +#[test] +fn many_short_texts_over_tiny_alphabets_sort_correctly() { + for seed in 0..2000u64 { + let alphabet = 2 + (seed % 2) as u8; + let len = 10 + (seed % 40) as usize; + let text = lcg_bytes(seed.wrapping_mul(0x9E37_79B9), len, alphabet); + assert_eq!(suffix_array(&text), naive(&text), "seed {seed}: {text:?}"); + } +} From fa57789e938f697c3a5c6309a7a707fa83ebd8bb Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 25 Sep 2026 22:54:03 +0300 Subject: [PATCH 09/36] perf(dictionary): drop the padded suffix-array copy in legacy The trainer copied the whole suffix array into a vector one slot longer at each end, for the two ranks that lead into the noise band. Those ranks are now answered by one compare instead: a negative rank wraps past every real one. The merge scan also reads the eight bytes after the new segment's start once rather than once per table entry, and walks the table as a slice. runner1 (x86, task-clock), --train-legacy -B4096 over the 16 MB decodecorpus set, interleaved with zstd 1.5.7, three rounds of perf stat -r 3: before 3280-3336 ms after 3080-3149 ms zstd 1.5.7 2323-2378 ms Dictionaries byte-identical. Part of #128 --- zstd/src/dictionary/legacy.rs | 101 +++++++++++++++++----------------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index ad00eb953..d5a898b10 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -124,31 +124,25 @@ impl Corpus { /// `suffix[bufferSize]`): a walk off either end compares against noise and /// stops there. struct Suffixes { - padded: Vec, - noise: u32, + sa: Vec, + noise: usize, } impl Suffixes { - fn new(sa: Vec, len: u32) -> Self { - let mut padded = Vec::with_capacity(sa.len() + 2); - padded.push(len); - padded.extend_from_slice(&sa); - padded.push(len); - Self { padded, noise: len } + fn new(sa: Vec, len: usize) -> Self { + Self { sa, noise: len } } - /// `suffix[at]`, for `at` in `-1..=len`; any rank further out is the noise - /// band as well, which a walk can only reach on a corpus whose tail matches - /// the noise. + /// `suffix[at]`, where ranks `-1` and `len` (and any further out, which a + /// walk only reaches on a corpus whose tail matches the noise) are the + /// noise band. A negative rank wraps past every real one, so a single + /// compare tells the two apart. #[inline] fn at(&self, at: i64) -> usize { - // A walk stops at rank -1 at the lowest, where the padding holds the - // noise position. - debug_assert!(at >= -1); - self.padded - .get((at + 1) as usize) - .copied() - .unwrap_or(self.noise) as usize + match self.sa.get(at as u64 as usize) { + Some(&pos) => pos as usize, + None => self.noise, + } } } @@ -242,7 +236,7 @@ fn find_segments(list: &mut [DictItem], corpus: &Corpus, len: usize, min_rep: u3 for (at, &pos) in sa.iter().enumerate() { rank[pos as usize] = at as u32; } - let suffixes = Suffixes::new(sa, len as u32); + let suffixes = Suffixes::new(sa, len); // Slack past the corpus, as the reference allocates it: a covered run may // be marked into the noise band. let mut done = vec![false; len + 16]; @@ -473,57 +467,66 @@ fn try_merge(list: &mut [DictItem], elt: DictItem, skip: usize, corpus: &Corpus) let size = list[0].pos as usize; let elt_end = elt.pos + elt.length; - // An existing entry starts inside `elt`: extend it backwards. - for u in 1..size { - if u == skip { - continue; - } - if list[u].pos > elt.pos && list[u].pos <= elt_end { - let added = list[u].pos - elt.pos; - list[u].length += added; - list[u].pos = elt.pos; - list[u].savings = list[u] - .savings - .wrapping_add(elt.savings.wrapping_mul(added) / elt.length); - list[u].savings = list[u].savings.wrapping_add(elt.length / 8); - return promote(list, u); + // An existing entry starts inside `elt`: extend it backwards. The scans + // walk the table as a slice, and the rare skipped entry is tested last. + let mut hit = 0; + for (u, item) in list[..size].iter().enumerate().skip(1) { + if item.pos > elt.pos && item.pos <= elt_end && u != skip { + hit = u; + break; } } - - // `elt` starts inside an existing entry, or right after a copy of it. + if hit != 0 { + let item = &mut list[hit]; + let added = item.pos - elt.pos; + item.length += added; + item.pos = elt.pos; + item.savings = item + .savings + .wrapping_add(elt.savings.wrapping_mul(added) / elt.length); + item.savings = item.savings.wrapping_add(elt.length / 8); + return promote(list, hit); + } + + // `elt` starts inside an existing entry, or right after a copy of it. The + // eight bytes after `elt`'s start are the same for every entry, so they + // are read once rather than per entry as the reference reads them. + let elt_head = corpus.read64(elt.pos as usize + 1); for u in 1..size { + let item = list[u]; if u == skip { continue; } - if list[u].pos + list[u].length >= elt.pos && list[u].pos < elt.pos { - let added = elt_end as i64 - i64::from(list[u].pos + list[u].length); - list[u].savings = list[u].savings.wrapping_add(elt.length / 8); + if item.pos + item.length >= elt.pos && item.pos < elt.pos { + let added = elt_end as i64 - i64::from(item.pos + item.length); + let item = &mut list[u]; + item.savings = item.savings.wrapping_add(elt.length / 8); if added > 0 { - list[u].length += added as u32; - list[u].savings = list[u] + item.length += added as u32; + item.savings = item .savings .wrapping_add(elt.savings.wrapping_mul(added as u32) / elt.length); } return promote(list, u); } - let head = corpus.read64(list[u].pos as usize); - if head.is_some() - && head == corpus.read64(elt.pos as usize + 1) + if elt_head.is_some() + && corpus.read64(item.pos as usize) == elt_head && is_included( corpus, - list[u].pos as usize, + item.pos as usize, elt.pos as usize + 1, - list[u].length as usize, + item.length as usize, ) { // The reference takes this product at pointer width, where it does // not wrap, unlike the two above. - let added = (i64::from(elt.length) - i64::from(list[u].length)).max(1) as u64; - list[u].pos = elt.pos; - list[u].savings = list[u] + let added = (i64::from(elt.length) - i64::from(item.length)).max(1) as u64; + let item = &mut list[u]; + item.pos = elt.pos; + item.savings = item .savings .wrapping_add((u64::from(elt.savings) * added / u64::from(elt.length)) as u32); - list[u].length = elt.length.min(list[u].length + 1); + item.length = elt.length.min(item.length + 1); return u; } } From f09e0d0c3b8cc30bc24191b8989b3cf048ae2dc8 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 09:55:01 +0300 Subject: [PATCH 10/36] perf(encoding): derive split-probe length codes once per block Each post-split probe copied its range of sequences, resolved their offsets into the copy, computed every length and offset code, and then had the three table selections compute the codes and count them again: six passes over the range per probe. Upstream codes the block once (ZSTD_seqToCodes) and a probe only counts. - The literal- and match-length codes and a prefix sum of their extra bits are derived once per block beside the existing prefix sums; a probe histograms its slices and reads its extra bits in O(1). - Offsets, the one part that depends on the probe's entry history, are resolved in a single pass without a copy. - The table selection takes the histograms already built. Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary. Also records at the post-split gate that turning the pass off is not a speed lever. runner1 (x86, task-clock), encode_loop_z000033 against ffi_encode_loop_z000033, 50 frames of z000033[..200000], interleaved, three rounds of perf stat -r 3, ms: level before after libzstd L12 (control) 481.6-483.8 481.2-482.3 412.0-413.4 L13 1191-1205 1173-1177 876-880 L15 1288-1300 1263-1280 1187-1202 L16 1982-1987 1948-1954 1626-1637 L19 2919-2947 2867-2874 2358-2377 The probe's own time fell from 7.9% to 3.6% of the L13 profile. Part of #128 --- zstd/src/encoding/blocks/compressed.rs | 256 ++++++++++--------- zstd/src/encoding/blocks/compressed/tests.rs | 40 ++- zstd/src/encoding/levels/fastest.rs | 6 + 3 files changed, 184 insertions(+), 118 deletions(-) diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index b5ee79762..69c86ceba 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -208,14 +208,58 @@ impl CompressedBlockScratch { struct SequencePrefixSums { lit: Vec, ml: Vec, + /// Each sequence's literal-length and match-length codes, and a prefix sum + /// of the extra bits the two carry. Neither depends on the offset history, + /// so they are derived once per block (upstream zstd `ZSTD_seqToCodes`) and + /// every split probe reads its range from here. + ll_code: Vec, + ml_code: Vec, + length_bits: Vec, +} + +/// The precomputed length codes of a run of sequences. +#[derive(Clone, Copy)] +struct LengthCodes<'a> { + ll: &'a [u8], + ml: &'a [u8], + /// Extra bits of every literal and match length in the run. + bits: usize, } impl SequencePrefixSums { fn heap_size(&self) -> usize { - (self.lit.capacity() + self.ml.capacity()) * core::mem::size_of::() + (self.lit.capacity() + self.ml.capacity() + self.length_bits.capacity()) + * core::mem::size_of::() + + self.ll_code.capacity() + + self.ml_code.capacity() + } + + /// The length codes of sequences `start..end`. + fn codes(&self, start: usize, end: usize) -> LengthCodes<'_> { + LengthCodes { + ll: &self.ll_code[start..end], + ml: &self.ml_code[start..end], + bits: self.length_bits[end] - self.length_bits[start], + } } fn rebuild(&mut self, sequences: &[RawSequence]) { + self.ll_code.clear(); + self.ml_code.clear(); + self.length_bits.clear(); + self.ll_code.reserve(sequences.len()); + self.ml_code.reserve(sequences.len()); + self.length_bits.reserve(sequences.len() + 1); + let mut bits = 0usize; + self.length_bits.push(0); + for seq in sequences { + let (ll, _, ll_bits) = encode_literal_length(seq.ll); + let (ml, _, ml_bits) = encode_match_len(seq.ml); + self.ll_code.push(ll); + self.ml_code.push(ml); + bits += ll_bits + ml_bits; + self.length_bits.push(bits); + } self.lit.clear(); self.ml.clear(); // `Vec::reserve_exact(additional)` adds `additional` elements ABOVE @@ -830,15 +874,12 @@ struct EstimatorWorkspace { ll_counts: Box<[usize; 256]>, ml_counts: Box<[usize; 256]>, of_counts: Box<[usize; 256]>, - sequences: Vec, } impl EstimatorWorkspace { - /// The four boxed count tables plus whatever the sequence buffer has grown - /// to. All four boxes are always present once the workspace exists. + /// The four boxed count tables, always present once the workspace exists. fn heap_size(&self) -> usize { 4 * core::mem::size_of::<[usize; 256]>() - + self.sequences.capacity() * core::mem::size_of::() } } @@ -849,7 +890,6 @@ impl Default for EstimatorWorkspace { ll_counts: Box::new([0; 256]), ml_counts: Box::new([0; 256]), of_counts: Box::new([0; 256]), - sequences: Vec::new(), } } } @@ -862,33 +902,33 @@ impl Default for EstimatorWorkspace { /// FSE bit-level write. Splitter probes use this path to get the same byte /// count `encode_block_parts` would produce while saving the dominant /// `encode_sequences` write cost on every probe. +#[cfg(test)] fn estimate_block_parts_size( state: &mut CompressState, literals_vec: &[u8], raw_sequences: &[RawSequence], workspace: &mut EstimatorWorkspace, ) -> usize { - // The probe cannot fill in place: it walks sub-ranges of the block's - // sequences repeatedly, from a scratch history, while the array itself is - // borrowed immutably by the estimator for the whole search. So it keeps a - // copy — which costs nothing that matters, since block splitting only runs - // from level 11 up and never on the band this array's copy was hurting. - workspace.sequences.clear(); - if workspace.sequences.capacity() < raw_sequences.len() { - workspace - .sequences - .reserve_exact(raw_sequences.len() - workspace.sequences.len()); - } - workspace.sequences.extend_from_slice(raw_sequences); - fill_wire_offsets( - &mut workspace.sequences, - &mut state.offset_hist, - matches!( - state.strategy_tag, - crate::encoding::strategy::StrategyTag::Fast - ), - ); + let mut sums = SequencePrefixSums::default(); + sums.rebuild(raw_sequences); + estimate_block_parts_size_with( + state, + literals_vec, + raw_sequences, + sums.codes(0, raw_sequences.len()), + workspace, + ) +} +/// [`estimate_block_parts_size`] for sequences whose length codes are already +/// derived: the splitter's probes, which price many ranges of one block. +fn estimate_block_parts_size_with( + state: &mut CompressState, + literals_vec: &[u8], + raw_sequences: &[RawSequence], + codes: LengthCodes<'_>, + workspace: &mut EstimatorWorkspace, +) -> usize { let lit_bytes = estimate_literals_section_bytes( literals_vec, &mut state.last_huff_table, @@ -900,15 +940,15 @@ fn estimate_block_parts_size( literals_suspected_incompressible(literals_vec.len(), raw_sequences.len()), ); - let seq_bytes = if workspace.sequences.is_empty() { + let seq_bytes = if raw_sequences.is_empty() { 1 } else { estimate_sequences_section_bytes( - &workspace.sequences, + raw_sequences, + codes, + &mut state.offset_hist, &mut state.fse_tables, - &mut workspace.ll_counts, - &mut workspace.ml_counts, - &mut workspace.of_counts, + workspace, state.strategy_tag, ) }; @@ -1091,28 +1131,65 @@ fn estimate_literals_section_bytes( total } +/// Price a sequence section. The offset codes are the one part that depends on +/// the history the section starts from, so they are resolved here, in one pass +/// that advances `offset_hist` as the emitter would; the length codes and +/// their extra bits come precomputed in `codes`. Each histogram is built once +/// and handed to the table selection, which the emitter's path counts again. fn estimate_sequences_section_bytes( sequences: &[RawSequence], + codes: LengthCodes<'_>, + offset_hist: &mut [u32; 3], fse_tables: &mut FseTables, - ll_counts: &mut [usize; 256], - ml_counts: &mut [usize; 256], - of_counts: &mut [usize; 256], + workspace: &mut EstimatorWorkspace, strategy: crate::encoding::strategy::StrategyTag, ) -> usize { - ll_counts.fill(0); - ml_counts.fill(0); + let EstimatorWorkspace { + ll_counts, + ml_counts, + of_counts, + .. + } = workspace; + let (ll_counts, ml_counts, of_counts) = (&mut **ll_counts, &mut **ml_counts, &mut **of_counts); + debug_assert_eq!(codes.ll.len(), sequences.len()); + let histogram = |codes: &[u8], counts: &mut [usize; 256]| { + counts.fill(0); + let mut max = 0u8; + for &code in codes { + counts[code as usize] += 1; + max = max.max(code); + } + max as usize + }; + let ll_max = histogram(codes.ll, ll_counts); + let ml_max = histogram(codes.ml, ml_counts); of_counts.fill(0); - let mut extra_bits: usize = 0; - for seq in sequences { - let (ll, _, ll_bits) = encode_literal_length(seq.ll); - let (ml, _, ml_bits) = encode_match_len(seq.ml); - let (of, _, _) = encode_offset(seq.off_base); - ll_counts[ll as usize] += 1; - ml_counts[ml as usize] += 1; + let mut of_max = 0usize; + let mut of_bits = 0usize; + let mut hist = *offset_hist; + let mut count_offset = |off_base: u32| { + let (of, _, _) = encode_offset(off_base); of_counts[of as usize] += 1; + of_max = of_max.max(of as usize); // Upstream zstd: OF code's value equals its additional-bits width. - extra_bits += ll_bits + ml_bits + of as usize; + of_bits += of as usize; + }; + if matches!(strategy, crate::encoding::strategy::StrategyTag::Fast) { + for seq in sequences { + count_offset(encode_offset_with_history_fast( + seq.off_base, + seq.ll, + &mut hist, + )); + } + } else { + for seq in sequences { + count_offset(encode_offset_with_history(seq.off_base, seq.ll, &mut hist)); + } } + *offset_hist = hist; + let extra_bits = codes.bits + of_bits; + let total = sequences.len(); // Destructured for the same reason as the emitter: the default accessors // borrow the whole struct, which would collide with the `*_next` slots. @@ -1131,30 +1208,38 @@ fn estimate_sequences_section_bytes( let ml_default: &FSETable = ml_default; let of_default: &FSETable = of_default; - // Same `choose_table` calls as the real encoder — counts the iterator - // internally, identical decision path. - let ll_mode = choose_table( + // The table selection the real encoder makes, from the same histograms. + let ll_mode = choose_table_from_counts( ll_previous.as_ref(), ll_default, - sequences.iter().map(|seq| encode_literal_length(seq.ll).0), + ll_counts, + total, + ll_max, 9, strategy, + None, ll_next, ); - let ml_mode = choose_table( + let ml_mode = choose_table_from_counts( ml_previous.as_ref(), ml_default, - sequences.iter().map(|seq| encode_match_len(seq.ml).0), + ml_counts, + total, + ml_max, 9, strategy, + None, ml_next, ); - let of_mode = choose_table( + let of_mode = choose_table_from_counts( of_previous.as_ref(), of_default, - sequences.iter().map(|seq| encode_offset(seq.off_base).0), + of_counts, + total, + of_max, 8, strategy, + None, of_next, ); @@ -1687,27 +1772,6 @@ fn highest_used_code(counts: &[usize; 256]) -> usize { .unwrap_or(0) } -/// [`fill_and_count`] without the histogram, for the block-split estimator: it -/// prices sub-ranges repeatedly from a scratch history and counts them itself. -fn fill_wire_offsets( - raw_sequences: &mut [RawSequence], - offset_hist: &mut [u32; 3], - fast_repcode: bool, -) { - // Local copy for the same reason as `fill_and_count`. - let mut hist = *offset_hist; - if fast_repcode { - for seq in raw_sequences.iter_mut() { - seq.off_base = encode_offset_with_history_fast(seq.off_base, seq.ll, &mut hist); - } - } else { - for seq in raw_sequences.iter_mut() { - seq.off_base = encode_offset_with_history(seq.off_base, seq.ll, &mut hist); - } - } - *offset_hist = hist; -} - fn clone_fse_tables(fse_tables: &FseTables) -> FseTables { // The `*_default` fields are cfg-typed via the // [`crate::fse::fse_encoder::FseDefaultTable`] alias — @@ -1799,10 +1863,11 @@ impl SplitEstimator<'_> { self.scratch_state.fse_tables.ml_previous = entry.ml_previous.clone(); self.scratch_state.fse_tables.of_previous = entry.of_previous.clone(); self.scratch_state.offset_hist = entry.offset_hist; - let emitted_payload = estimate_block_parts_size( + let emitted_payload = estimate_block_parts_size_with( &mut self.scratch_state, &self.parts.literals[lit_start..lit_end], &self.parts.sequences[start_idx..end_idx], + self.prefix_sums.codes(start_idx, end_idx), &mut self.workspace, ); let source_len = (lit_end - lit_start) + match_len; @@ -2045,47 +2110,8 @@ fn fse_bit_cost(counts: &[usize; 256], max_symbol: usize, table: &FSETable) -> O Some(cost >> 8) } -fn choose_table<'a>( - previous: Option<&'a PreviousFseTable>, - default_table: &'a FSETable, - data: impl Iterator, - max_log: u8, - strategy: crate::encoding::strategy::StrategyTag, - next: &'a mut Option, -) -> FseTableMode<'a> { - // Collect symbol distribution, tracking the highest code so the selector - // skips the full-256 reverse scan (see `choose_table_from_counts`). - let mut counts = [0usize; 256]; - let mut total = 0usize; - let mut max_symbol = 0usize; - for symbol in data { - let symbol = symbol as usize; - counts[symbol] += 1; - total += 1; - max_symbol = max_symbol.max(symbol); - } - choose_table_from_counts( - previous, - default_table, - &mut counts, - total, - max_symbol, - max_log, - strategy, - // Estimator-only path (no emitted table): price the unadjusted histogram, - // matching upstream's `ZSTD_NCountCost`. - None, - next, - ) -} - -/// Same decision logic as [`choose_table`] but takes pre-computed -/// symbol counts and total directly. Hot-path callers in -/// `compress_literals_and_sequences` use this overload to avoid -/// re-iterating the sequence vec three times (one pass per -/// ll/ml/of stream); the iterator form is kept for the cost -/// estimator's call sites where the data is already in iterator -/// form. +/// Choose an FSE table mode from a stream's symbol counts, which every caller +/// has already built while visiting the codes once. // The eight inputs are the cohesive FSE-table-selection set, each carrying its // own perf / correctness rationale below (the `&mut` histogram for the no-copy // emit build, the caller-tracked `max_symbol` / `last_code` that avoid a diff --git a/zstd/src/encoding/blocks/compressed/tests.rs b/zstd/src/encoding/blocks/compressed/tests.rs index a09228bdf..fca09ac79 100644 --- a/zstd/src/encoding/blocks/compressed/tests.rs +++ b/zstd/src/encoding/blocks/compressed/tests.rs @@ -1,14 +1,48 @@ use super::{ - FseTableMode, LastUsedTable, RawSequence, choose_table, emit_single_sequence_block, + FseTableMode, LastUsedTable, RawSequence, choose_table_from_counts, emit_single_sequence_block, encode_literal_length, encode_match_len, encode_offset_with_history, min_gain, min_literals_to_compress, previous_table, remember_last_used_tables, }; -use crate::encoding::frame_compressor::{CompressState, FseTables, PreviousFseTable}; +use crate::encoding::frame_compressor::{ + CompressState, FseTables, PreviousFseTable, SharedFseTable, +}; use crate::encoding::strategy::StrategyTag; -use crate::fse::fse_encoder::build_table_from_symbol_counts; +use crate::fse::fse_encoder::{FSETable, build_table_from_symbol_counts}; use crate::huff0::huff0_encoder; use alloc::vec::Vec; +/// The table selection for a stream given as codes: counts them, then selects +/// as the encoder does, pricing the unadjusted histogram. +fn choose_table<'a>( + previous: Option<&'a PreviousFseTable>, + default_table: &'a FSETable, + data: impl Iterator, + max_log: u8, + strategy: StrategyTag, + next: &'a mut Option, +) -> FseTableMode<'a> { + let mut counts = [0usize; 256]; + let mut total = 0usize; + let mut max_symbol = 0usize; + for symbol in data { + let symbol = symbol as usize; + counts[symbol] += 1; + total += 1; + max_symbol = max_symbol.max(symbol); + } + choose_table_from_counts( + previous, + default_table, + &mut counts, + total, + max_symbol, + max_log, + strategy, + None, + next, + ) +} + fn tables_match( lhs: &crate::fse::fse_encoder::FSETable, rhs: &crate::fse::fse_encoder::FSETable, diff --git a/zstd/src/encoding/levels/fastest.rs b/zstd/src/encoding/levels/fastest.rs index d79c7d7c8..a3d113183 100644 --- a/zstd/src/encoding/levels/fastest.rs +++ b/zstd/src/encoding/levels/fastest.rs @@ -572,6 +572,12 @@ pub(crate) fn compress_block_encoded_borrowed( /// (`zstd_compress.c`, `ZSTD_resolveBlockSplitterMode`: `strategy >= btopt && /// windowLog >= 17`), so a parameter set that moves the strategy moves the /// pass with it. +/// +/// Turning the pass off is not a speed lever. Gated by level it was skipped on +/// the btopt / btultra frames of L13-15, which ran faster there only by paying +/// 6-9% in bytes (z000033[..200000], L13: 90378 against 85017 with the pass). +/// Where the pass costs time, the cost is in the splitter itself, and that is +/// what has to get cheaper. #[inline] fn post_split_enabled(strategy_tag: StrategyTag, window_size: u64) -> bool { matches!( From d2a3570a2e36d05f94af7c8a5328fe3189f32a19 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 10:08:08 +0300 Subject: [PATCH 11/36] perf(encoding): borrow the split probes' Huffman table Each post-split probe copied the Huffman table it could repeat into a scratch state, let the pricing rewrite that copy, and copied the result again as the state the next probe starts from: two table copies per probe, plus one per block to seed the entry state. The table is never changed in place; a section only keeps it, drops it, or builds a new one. Pricing now reads the previous table by reference and reports which of the three it did. A probe state names its table instead of holding it: the block's entry table borrowed from the compressor, or one a probe built, kept once in the estimator and returned to the weight builder when the block is done. The FSE repeat handles move out of the scratch state rather than being cloned. Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary. Fewer operations, no measurable time change: runner1 (x86, task-clock), 50 frames of z000033[..200000], interleaved with the previous build, three rounds of perf stat -r 3, the ranges overlap at every level (L13 1168-1173 ms against 1169-1175; L12, L15, L16, L19 likewise). Part of #128 --- zstd/src/encoding/blocks/compressed.rs | 163 +++++++++++++++++-------- 1 file changed, 109 insertions(+), 54 deletions(-) diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 69c86ceba..c734b1f8e 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -428,12 +428,18 @@ pub(crate) fn compress_block_with_post_split( parts: &scratch.parts, prefix_sums: &scratch.prefix_sums, block_entry: ProbeEntryState { - last_huff_table: state.last_huff_table.clone(), + huff: if state.last_huff_table.is_some() { + HuffRef::BlockEntry + } else { + HuffRef::None + }, ll_previous: state.fse_tables.ll_previous.clone(), ml_previous: state.fse_tables.ml_previous.clone(), of_previous: state.fse_tables.of_previous.clone(), offset_hist: state.offset_hist, }, + entry_huff: state.last_huff_table.as_ref(), + built_huff: Vec::new(), scratch_state: CompressState { matcher: EntropyOnlyMatcher, // The splitter's scratch state never reaches the raw-skip, which @@ -442,7 +448,9 @@ pub(crate) fn compress_block_with_post_split( // Inherited rather than re-resolved: this scratch state stands in // for the same compressor on the same CPU. copy_tier: state.copy_tier, - last_huff_table: state.last_huff_table.clone(), + // Probes read the table they repeat from `entry_huff` / + // `built_huff`; this slot stays empty. + last_huff_table: None, huff_table_spare: None, huff_rollback: None, // Lent, not created: the estimator builds a table per split @@ -469,9 +477,13 @@ pub(crate) fn compress_block_with_post_split( // builder's buffers back so the emitter and the next block reuse them. let CompressState { block_scratch: inner_block_scratch, - huff_weights, + mut huff_weights, .. } = estimator.scratch_state; + // The tables the probes built go back to the builder for its next tables. + for table in estimator.built_huff { + huff_weights.recycle(table); + } state.huff_weights = huff_weights; scratch.estimator_inner = Some(Box::new(inner_block_scratch)); @@ -911,27 +923,49 @@ fn estimate_block_parts_size( ) -> usize { let mut sums = SequencePrefixSums::default(); sums.rebuild(raw_sequences); - estimate_block_parts_size_with( + let previous = state.last_huff_table.take(); + let (bytes, outcome) = estimate_block_parts_size_with( state, + previous.as_ref(), literals_vec, raw_sequences, sums.codes(0, raw_sequences.len()), workspace, - ) + ); + state.last_huff_table = match outcome { + HuffOutcome::Keep => previous, + HuffOutcome::Clear => None, + HuffOutcome::New(table) => Some(table), + }; + bytes +} + +/// What pricing a literals section did to the Huffman table the next section +/// may repeat: keep the one it was given, drop it, or take a new one. A probe +/// reads the previous table and reports this, rather than rewriting a copy of +/// it, so the table it starts from is borrowed. +enum HuffOutcome { + Keep, + Clear, + New(huff0_encoder::HuffmanTable), } /// [`estimate_block_parts_size`] for sequences whose length codes are already -/// derived: the splitter's probes, which price many ranges of one block. +/// derived: the splitter's probes, which price many ranges of one block. The +/// Huffman table the section may repeat is `previous`, and what the section +/// does with it is returned beside the size; the FSE repeat tables and the +/// offset history are advanced in `state`. fn estimate_block_parts_size_with( state: &mut CompressState, + previous: Option<&huff0_encoder::HuffmanTable>, literals_vec: &[u8], raw_sequences: &[RawSequence], codes: LengthCodes<'_>, workspace: &mut EstimatorWorkspace, -) -> usize { - let lit_bytes = estimate_literals_section_bytes( +) -> (usize, HuffOutcome) { + let (lit_bytes, outcome) = estimate_literals_section_bytes( literals_vec, - &mut state.last_huff_table, + previous, &mut workspace.lit_counts, state.strategy_tag, state.huf_optimal_search, @@ -953,7 +987,7 @@ fn estimate_block_parts_size_with( ) }; - lit_bytes + seq_bytes + (lit_bytes + seq_bytes, outcome) } // One argument over the lint's threshold. Every one of them is a distinct @@ -963,20 +997,25 @@ fn estimate_block_parts_size_with( #[allow(clippy::too_many_arguments)] fn estimate_literals_section_bytes( literals: &[u8], - last_huff: &mut Option, + last_huff: Option<&huff0_encoder::HuffmanTable>, counts: &mut [usize; 256], strategy: crate::encoding::strategy::StrategyTag, huf_search: bool, lit_disabled: bool, weight_scratch: &mut huff0_encoder::WeightScratch, suspected_incompressible: bool, -) -> usize { +) -> (usize, HuffOutcome) { + let raw = || { + ( + uncompressed_literals_header_bytes(literals.len()) + literals.len(), + HuffOutcome::Clear, + ) + }; // Mirror `encode_block_parts` literal-mode branches // **in the same order**. The disabled gate (negative levels: raw literals, // no Huffman) is checked FIRST exactly as the emitter does. if lit_disabled { - *last_huff = None; - return uncompressed_literals_header_bytes(literals.len()) + literals.len(); + return raw(); } // The emitter pre-checks `all_identical` // (any non-empty section) BEFORE the `min_lits` gate — RLE and raw @@ -987,13 +1026,14 @@ fn estimate_literals_section_bytes( // regardless of strategy. Estimator must use the same ordering and // predicate so probe costs match emit byte-for-byte. if !literals.is_empty() && all_bytes_identical(literals) { - *last_huff = None; - return uncompressed_literals_header_bytes(literals.len()) + 1; + return ( + uncompressed_literals_header_bytes(literals.len()) + 1, + HuffOutcome::Clear, + ); } let min_lits = min_literals_to_compress(strategy, last_huff.is_some()); if literals.len() < min_lits { - *last_huff = None; - return uncompressed_literals_header_bytes(literals.len()) + literals.len(); + return raw(); } // Upstream zstd preferRepeat fast-path: skip the histogram + @@ -1011,18 +1051,16 @@ fn estimate_literals_section_bytes( // short-circuit so we still fall through to rebuild when the // prior table can't encode the current literals. if prefer_repeat_eligible(strategy, literals.len()) - && let Some(prev) = last_huff.as_ref() + && let Some(prev) = last_huff && let Some(reuse_payload) = estimate_huff_payload_bytes_checked(prev, literals) { let compressed_header = compressed_literals_header_bytes(literals.len()); let total = compressed_header + reuse_payload; // no tree_desc on reuse - let raw_section_bytes = uncompressed_literals_header_bytes(literals.len()) + literals.len(); let huf_section_size = total - compressed_header; if use_raw_literal_fallback(huf_section_size, literals.len(), strategy) { - *last_huff = None; - return raw_section_bytes; + return raw(); } - return total; + return (total, HuffOutcome::Keep); } // Mirror the emitter's end-sample shortcut, in the same position. Without @@ -1030,8 +1068,7 @@ fn estimate_literals_section_bytes( // Huffman-compressed here and is emitted raw there, and the splitter picks // a partition on a price the emitter cannot produce. if suspected_incompressible && end_samples_look_flat(literals, counts) { - *last_huff = None; - return uncompressed_literals_header_bytes(literals.len()) + literals.len(); + return raw(); } let (max_sym, largest_count) = crate::histogram::count_bytes(literals, counts); @@ -1039,8 +1076,7 @@ fn estimate_literals_section_bytes( // byte-for-byte (flat histogram → raw section, no tree build) so // splitter probe costs match what the emitter writes. if largest_count <= (literals.len() >> 7) + 4 { - *last_huff = None; - return uncompressed_literals_header_bytes(literals.len()) + literals.len(); + return raw(); } // Mutable because the size query is what encodes the weight description // into the table's own buffer, so the emitter that follows reads it. @@ -1051,11 +1087,10 @@ fn estimate_literals_section_bytes( ); let Some(new_desc) = new_table.writeable_table_description_size() else { - *last_huff = None; // Nothing downstream reads this table; hand its buffers to the next // build rather than dropping them. weight_scratch.recycle(new_table); - return uncompressed_literals_header_bytes(literals.len()) + literals.len(); + return raw(); }; // For lit_size ≥ 256, upstream zstd `compress_literals` calls `encoder.encode4x` // which splits the data in 4 streams with a 6-byte jumptable and per-stream @@ -1072,20 +1107,12 @@ fn estimate_literals_section_bytes( // Using the 4-stream `estimate_huff_payload_bytes_checked` here would // disagree with the encoder and bias the splitter to pick a different // table than the encoder ultimately emits. - let use_new = decide_huff_reuse_like_encoder( - &new_table, - last_huff.as_ref(), - new_desc, - literals, - counts, - strategy, - ); + let use_new = + decide_huff_reuse_like_encoder(&new_table, last_huff, new_desc, literals, counts, strategy); let reuse_payload = if !use_new { // Safe to recompute with 4-stream model now that the table is chosen: // the chosen-table path always returns the actual wire cost. - last_huff - .as_ref() - .and_then(|t| estimate_huff_payload_bytes_checked(t, literals)) + last_huff.and_then(|t| estimate_huff_payload_bytes_checked(t, literals)) } else { None }; @@ -1115,20 +1142,16 @@ fn estimate_literals_section_bytes( let raw_section_bytes = uncompressed_literals_header_bytes(literals.len()) + literals.len(); let huf_section_size = total - compressed_header; // tree_desc + payload, no lhSize if use_raw_literal_fallback(huf_section_size, literals.len(), strategy) { - *last_huff = None; weight_scratch.recycle(new_table); - return raw_section_bytes; + return (raw_section_bytes, HuffOutcome::Clear); } if use_new { - // The table this displaces is the one to recycle; the new one is kept. - if let Some(displaced) = last_huff.replace(new_table) { - weight_scratch.recycle(displaced); - } + (total, HuffOutcome::New(new_table)) } else { weight_scratch.recycle(new_table); + (total, HuffOutcome::Keep) } - total } /// Price a sequence section. The offset codes are the one part that depends on @@ -1821,17 +1844,31 @@ fn clone_fse_tables(fse_tables: &FseTables) -> FseTables { /// estimator replaces. #[derive(Clone)] struct ProbeEntryState { - last_huff_table: Option, + huff: HuffRef, ll_previous: Option, ml_previous: Option, of_previous: Option, offset_hist: [u32; 3], } +/// Which Huffman table a probe state may repeat. Tables are never copied into +/// a state: the block's entry table is borrowed from the compressor, and a +/// table a probe builds is kept once in the estimator's arena. +#[derive(Clone, Copy)] +enum HuffRef { + None, + BlockEntry, + Built(usize), +} + struct SplitEstimator<'a> { parts: &'a EncodedBlockParts, prefix_sums: &'a SequencePrefixSums, block_entry: ProbeEntryState, + /// The table the block starts from, borrowed. + entry_huff: Option<&'a huff0_encoder::HuffmanTable>, + /// Every table a probe built, addressed by [`HuffRef::Built`]. + built_huff: Vec, scratch_state: CompressState, workspace: EstimatorWorkspace, } @@ -1858,13 +1895,20 @@ impl SplitEstimator<'_> { } else { lit_start + lit_len }; - self.scratch_state.last_huff_table = entry.last_huff_table.clone(); + // The FSE repeat tables are shared handles, so seeding them is a + // reference-count bump; the Huffman table is only borrowed. self.scratch_state.fse_tables.ll_previous = entry.ll_previous.clone(); self.scratch_state.fse_tables.ml_previous = entry.ml_previous.clone(); self.scratch_state.fse_tables.of_previous = entry.of_previous.clone(); self.scratch_state.offset_hist = entry.offset_hist; - let emitted_payload = estimate_block_parts_size_with( + let previous = match entry.huff { + HuffRef::None => None, + HuffRef::BlockEntry => self.entry_huff, + HuffRef::Built(at) => Some(&self.built_huff[at]), + }; + let (emitted_payload, outcome) = estimate_block_parts_size_with( &mut self.scratch_state, + previous, &self.parts.literals[lit_start..lit_end], &self.parts.sequences[start_idx..end_idx], self.prefix_sums.codes(start_idx, end_idx), @@ -1881,13 +1925,24 @@ impl SplitEstimator<'_> { // Real emit on raw fallback restores the entry state — see // `emit_single_sequence_block`'s saved-state restore branch. let post = if raw_fallback { + if let HuffOutcome::New(table) = outcome { + self.scratch_state.huff_weights.recycle(table); + } entry.clone() } else { + let huff = match outcome { + HuffOutcome::Keep => entry.huff, + HuffOutcome::Clear => HuffRef::None, + HuffOutcome::New(table) => { + self.built_huff.push(table); + HuffRef::Built(self.built_huff.len() - 1) + } + }; ProbeEntryState { - last_huff_table: self.scratch_state.last_huff_table.clone(), - ll_previous: self.scratch_state.fse_tables.ll_previous.clone(), - ml_previous: self.scratch_state.fse_tables.ml_previous.clone(), - of_previous: self.scratch_state.fse_tables.of_previous.clone(), + huff, + ll_previous: self.scratch_state.fse_tables.ll_previous.take(), + ml_previous: self.scratch_state.fse_tables.ml_previous.take(), + of_previous: self.scratch_state.fse_tables.of_previous.take(), offset_hist: self.scratch_state.offset_hist, } }; From 0a15716fedce30f3e99ef784c99810c77bbb5fd1 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 11:56:17 +0300 Subject: [PATCH 12/36] perf(encoding): build FSE tables in place without full-width clears Three copies of an FSE encoder table (about 13 KiB each) sat on the entropy path, most visibly under the post-split probes: - every table build cleared all 256 per-symbol entries past the new alphabet, although only what an earlier build wrote there can be non-default; the table now tracks that extent and clears only it - each Huffman weight description built its FSE table by value and moved it into the encoder; the encoder now borrows the table, and the weight builder's scratch keeps one to build into - each probe dropped its build slots and so built every new table on the stack and moved it into a fresh shared handle; handles no probe state holds any more now go to a pool in the estimator workspace, and a probe takes its build slots from it A rebuild into a used table also left an absent symbol's start state and bit width from the earlier build; they are now reset, with a regression test comparing a reused build against a fresh one. Part of #128 --- zstd/src/dictionary/mod.rs | 5 +- zstd/src/encoding/blocks/compressed.rs | 117 ++++++++++++++++--- zstd/src/encoding/blocks/compressed/tests.rs | 2 +- zstd/src/encoding/frame_compressor.rs | 2 +- zstd/src/fse/fse_encoder.rs | 36 ++++-- zstd/src/fse/mod.rs | 7 +- zstd/src/fse/tests.rs | 33 ++++++ zstd/src/huff0/huff0_decoder/tests.rs | 6 +- zstd/src/huff0/huff0_encoder.rs | 84 +++++++++---- zstd/src/huff0/huff0_encoder/tests.rs | 31 +++-- zstd/src/huff0/mod.rs | 2 +- 11 files changed, 254 insertions(+), 71 deletions(-) diff --git a/zstd/src/dictionary/mod.rs b/zstd/src/dictionary/mod.rs index 9339c37a2..3266ab601 100644 --- a/zstd/src/dictionary/mod.rs +++ b/zstd/src/dictionary/mod.rs @@ -364,7 +364,10 @@ fn serialize_huffman_table(sample_data: &[u8], raw_content: &[u8]) -> io::Result } let mut table = HuffmanEncoderTable::build_from_data(stats.as_slice()); - if table.writeable_table_description_size().is_none() { + if table + .writeable_table_description_size(&mut crate::fse::fse_encoder::FSETable::blank()) + .is_none() + { // Sampled real data can land on the same shape: a flat alphabet wider // than 128 symbols. Fall back to the synthetic narrow one, which always // has a description. diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index c734b1f8e..a5a9fee8f 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -471,15 +471,30 @@ pub(crate) fn compress_block_with_post_split( estimator.derive_block_splits(0, scratch.parts.sequences.len(), &mut scratch.partitions); scratch.partitions.push(scratch.parts.sequences.len()); workspace = estimator.workspace; - scratch.estimator_workspace = Some(workspace); // Stash the inner scratch back for the next frame (its buffers stay // allocated; the estimator clears them per use), and take the weight // builder's buffers back so the emitter and the next block reuse them. let CompressState { block_scratch: inner_block_scratch, mut huff_weights, + fse_tables: probe_tables, .. } = estimator.scratch_state; + // The last probe's tables join the pool for the next block's probes. + workspace.recycle_previous(probe_tables.ll_previous); + workspace.recycle_previous(probe_tables.ml_previous); + workspace.recycle_previous(probe_tables.of_previous); + for handle in [ + probe_tables.ll_next, + probe_tables.ml_next, + probe_tables.of_next, + ] + .into_iter() + .flatten() + { + workspace.recycle_fse(handle); + } + scratch.estimator_workspace = Some(workspace); // The tables the probes built go back to the builder for its next tables. for table in estimator.built_huff { huff_weights.recycle(table); @@ -886,12 +901,35 @@ struct EstimatorWorkspace { ll_counts: Box<[usize; 256]>, ml_counts: Box<[usize; 256]>, of_counts: Box<[usize; 256]>, + /// FSE table handles no probe state holds any more, for the next probe to + /// build into. Every one is unique, so a build writes it in place. + spare_fse: Vec, } impl EstimatorWorkspace { - /// The four boxed count tables, always present once the workspace exists. + /// The four boxed count tables, always present once the workspace exists, + /// and the pooled tables with their reference counts. fn heap_size(&self) -> usize { 4 * core::mem::size_of::<[usize; 256]>() + + self.spare_fse.capacity() * core::mem::size_of::() + + self.spare_fse.len() + * (core::mem::size_of::() + + crate::encoding::frame_compressor::shared_table_overhead()) + } + + /// Pool a handle, unless something else still holds it: a shared table + /// cannot be built into, so it is only let go. + fn recycle_fse(&mut self, mut handle: SharedFseTable) { + if SharedFseTable::get_mut(&mut handle).is_some() { + self.spare_fse.push(handle); + } + } + + /// Pool the custom tables of an axis state that is going away. + fn recycle_previous(&mut self, previous: Option) { + if let Some(PreviousFseTable::Custom(handle)) = previous { + self.recycle_fse(handle); + } } } @@ -902,6 +940,7 @@ impl Default for EstimatorWorkspace { ll_counts: Box::new([0; 256]), ml_counts: Box::new([0; 256]), of_counts: Box::new([0; 256]), + spare_fse: Vec::new(), } } } @@ -1086,7 +1125,9 @@ fn estimate_literals_section_bytes( weight_scratch, ); - let Some(new_desc) = new_table.writeable_table_description_size() else { + let Some(new_desc) = + new_table.writeable_table_description_size(weight_scratch.weight_fse_table()) + else { // Nothing downstream reads this table; hand its buffers to the next // build rather than dropping them. weight_scratch.recycle(new_table); @@ -1302,11 +1343,17 @@ fn estimate_sequences_section_bytes( // The emitter keeps the handle a commit displaces, to build the next // block's table into. A probe must not: the splitter holds many of these // states at once, and a spare per axis per probe doubles the tables alive - // at any moment. Dropping it leaves a probe with exactly what it needs, - // which is what the allocate-per-table form gave it. - fse_tables.ll_next = None; - fse_tables.ml_next = None; - fse_tables.of_next = None; + // at any moment. It goes to the workspace pool instead, which every probe + // draws its build slots from. + for next in [ + &mut fse_tables.ll_next, + &mut fse_tables.ml_next, + &mut fse_tables.of_next, + ] { + if let Some(handle) = next.take() { + workspace.recycle_fse(handle); + } + } nb_seq_header + mode_byte @@ -1896,10 +1943,33 @@ impl SplitEstimator<'_> { lit_start + lit_len }; // The FSE repeat tables are shared handles, so seeding them is a - // reference-count bump; the Huffman table is only borrowed. - self.scratch_state.fse_tables.ll_previous = entry.ll_previous.clone(); - self.scratch_state.fse_tables.ml_previous = entry.ml_previous.clone(); - self.scratch_state.fse_tables.of_previous = entry.of_previous.clone(); + // reference-count bump; the Huffman table is only borrowed. What the + // last probe left behind goes to the pool, and a build slot comes from + // it, so a probe that builds a table writes into one it already has. + let tables = &mut self.scratch_state.fse_tables; + for (previous, next, seed) in [ + ( + &mut tables.ll_previous, + &mut tables.ll_next, + &entry.ll_previous, + ), + ( + &mut tables.ml_previous, + &mut tables.ml_next, + &entry.ml_previous, + ), + ( + &mut tables.of_previous, + &mut tables.of_next, + &entry.of_previous, + ), + ] { + self.workspace + .recycle_previous(core::mem::replace(previous, seed.clone())); + if next.is_none() { + *next = self.workspace.spare_fse.pop(); + } + } self.scratch_state.offset_hist = entry.offset_hist; let previous = match entry.huff { HuffRef::None => None, @@ -1949,6 +2019,13 @@ impl SplitEstimator<'_> { (cost, raw_fallback, post) } + /// Pool the tables of a probe state nothing will read again. + fn recycle_state(&mut self, state: ProbeEntryState) { + self.workspace.recycle_previous(state.ll_previous); + self.workspace.recycle_previous(state.ml_previous); + self.workspace.recycle_previous(state.of_previous); + } + fn derive_block_splits( &mut self, start_idx: usize, @@ -1961,7 +2038,9 @@ impl SplitEstimator<'_> { return; } let entry = self.block_entry.clone(); - let (full, full_raw_fallback, _) = self.estimate_subblock_size(start_idx, end_idx, &entry); + let (full, full_raw_fallback, full_post) = + self.estimate_subblock_size(start_idx, end_idx, &entry); + self.recycle_state(full_post); // G3 — whole-block bail-out before partition split. Upstream zstd // `ZSTD_compressSubBlock_multi` (`zstd_compress_superblock.c:530-532`) // bails when `estBlockSize > srcSize` (strict). Our trigger is @@ -2007,7 +2086,8 @@ impl SplitEstimator<'_> { if full_raw_fallback { return; } - self.derive_block_splits_with_full(start_idx, end_idx, full, entry, partitions); + let exit = self.derive_block_splits_with_full(start_idx, end_idx, full, entry, partitions); + self.recycle_state(exit); } /// Returns the post-emit state at `end_idx` produced by whichever @@ -2030,6 +2110,7 @@ impl SplitEstimator<'_> { // exit state is the post-state of that single-partition probe. let (_cost, _raw_fallback, post) = self.estimate_subblock_size(start_idx, end_idx, &entry); + self.recycle_state(entry); return post; } let mid_idx = (start_idx + end_idx) / 2; @@ -2038,7 +2119,9 @@ impl SplitEstimator<'_> { // not from the parent's block-entry state. Without this propagation // `second` is evaluated as a fresh-block start, biasing the // `first + second < full` decision toward overly optimistic splits. - let (second, _, _) = self.estimate_subblock_size(mid_idx, end_idx, &first_post); + let (second, _, second_post) = self.estimate_subblock_size(mid_idx, end_idx, &first_post); + self.recycle_state(second_post); + self.recycle_state(first_post); if first + second < full { // If the left side gets further split, the true state at // `mid_idx` is the left subtree's exit state, not `first_post`. @@ -2055,6 +2138,7 @@ impl SplitEstimator<'_> { } // No split here — this range will be emitted as one partition. let (_cost, _raw_fallback, post) = self.estimate_subblock_size(start_idx, end_idx, &entry); + self.recycle_state(entry); post } } @@ -3177,7 +3261,8 @@ fn compress_literals( weight_scratch, ); - let Some(new_table_description_size) = new_encoder_table.writeable_table_description_size() + let Some(new_table_description_size) = + new_encoder_table.writeable_table_description_size(weight_scratch.weight_fse_table()) else { raw_literals(literals, writer); weight_scratch.recycle(new_encoder_table); diff --git a/zstd/src/encoding/blocks/compressed/tests.rs b/zstd/src/encoding/blocks/compressed/tests.rs index fca09ac79..0715512bd 100644 --- a/zstd/src/encoding/blocks/compressed/tests.rs +++ b/zstd/src/encoding/blocks/compressed/tests.rs @@ -232,7 +232,7 @@ fn decide_huff_reuse_prefer_repeat_forces_reuse_for_fast_band() { skewed_literals.extend((0..16u8).map(|i| 200 + i)); let mut new_tbl = huff0_encoder::HuffmanTable::build_from_data(&skewed_literals); let new_desc = new_tbl - .writeable_table_description_size() + .writeable_table_description_size(&mut crate::fse::fse_encoder::FSETable::blank()) .expect("non-empty table emits a description"); // The decision reads its sizes off the histogram of the very literals it diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 8bb03cbf7..f0393ca3a 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -443,7 +443,7 @@ pub(crate) type SharedFseTable = alloc::rc::Rc; /// reference counts, then whatever padding the table's alignment adds. One /// allocation holds both, so a caller sizing a context is told about the whole /// of it rather than the payload alone. -const fn shared_table_overhead() -> usize { +pub(crate) const fn shared_table_overhead() -> usize { let counts = 2 * core::mem::size_of::(); let align = core::mem::align_of::(); counts.div_ceil(align) * align diff --git a/zstd/src/fse/fse_encoder.rs b/zstd/src/fse/fse_encoder.rs index 4d6def096..95aaafbf0 100644 --- a/zstd/src/fse/fse_encoder.rs +++ b/zstd/src/fse/fse_encoder.rs @@ -1,21 +1,18 @@ use crate::bit_io::BitWriter; use alloc::vec::Vec; -pub(crate) struct FSEEncoder<'output, V: AsMut>> { - pub(super) table: FSETable, +/// Encodes a stream with a table the caller holds, so a table built into +/// reusable storage is never moved into the encoder. +pub(crate) struct FSEEncoder<'table, 'output, V: AsMut>> { + pub(super) table: &'table FSETable, writer: &'output mut BitWriter, } -impl>> FSEEncoder<'_, V> { - pub fn new(table: FSETable, writer: &mut BitWriter) -> FSEEncoder<'_, V> { +impl<'table, 'output, V: AsMut>> FSEEncoder<'table, 'output, V> { + pub fn new(table: &'table FSETable, writer: &'output mut BitWriter) -> Self { FSEEncoder { table, writer } } - #[cfg(any(test, feature = "fuzz-exports"))] - pub fn into_table(self) -> FSETable { - self.table - } - /// Encodes the data using the provided table /// Writes /// * Table description @@ -180,6 +177,10 @@ pub struct FSETable { pub(super) state_table_flat: [u16; MAX_FSE_TABLE_SIZE], /// Per-symbol upstream zstd-parity coding transform — see [`SymbolTT`]. pub(super) symbol_tt: [SymbolTT; 256], + /// How many leading entries of `states` / `symbol_tt` may hold a non-default + /// value; everything past it is default. Lets a rebuild clear only what the + /// previous build wrote beyond the new alphabet instead of the whole array. + live_symbols: usize, } impl FSETable { @@ -193,6 +194,7 @@ impl FSETable { table_size: 0, state_table_flat: [0u16; MAX_FSE_TABLE_SIZE], symbol_tt: [SymbolTT::default(); 256], + live_symbols: 0, } } @@ -842,15 +844,21 @@ pub(super) fn build_table_from_probabilities_into(probs: &[i32], acc_log: u8, ou states: symbol_states, state_table_flat, symbol_tt, + live_symbols, } = out; *out_table_size = table_size; // The destination may be a table from an earlier block: reset what the // build below does not overwrite in full. `state_table_flat` needs none of // it, the scatter fills every slot under `table_size` and nothing reads // past it; `symbol_tt` and `states` are written only for `probs.len()` - // symbols, so the tail above that has to be cleared rather than inherited. - symbol_states[probs.len()..].fill_with(SymbolStates::default); - symbol_tt[probs.len()..].fill(SymbolTT::default()); + // symbols, so what an earlier build wrote above that has to be cleared + // rather than inherited. Past `live_symbols` the arrays are already + // default, so the clear stops there. + if *live_symbols > probs.len() { + symbol_states[probs.len()..*live_symbols].fill_with(SymbolStates::default); + symbol_tt[probs.len()..*live_symbols].fill(SymbolTT::default()); + } + *live_symbols = probs.len(); // Upstream zstd `FSE_buildCTable_wksp` (lib/compress/fse_compress.c) — build // `nextStateTable` (== `state_table_flat`) once via cumul + spread + @@ -977,6 +985,10 @@ pub(super) fn build_table_from_probabilities_into(probs: &[i32], acc_log: u8, ou for (symbol, &prob) in probs.iter().enumerate() { symbol_states[symbol].probability = prob; if prob == 0 { + // A slot reused from an earlier build may still hold that build's + // start state and bit width for this symbol; an absent one has none. + symbol_states[symbol].start_state = None; + symbol_states[symbol].max_num_bits = None; // Upstream zstd fills `symbolTT` for prob==0 too, so `FSE_getMaxNbBits` // still works (returns `acc_log + 1` for absent symbols). // We don't expose that path, but mirror the value for parity. diff --git a/zstd/src/fse/mod.rs b/zstd/src/fse/mod.rs index d7bcc1e50..ae98719e4 100644 --- a/zstd/src/fse/mod.rs +++ b/zstd/src/fse/mod.rs @@ -49,14 +49,11 @@ pub fn round_trip(data: &[u8]) { } let mut writer = BitWriter::new(); - let mut encoder = FSEEncoder::new( - fse_encoder::build_table_from_data(data.iter().copied(), 6, false), - &mut writer, - ); + let enc_table = fse_encoder::build_table_from_data(data.iter().copied(), 6, false); + let mut encoder = FSEEncoder::new(&enc_table, &mut writer); let mut dec_table = FSETable::new(255); encoder.encode(data); let acc_log = encoder.acc_log(); - let enc_table = encoder.into_table(); let encoded = writer.dump(); let table_bytes = dec_table.build_decoder(&encoded, acc_log).unwrap(); diff --git a/zstd/src/fse/tests.rs b/zstd/src/fse/tests.rs index 112cd4341..d85a4e278 100644 --- a/zstd/src/fse/tests.rs +++ b/zstd/src/fse/tests.rs @@ -21,6 +21,39 @@ fn an_encoder_table_holds_nothing_on_the_heap() { ); } +#[test] +fn rebuilding_a_used_table_matches_a_fresh_build() { + // Block tables are rebuilt in place. A symbol the earlier build had and the + // new one lacks, inside the new alphabet or past it, must read as absent: + // the dictionary cost seed takes `max_num_bits_for_symbol` of an absent + // symbol as zero, and a stale width there prices it as codable. + let mut wide = [0usize; 64]; + for (symbol, count) in wide.iter_mut().enumerate() { + *count = symbol + 1; + } + let mut narrow = [0usize; 40]; + for (symbol, count) in narrow.iter_mut().enumerate() { + *count = if symbol == 5 { 0 } else { 2 * symbol + 3 }; + } + let mut reused = fse_encoder::FSETable::blank(); + fse_encoder::build_table_from_symbol_counts_into(&wide, 9, false, &mut reused); + fse_encoder::build_table_from_symbol_counts_into(&narrow, 9, false, &mut reused); + let fresh = fse_encoder::build_table_from_symbol_counts(&narrow, 9, false); + for symbol in 0..=255u8 { + assert_eq!( + reused.symbol_probability(symbol), + fresh.symbol_probability(symbol), + "probability of symbol {symbol}" + ); + assert_eq!( + reused.max_num_bits_for_symbol(symbol), + fresh.max_num_bits_for_symbol(symbol), + "max bits of symbol {symbol}" + ); + } + assert_eq!(reused.table_header_bits(), fresh.table_header_bits()); +} + #[test] fn decoder_entry_layout_is_four_bytes_for_huffman_weights() { assert_eq!(core::mem::size_of::(), 4); diff --git a/zstd/src/huff0/huff0_decoder/tests.rs b/zstd/src/huff0/huff0_decoder/tests.rs index 22a19a6c3..c39fbd0bd 100644 --- a/zstd/src/huff0/huff0_decoder/tests.rs +++ b/zstd/src/huff0/huff0_decoder/tests.rs @@ -45,10 +45,8 @@ fn build_decoder_rejects_fse_streams_with_256_explicit_weights() { for &w in &weights { counts[w as usize] += 1; } - let mut encoder = FSEEncoder::new( - build_table_from_symbol_counts(&counts, 6, false), - &mut writer, - ); + let table = build_table_from_symbol_counts(&counts, 6, false); + let mut encoder = FSEEncoder::new(&table, &mut writer); encoder.encode_interleaved(&weights); writer.flush(); } diff --git a/zstd/src/huff0/huff0_encoder.rs b/zstd/src/huff0/huff0_encoder.rs index 6f99a2cd4..9ed0897ca 100644 --- a/zstd/src/huff0/huff0_encoder.rs +++ b/zstd/src/huff0/huff0_encoder.rs @@ -1,3 +1,4 @@ +use alloc::boxed::Box; use alloc::vec::Vec; use core::cmp::Ordering; @@ -380,7 +381,8 @@ impl>> HuffmanEncoder<'_, '_, V> { let weights = self.weights_into(&mut buf); let weights = &weights[..weights.len() - 1]; let mut encoded = Vec::new(); - if Self::encode_weight_description_into(weights, &mut encoded) { + let mut fse_table = fse_encoder::FSETable::blank(); + if Self::encode_weight_description_into(weights, &mut encoded, &mut fse_table) { self.writer.write_bits(encoded.len() as u8, 8); self.writer.append_bytes(&encoded); } else { @@ -396,7 +398,8 @@ impl>> HuffmanEncoder<'_, '_, V> { let len = weights.len(); let weights = &buf[..len - 1]; let mut encoded = Vec::new(); - if Self::encode_weight_description_into(weights, &mut encoded) { + let mut fse_table = fse_encoder::FSETable::blank(); + if Self::encode_weight_description_into(weights, &mut encoded, &mut fse_table) { self.writer.write_bits(encoded.len() as u8, 8); self.writer.append_bytes(&encoded); } else { @@ -413,7 +416,13 @@ impl>> HuffmanEncoder<'_, '_, V> { /// rebuilt many times per frame reuses one allocation; on `false` its /// contents are meaningless and the raw nibble description is written /// instead. - fn encode_weight_description_into(weights: &[u8], encoded: &mut Vec) -> bool { + /// `fse_table` is where the weights' FSE table is built; the caller keeps it + /// between tables, as upstream keeps its `HUF_CompressWeightsWksp`. + fn encode_weight_description_into( + weights: &[u8], + encoded: &mut Vec, + fse_table: &mut fse_encoder::FSETable, + ) -> bool { encoded.clear(); if weights.len() <= 2 { return false; @@ -454,11 +463,9 @@ impl>> HuffmanEncoder<'_, '_, V> { encoded.reserve(want - encoded.len()); } { + fse_encoder::build_table_from_symbol_counts_into(&counts, 6, false, fse_table); let mut writer = BitWriter::from(&mut *encoded); - let mut encoder = FSEEncoder::new( - fse_encoder::build_table_from_symbol_counts(&counts, 6, false), - &mut writer, - ); + let mut encoder = FSEEncoder::new(fse_table, &mut writer); encoder.encode_interleaved(weights); writer.flush(); } @@ -817,15 +824,19 @@ impl HuffmanTable { /// std build path: consults the lazy cache to avoid re-encoding the /// weight stream when both planner and emitter call this for the /// same table. no_std build path: recomputes via the direct encoder - /// every call (cache field absent — preserves `Sync`). - pub(crate) fn try_table_description_size(&mut self) -> Option { + /// every call (cache field absent — preserves `Sync`). `fse_table` is the + /// caller's storage for the weights' FSE table. + pub(crate) fn try_table_description_size( + &mut self, + fse_table: &mut fse_encoder::FSETable, + ) -> Option { #[cfg(feature = "std")] { // Encodes on the first call for these contents and caches it, so // the writer that follows reads rather than repeats the work. This // is also where the caching happens at all: it is the only step in // the emit path holding the table mutably. - self.fill_weight_description_from_codes(); + self.fill_weight_description_from_codes(fse_table); if let Some(fse_description) = self.cached_encoded_weight_description() { return Some(fse_description.len() + 1); } @@ -843,7 +854,11 @@ impl HuffmanTable { let len = weights.len(); let weights = &buf[..len - 1]; let mut encoded = Vec::new(); - if HuffmanEncoder::>::encode_weight_description_into(weights, &mut encoded) { + if HuffmanEncoder::>::encode_weight_description_into( + weights, + &mut encoded, + fse_table, + ) { return Some(encoded.len() + 1); } if weights.len() <= 128 { @@ -855,8 +870,11 @@ impl HuffmanTable { } /// Alias for `try_table_description_size` used by call sites that require explicit writeability. - pub(crate) fn writeable_table_description_size(&mut self) -> Option { - self.try_table_description_size() + pub(crate) fn writeable_table_description_size( + &mut self, + fse_table: &mut fse_encoder::FSETable, + ) -> Option { + self.try_table_description_size(fse_table) } /// Owning form of [`Self::weights_into`], for tests that want the weights @@ -884,21 +902,24 @@ impl HuffmanTable { /// which is the whole point: the previous lazy form populated through /// `&self` and so had to hand the cache a freshly allocated one. #[cfg(feature = "std")] - fn fill_weight_description(&mut self, weights: &[u8]) { + fn fill_weight_description(&mut self, weights: &[u8], fse_table: &mut fse_encoder::FSETable) { if self.cached_encoded_weight_description.state != DescriptionState::NotComputed { return; } let cache = &mut self.cached_encoded_weight_description; - cache.state = - if HuffmanEncoder::>::encode_weight_description_into(weights, &mut cache.buf) { - DescriptionState::Encoded(cache.buf.len()) - } else { - DescriptionState::NotEncodable - }; + cache.state = if HuffmanEncoder::>::encode_weight_description_into( + weights, + &mut cache.buf, + fse_table, + ) { + DescriptionState::Encoded(cache.buf.len()) + } else { + DescriptionState::NotEncodable + }; } #[cfg(feature = "std")] - fn fill_weight_description_from_codes(&mut self) { + fn fill_weight_description_from_codes(&mut self, fse_table: &mut fse_encoder::FSETable) { // Before deriving the weights, not after: they cost a pass over the // whole alphabet, and a warm cache needs none of it. if self.cached_encoded_weight_description.state != DescriptionState::NotComputed { @@ -910,7 +931,7 @@ impl HuffmanTable { let weights = self.weights_into(&mut buf); let len = weights.len(); let weights = &buf[..len - 1]; - self.fill_weight_description(weights); + self.fill_weight_description(weights, fse_table); } /// The cached encoding, or `None` when the FSE form was rejected OR has not @@ -1579,6 +1600,9 @@ pub(crate) struct WeightScratch { /// The last table the caller built and threw away, handed back so the next /// build fills its buffers instead of taking new ones. spare_table: Option, + /// Where each table's weight description builds its FSE table. Taken on + /// first use and kept, so a description costs no table allocation or copy. + weight_fse_table: Option>, } impl WeightScratch { @@ -1595,12 +1619,22 @@ impl WeightScratch { + self.work.capacity() * core::mem::size_of::() + self.weights.capacity() * core::mem::size_of::() + self.spare_table.as_ref().map_or(0, HuffmanTable::heap_size) + + self + .weight_fse_table + .as_ref() + .map_or(0, |_| core::mem::size_of::()) } /// Park a table the caller is done with, for the next build to fill. pub(crate) fn recycle(&mut self, table: HuffmanTable) { self.spare_table = Some(table); } + + /// The storage a weight description builds its FSE table in. + pub(crate) fn weight_fse_table(&mut self) -> &mut fse_encoder::FSETable { + self.weight_fse_table + .get_or_insert_with(|| Box::new(fse_encoder::FSETable::blank())) + } } /// [`build_limited_weights`] filling caller-owned scratch. The weights land in @@ -1902,7 +1936,11 @@ pub(crate) fn huf_weight_description_for_test(data: &[u8]) -> (Vec, Vec) weights.pop(); let mut encoded = Vec::new(); assert!( - HuffmanEncoder::>::encode_weight_description_into(&weights, &mut encoded), + HuffmanEncoder::>::encode_weight_description_into( + &weights, + &mut encoded, + &mut fse_encoder::FSETable::blank(), + ), "expected FSE weights", ); let mut description = Vec::with_capacity(encoded.len() + 1); diff --git a/zstd/src/huff0/huff0_encoder/tests.rs b/zstd/src/huff0/huff0_encoder/tests.rs index 58c837739..3cb9e900d 100644 --- a/zstd/src/huff0/huff0_encoder/tests.rs +++ b/zstd/src/huff0/huff0_encoder/tests.rs @@ -88,7 +88,7 @@ fn the_cached_weight_description_counts_as_retained() { let before = table.heap_size(); let cached_len = table - .writeable_table_description_size() + .writeable_table_description_size(&mut fse_encoder::FSETable::blank()) .expect("the full-alphabet fixture caches an encoded description") - 1; @@ -404,7 +404,7 @@ fn cheap_desc_size_proxy_is_conservative_vs_exact() { // `try_table_description_size` trims internally; mirror that // on the proxy call so both score the same slice. let trimmed = &weights[..weights.len() - 1]; - let exact = table.try_table_description_size(); + let exact = table.try_table_description_size(&mut fse_encoder::FSETable::blank()); let proxy = cheap_desc_size_proxy(trimmed); match (proxy, exact) { (Some(p), Some(e)) => { @@ -523,6 +523,9 @@ fn fse_weight_descriptions_roundtrip() { // alphabet such as 4 symbols → weights [1,1,1] produced a description the // decoder rejected. let mut fails: Vec<(usize, u32, alloc::vec::Vec)> = alloc::vec::Vec::new(); + // One table for every case, as a compressor keeps one: each description + // is built over whatever the previous case left in it. + let mut fse_table = fse_encoder::FSETable::blank(); for card in 2usize..=255 { for skew in 0u32..4 { let mut data: Vec = Vec::new(); @@ -553,7 +556,11 @@ fn fse_weight_descriptions_roundtrip() { // for streams it actually FSE-encodes; None means it chose the raw // description (nothing to round-trip). Every Some MUST decode back. let mut encoded = Vec::new(); - if !HuffmanEncoder::>::encode_weight_description_into(&weights, &mut encoded) { + if !HuffmanEncoder::>::encode_weight_description_into( + &weights, + &mut encoded, + &mut fse_table, + ) { continue; } let mut description = Vec::with_capacity(encoded.len() + 1); @@ -608,7 +615,11 @@ fn large_alphabet_weight_description_uses_fse_when_raw_is_unrepresentable() { let mut encoded = Vec::new(); assert!( - HuffmanEncoder::>::encode_weight_description_into(&weights, &mut encoded), + HuffmanEncoder::>::encode_weight_description_into( + &weights, + &mut encoded, + &mut fse_encoder::FSETable::blank(), + ), "FSE weight description must be available when raw weights cannot be represented", ); let mut description = Vec::with_capacity(encoded.len() + 1); @@ -644,7 +655,7 @@ fn cached_encoded_weight_description_is_reused_for_write_table() { } let mut table = HuffmanTable::build_from_data(&data); let desc_size = table - .writeable_table_description_size() + .writeable_table_description_size(&mut fse_encoder::FSETable::blank()) .expect("table description must be writable"); let cached = table .cached_encoded_weight_description() @@ -674,7 +685,9 @@ fn flat_wide_alphabet_has_no_writeable_description() { let alphabet: Vec = (0u8..=255).collect(); let mut table = HuffmanTable::build_from_data(&alphabet); assert!( - table.writeable_table_description_size().is_none(), + table + .writeable_table_description_size(&mut fse_encoder::FSETable::blank()) + .is_none(), "a table this wide and this flat has no representation to write" ); } @@ -710,7 +723,11 @@ fn a_rejected_description_is_recorded_and_the_raw_form_written() { } // The size query is what encodes, and it is what a writer is preceded by. - assert!(table.writeable_table_description_size().is_some()); + assert!( + table + .writeable_table_description_size(&mut fse_encoder::FSETable::blank()) + .is_some() + ); assert_eq!( table.weight_description_state(), DescriptionState::NotEncodable, diff --git a/zstd/src/huff0/mod.rs b/zstd/src/huff0/mod.rs index f22db343b..3e0db2ee6 100644 --- a/zstd/src/huff0/mod.rs +++ b/zstd/src/huff0/mod.rs @@ -26,7 +26,7 @@ pub fn round_trip(data: &[u8]) { let mut writer = BitWriter::new(); let mut encoder_table = huff0_encoder::HuffmanTable::build_from_data(data); encoder_table - .writeable_table_description_size() + .writeable_table_description_size(&mut crate::fse::fse_encoder::FSETable::blank()) .expect("round_trip must only build Huffman tables with a writeable description"); let mut encoder = huff0_encoder::HuffmanEncoder::new(&encoder_table, &mut writer); From 3644bf0679e8b187bb6ba2c76cf8aee839dee4e1 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 12:04:44 +0300 Subject: [PATCH 13/36] build(fse): gate the owning table builders on their consumers The weight description now builds into caller storage, which left the owning builders with no caller in a build without tests, fuzz exports or the dictionary builder, and a dead-code warning on the no-std target. --- zstd/src/fse/fse_encoder.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zstd/src/fse/fse_encoder.rs b/zstd/src/fse/fse_encoder.rs index 95aaafbf0..e8cbbc347 100644 --- a/zstd/src/fse/fse_encoder.rs +++ b/zstd/src/fse/fse_encoder.rs @@ -425,6 +425,7 @@ pub fn build_table_from_data( build_table_from_counts(&counts[..=max_symbol], max_log, avoid_0_numbit) } +#[cfg(any(test, feature = "fuzz-exports", feature = "dict-builder"))] pub(crate) fn build_table_from_symbol_counts( counts: &[usize], max_log: u8, @@ -498,6 +499,7 @@ pub(crate) fn build_seq_ctable_into( build_table_from_probabilities_into(&probs[..=max_symbol], table_log, out); } +#[cfg(any(test, feature = "fuzz-exports", feature = "dict-builder"))] fn build_table_from_counts(counts: &[usize], max_log: u8, avoid_0_numbit: bool) -> FSETable { let mut out = FSETable::blank(); build_table_from_counts_into(counts, max_log, avoid_0_numbit, &mut out); From 43ac634be7498aeebbf47b08f9f33430edad0d93 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 12:57:05 +0300 Subject: [PATCH 14/36] perf(encoding): make the optimal parser's price mode a const The weight a price is taken in (integer bit weight, or the fractional one upstream uses from optLevel 1) is a property of the strategy, but it reached the parser as a field of the cost profile. The profile is passed into the per-segment DP by value, so there the field is a runtime bool, and every literal, literal-length and match-length price carried a test on it plus both weight forms. Offset prices already took the mode as the DP's `ACCURATE_PRICE` const. The literal, literal-length and match-length prices, their cached forms and the price-set range kernels now take the mode as a const as well, and the profile no longer carries it. The per-block base-price derivation takes the strategy's constant directly. Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary. runner1 (x86, task-clock), 50 frames of z000033[..200000], interleaved with the previous build and libzstd, three rounds of perf stat -r 3: L13 1121-1127 -> 1096-1101 ms (-2.3%; libzstd 876-880) L19 2820-2827 -> 2752-2760 ms (-2.4%; libzstd 2378-2392) L16 1879-1888 -> 1866-1877 ms (-0.7%, at the noise floor) Controls that never enter the optimal parser: L3 flat, L12 +0.8% (layout), which bounds what the build alone moves. Part of #128 --- zstd/src/encoding/bt/mod.rs | 24 ++++++------ zstd/src/encoding/cost_model/mod.rs | 36 +++++++++++------- zstd/src/encoding/hc/optimal.rs | 43 +++++++++++----------- zstd/src/encoding/hc/priceset.rs | 37 ++++++++++--------- zstd/src/encoding/match_generator/tests.rs | 29 ++++++++------- 5 files changed, 93 insertions(+), 76 deletions(-) diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index 6f0aa0a31..ef5f7e5d2 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -649,7 +649,7 @@ impl BtMatcher { } #[inline(always)] - pub(crate) fn cached_literal_price( + pub(crate) fn cached_literal_price( profile: HcOptimalCostProfile, stats: &HcOptState, byte: u8, @@ -666,7 +666,7 @@ impl BtMatcher { if *generations.get_unchecked(idx) == stamp { return *prices.get_unchecked(idx); } - let price = profile.literal_price(stats, byte); + let price = profile.literal_price::(stats, byte); *prices.get_unchecked_mut(idx) = price; *generations.get_unchecked_mut(idx) = stamp; price @@ -674,7 +674,7 @@ impl BtMatcher { } #[inline(always)] - pub(crate) fn cached_lit_length_price( + pub(crate) fn cached_lit_length_price( profile: HcOptimalCostProfile, stats: &HcOptState, lit_len: usize, @@ -682,7 +682,7 @@ impl BtMatcher { stamp: u32, ) -> u32 { if lit_len >= cache.len() { - return profile.lit_length_price(stats, lit_len); + return profile.lit_length_price::(stats, lit_len); } // SAFETY: the early-return above proves `lit_len < cache.len()`. // Each cell pairs `[price, generation]`, so the stamp check and the @@ -693,7 +693,7 @@ impl BtMatcher { if cell[1] == stamp { return cell[0]; } - let price = profile.lit_length_price(stats, lit_len); + let price = profile.lit_length_price::(stats, lit_len); cell[0] = price; cell[1] = stamp; price @@ -701,7 +701,7 @@ impl BtMatcher { } #[inline(always)] - pub(crate) fn cached_lit_length_delta_price( + pub(crate) fn cached_lit_length_delta_price( profile: HcOptimalCostProfile, stats: &HcOptState, lit_len: usize, @@ -715,13 +715,15 @@ impl BtMatcher { // No need to compute `0_usize - 1`. return 0; } - let price = Self::cached_lit_length_price(profile, stats, lit_len, cache, stamp); - let previous = Self::cached_lit_length_price(profile, stats, lit_len - 1, cache, stamp); + let price = + Self::cached_lit_length_price::(profile, stats, lit_len, cache, stamp); + let previous = + Self::cached_lit_length_price::(profile, stats, lit_len - 1, cache, stamp); price as i32 - previous as i32 } #[inline(always)] - pub(crate) fn cached_match_length_price( + pub(crate) fn cached_match_length_price( profile: HcOptimalCostProfile, stats: &HcOptState, match_len: usize, @@ -729,7 +731,7 @@ impl BtMatcher { stamp: u32, ) -> u32 { if match_len >= cache.len() { - return profile.match_length_price(stats, match_len); + return profile.match_length_price::(stats, match_len); } // SAFETY: see `cached_lit_length_price` — paired `[price, generation]` // cells, one cache line per probe; early return proves @@ -739,7 +741,7 @@ impl BtMatcher { if cell[1] == stamp { return cell[0]; } - let price = profile.match_length_price(stats, match_len); + let price = profile.match_length_price::(stats, match_len); cell[0] = price; cell[1] = stamp; price diff --git a/zstd/src/encoding/cost_model/mod.rs b/zstd/src/encoding/cost_model/mod.rs index bb8b5ece1..2f14d5e85 100644 --- a/zstd/src/encoding/cost_model/mod.rs +++ b/zstd/src/encoding/cost_model/mod.rs @@ -301,7 +301,9 @@ impl HcOptState { } } - pub(crate) fn rescale_freqs(&mut self, src: &[u8], profile: HcOptimalCostProfile) { + /// `accurate` is the strategy's price mode, which the base prices are + /// derived in. + pub(crate) fn rescale_freqs(&mut self, src: &[u8], accurate: bool) { self.price_type = HcOptPriceType::Dynamic; if self.lit_length_sum == 0 { if src.len() <= HC_PREDEF_THRESHOLD { @@ -404,7 +406,7 @@ impl HcOptState { self.match_length_sum = Self::scale_stats(&mut self.match_length_freq, 11); self.off_code_sum = Self::scale_stats(&mut self.off_code_freq, 11); } - self.set_base_prices(profile.accurate); + self.set_base_prices(accurate); } pub(crate) fn update_stats( @@ -442,7 +444,6 @@ impl HcOptState { pub(crate) struct HcOptimalCostProfile { pub(crate) max_chain_depth: usize, pub(crate) sufficient_match_len: usize, - pub(crate) accurate: bool, pub(crate) favor_small_offsets: bool, } @@ -470,12 +471,13 @@ impl HcOptimalCostProfile { Self { max_chain_depth: S::MAX_CHAIN_DEPTH, sufficient_match_len: S::SUFFICIENT_MATCH_LEN, - accurate: S::ACCURATE_PRICE, favor_small_offsets: S::FAVOR_SMALL_OFFSETS, } } - pub(crate) fn literal_price(&self, stats: &HcOptState, byte: u8) -> u32 { + /// `ACCURATE` is the strategy's price mode (upstream `optLevel >= 1`), a + /// const so the parser's per-candidate prices carry no weight-mode branch. + pub(crate) fn literal_price(&self, stats: &HcOptState, byte: u8) -> u32 { if !stats.literals_compressed() { return 8 * HC_BITCOST_MULTIPLIER; } @@ -487,28 +489,32 @@ impl HcOptimalCostProfile { // final subtract never underflows. debug_assert!(stats.lit_sum_base_price >= HC_BITCOST_MULTIPLIER); let lit_max = stats.lit_sum_base_price - HC_BITCOST_MULTIPLIER; - let mut lit_weight = HcOptState::weight(stats.lit_freq[byte as usize], self.accurate); + let mut lit_weight = HcOptState::weight(stats.lit_freq[byte as usize], ACCURATE); if lit_weight > lit_max { lit_weight = lit_max; } stats.lit_sum_base_price - lit_weight } - pub(crate) fn lit_length_price(&self, stats: &HcOptState, lit_len: usize) -> u32 { + pub(crate) fn lit_length_price( + &self, + stats: &HcOptState, + lit_len: usize, + ) -> u32 { if lit_len == HC_BLOCKSIZE_MAX { // Upstream zstd parity: ZSTD_litLengthPrice() handles the non-representable // BLOCKSIZE_MAX literal-length by charging one extra bit over the // largest encodable litLength symbol. return HC_BITCOST_MULTIPLIER - + self.lit_length_price(stats, HC_BLOCKSIZE_MAX.saturating_sub(1)); + + self.lit_length_price::(stats, HC_BLOCKSIZE_MAX.saturating_sub(1)); } if matches!(stats.price_type, HcOptPriceType::Predefined) { - return HcOptState::weight(lit_len as u32, self.accurate); + return HcOptState::weight(lit_len as u32, ACCURATE); } // ll_bits ≤ 16 ⇒ ll_bits * 256 ≤ 4096, sum no overflow. let (ll_code, ll_bits) = HcOptState::lit_code_and_bits(lit_len); ll_bits * HC_BITCOST_MULTIPLIER + stats.lit_length_sum_base_price - - HcOptState::weight(stats.lit_length_freq[ll_code], self.accurate) + - HcOptState::weight(stats.lit_length_freq[ll_code], ACCURATE) } #[inline(always)] @@ -533,18 +539,22 @@ impl HcOptimalCostProfile { } #[inline(always)] - pub(crate) fn match_length_price(&self, stats: &HcOptState, match_len: usize) -> u32 { + pub(crate) fn match_length_price( + &self, + stats: &HcOptState, + match_len: usize, + ) -> u32 { // Upstream zstd parity: mlBase = match_len - MINMATCH; callers guarantee // match_len ≥ HC_FORMAT_MINMATCH. ml_bits ≤ 16, * 256 ≤ 4096. debug_assert!(match_len >= HC_FORMAT_MINMATCH); let ml_base = match_len - HC_FORMAT_MINMATCH; if matches!(stats.price_type, HcOptPriceType::Predefined) { - return HcOptState::weight(ml_base as u32, self.accurate); + return HcOptState::weight(ml_base as u32, ACCURATE); } let (ml_code, ml_bits) = HcOptState::ml_code_and_bits(match_len); ml_bits * HC_BITCOST_MULTIPLIER + (stats.match_length_sum_base_price - - HcOptState::weight(stats.match_length_freq[ml_code], self.accurate)) + - HcOptState::weight(stats.match_length_freq[ml_code], ACCURATE)) } #[inline(always)] diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index 95d83e77f..976cd7d42 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -347,7 +347,7 @@ macro_rules! build_optimal_plan_impl_body { // Deferred base/seed prices: only reached on a matched seed (see // the declarations above). Assign before the forward DP / seed // paths below read them. - let node0_price = BtMatcher::cached_lit_length_price( + let node0_price = BtMatcher::cached_lit_length_price::( profile, $stats, initial_litlen, @@ -363,14 +363,14 @@ macro_rules! build_optimal_plan_impl_body { ..HcOptimalNode::default() }); } - ll0_price = BtMatcher::cached_lit_length_price( + ll0_price = BtMatcher::cached_lit_length_price::( profile, $stats, 0, &mut ll_cache, ll_price_stamp, ); - ll1_price = BtMatcher::cached_lit_length_price( + ll1_price = BtMatcher::cached_lit_length_price::( profile, $stats, 1, @@ -399,7 +399,7 @@ macro_rules! build_optimal_plan_impl_body { ); let off_price = profile .offset_price_for::($stats, off_base); - let ml_price = BtMatcher::cached_match_length_price( + let ml_price = BtMatcher::cached_match_length_price::( profile, $stats, longest_len, @@ -460,7 +460,7 @@ macro_rules! build_optimal_plan_impl_body { // SAFETY: cell 0 was written by the seed above. let nodes0_price = unsafe { *node_prices }; for match_len in (start_len..=max_match_len).rev() { - let ml_price = BtMatcher::cached_match_length_price( + let ml_price = BtMatcher::cached_match_length_price::( profile, $stats, match_len, @@ -513,7 +513,7 @@ macro_rules! build_optimal_plan_impl_body { let lit_len = prev_node.litlen as usize + 1; let lit_price = { let bt = $self.backend.bt_mut(); - BtMatcher::cached_literal_price( + BtMatcher::cached_literal_price::( profile, $stats, $current[pos - 1], @@ -522,7 +522,7 @@ macro_rules! build_optimal_plan_impl_body { lit_price_stamp, ) }; - let ll_delta = BtMatcher::cached_lit_length_delta_price( + let ll_delta = BtMatcher::cached_lit_length_delta_price::( profile, $stats, lit_len, @@ -558,7 +558,7 @@ macro_rules! build_optimal_plan_impl_body { if ll1_price < ll0_price { let next_lit_price = { let bt = $self.backend.bt_mut(); - BtMatcher::cached_literal_price( + BtMatcher::cached_literal_price::( profile, $stats, $current[pos], @@ -572,7 +572,8 @@ macro_rules! build_optimal_plan_impl_body { next_lit_price, ll1_price as i32 - ll0_price as i32, ); - let ll_delta_next = BtMatcher::cached_lit_length_delta_price( + let ll_delta_next = + BtMatcher::cached_lit_length_delta_price::( profile, $stats, lit_len + 1, @@ -722,7 +723,7 @@ macro_rules! build_optimal_plan_impl_body { ); let off_price = profile .offset_price_for::($stats, off_base); - let ml_price = BtMatcher::cached_match_length_price( + let ml_price = BtMatcher::cached_match_length_price::( profile, $stats, longest_len, @@ -786,7 +787,7 @@ macro_rules! build_optimal_plan_impl_body { // skipped. Order-dependent, stays scalar. for match_len in (start_len..=max_match_len).rev() { let next = pos + match_len; - let ml_price = BtMatcher::cached_match_length_price( + let ml_price = BtMatcher::cached_match_length_price::( profile, $stats, match_len, @@ -1244,7 +1245,7 @@ impl HcMatchGenerator { debug_assert_eq!(profile.favor_small_offsets, S::FAVOR_SMALL_OFFSETS); let mut opt_state = core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); - opt_state.rescale_freqs(current, profile); + opt_state.rescale_freqs(current, S::ACCURATE_PRICE); let mut best_plan = core::mem::take(&mut self.backend.bt_mut().opt_segment_plan_scratch); best_plan.clear(); let mut plan_reps = self.table.offset_hist; @@ -1375,7 +1376,7 @@ impl HcMatchGenerator { &mut plan_literals_cursor, &mut plan_reps, &mut opt_state, - profile.accurate, + S::ACCURATE_PRICE, ); } plan_reps = end_reps; @@ -1466,7 +1467,7 @@ impl HcMatchGenerator { debug_assert_eq!(seed_profile.favor_small_offsets, S::FAVOR_SMALL_OFFSETS); let mut opt_state = core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); - opt_state.rescale_freqs(current, seed_profile); + opt_state.rescale_freqs(current, S::ACCURATE_PRICE); let mut seed_reps = self.table.offset_hist; let (mut cursor, mut seed_litlen) = self.table.opt_start_cursor_and_litlen(current_abs_start); @@ -1591,7 +1592,7 @@ impl HcMatchGenerator { &mut seed_literals_cursor, &mut seed_reps, &mut opt_state, - seed_profile.accurate, + S::ACCURATE_PRICE, ); seed_plan.truncate(segment_start); } @@ -1749,7 +1750,7 @@ impl HcMatchGenerator { out, buffers, collect_optimal_candidates_initialized_neon, - crate::encoding::hc::priceset::priceset_range_nonabort_neon, + crate::encoding::hc::priceset::priceset_range_nonabort_neon::, ) } @@ -1788,7 +1789,7 @@ impl HcMatchGenerator { out, buffers, collect_optimal_candidates_initialized_sse42, - crate::encoding::hc::priceset::priceset_range_nonabort_sse41, + crate::encoding::hc::priceset::priceset_range_nonabort_sse41::, ) } @@ -1827,7 +1828,7 @@ impl HcMatchGenerator { out, buffers, collect_optimal_candidates_initialized_sse2, - crate::encoding::hc::priceset::priceset_range_nonabort_sse2, + crate::encoding::hc::priceset::priceset_range_nonabort_sse2::, ) } @@ -1863,7 +1864,7 @@ impl HcMatchGenerator { out, buffers, collect_optimal_candidates_initialized_avx2_bmi2, - crate::encoding::hc::priceset::priceset_range_nonabort_avx2, + crate::encoding::hc::priceset::priceset_range_nonabort_avx2::, ) } @@ -1914,7 +1915,7 @@ impl HcMatchGenerator { out, buffers, collect_optimal_candidates_initialized_scalar, - crate::encoding::hc::priceset::priceset_range_nonabort_scalar, + crate::encoding::hc::priceset::priceset_range_nonabort_scalar::, ) } @@ -1957,7 +1958,7 @@ impl HcMatchGenerator { out, buffers, collect_optimal_candidates_initialized_simd128, - crate::encoding::hc::priceset::priceset_range_nonabort_simd128, + crate::encoding::hc::priceset::priceset_range_nonabort_simd128::, ) } diff --git a/zstd/src/encoding/hc/priceset.rs b/zstd/src/encoding/hc/priceset.rs index eeac395ca..5d493170a 100644 --- a/zstd/src/encoding/hc/priceset.rs +++ b/zstd/src/encoding/hc/priceset.rs @@ -51,7 +51,7 @@ unsafe fn priceset_improved_mask8_avx2(next_cost: &[u32; 8], node_price: &[u32]) /// so the SoA vector path stays byte-identical. #[inline(always)] #[allow(clippy::too_many_arguments)] -fn priceset_next_cost( +fn priceset_next_cost( profile: HcOptimalCostProfile, stats: &HcOptState, ml_cache: &mut [[u32; 2]], @@ -61,8 +61,9 @@ fn priceset_next_cost( off_price: u32, base_cost: u32, ) -> u32 { - let ml_price = - BtMatcher::cached_match_length_price(profile, stats, match_len, ml_cache, ml_stamp); + let ml_price = BtMatcher::cached_match_length_price::( + profile, stats, match_len, ml_cache, ml_stamp, + ); let seq_cost = BtMatcher::add_prices( ll0_price, profile.match_price_from_parts(off_price, ml_price, stats), @@ -94,7 +95,7 @@ fn priceset_next_cost( ), allow(dead_code) )] -pub(crate) fn priceset_range_nonabort_scalar( +pub(crate) fn priceset_range_nonabort_scalar( node_prices: &mut [u32], nodes: &mut [MaybeUninit], ml_cache: &mut [[u32; 2]], @@ -113,7 +114,7 @@ pub(crate) fn priceset_range_nonabort_scalar( ) -> usize { let mut new_last = last_pos; for ml in start..=max { - let next_cost = priceset_next_cost( + let next_cost = priceset_next_cost::( profile, stats, ml_cache, ml_stamp, ml, ll0_price, off_price, base_cost, ); let next = pos + ml; @@ -162,7 +163,7 @@ pub(crate) fn priceset_range_nonabort_scalar( )), allow(dead_code) )] -fn priceset_range_vec( +fn priceset_range_vec( node_prices: &mut [u32], nodes: &mut [MaybeUninit], ml_cache: &mut [[u32; 2]], @@ -215,7 +216,7 @@ fn priceset_range_vec( } } else { for (k, slot) in buf.iter_mut().enumerate() { - *slot = priceset_next_cost( + *slot = priceset_next_cost::( profile, stats, ml_cache, @@ -247,7 +248,7 @@ fn priceset_range_vec( ml += W; } while ml <= max { - let next_cost = priceset_next_cost( + let next_cost = priceset_next_cost::( profile, stats, ml_cache, ml_stamp, ml, ll0_price, off_price, base_cost, ); let next = pos + ml; @@ -322,7 +323,7 @@ unsafe fn priceset_cached_prices8_avx2(cells: &[[u32; 2]], stamp: u32) -> Option #[target_feature(enable = "avx2")] #[inline] #[allow(clippy::too_many_arguments)] -pub(crate) unsafe fn priceset_range_nonabort_avx2( +pub(crate) unsafe fn priceset_range_nonabort_avx2( node_prices: &mut [u32], nodes: &mut [MaybeUninit], ml_cache: &mut [[u32; 2]], @@ -339,7 +340,7 @@ pub(crate) unsafe fn priceset_range_nonabort_avx2( reps: [u32; 3], last_pos: usize, ) -> usize { - priceset_range_vec::<8>( + priceset_range_vec::<8, ACCURATE>( node_prices, nodes, ml_cache, @@ -405,7 +406,7 @@ unsafe fn priceset_improved_mask4_neon(next_cost: &[u32; 4], node_price: &[u32]) #[target_feature(enable = "neon")] #[inline] #[allow(clippy::too_many_arguments)] -pub(crate) unsafe fn priceset_range_nonabort_neon( +pub(crate) unsafe fn priceset_range_nonabort_neon( node_prices: &mut [u32], nodes: &mut [MaybeUninit], ml_cache: &mut [[u32; 2]], @@ -422,7 +423,7 @@ pub(crate) unsafe fn priceset_range_nonabort_neon( reps: [u32; 3], last_pos: usize, ) -> usize { - priceset_range_vec::<4>( + priceset_range_vec::<4, ACCURATE>( node_prices, nodes, ml_cache, @@ -537,7 +538,7 @@ unsafe fn priceset_improved_mask4_sse41(next_cost: &[u32; 4], node_price: &[u32] #[target_feature(enable = "sse4.2")] #[inline] #[allow(clippy::too_many_arguments)] -pub(crate) unsafe fn priceset_range_nonabort_sse41( +pub(crate) unsafe fn priceset_range_nonabort_sse41( node_prices: &mut [u32], nodes: &mut [MaybeUninit], ml_cache: &mut [[u32; 2]], @@ -554,7 +555,7 @@ pub(crate) unsafe fn priceset_range_nonabort_sse41( reps: [u32; 3], last_pos: usize, ) -> usize { - priceset_range_vec::<4>( + priceset_range_vec::<4, ACCURATE>( node_prices, nodes, ml_cache, @@ -584,7 +585,7 @@ pub(crate) unsafe fn priceset_range_nonabort_sse41( #[target_feature(enable = "sse2")] #[inline] #[allow(clippy::too_many_arguments)] -pub(crate) unsafe fn priceset_range_nonabort_sse2( +pub(crate) unsafe fn priceset_range_nonabort_sse2( node_prices: &mut [u32], nodes: &mut [MaybeUninit], ml_cache: &mut [[u32; 2]], @@ -601,7 +602,7 @@ pub(crate) unsafe fn priceset_range_nonabort_sse2( reps: [u32; 3], last_pos: usize, ) -> usize { - priceset_range_vec::<4>( + priceset_range_vec::<4, ACCURATE>( node_prices, nodes, ml_cache, @@ -678,7 +679,7 @@ unsafe fn priceset_improved_mask4_simd128(next_cost: &[u32; 4], node_price: &[u3 #[target_feature(enable = "simd128")] #[inline] #[allow(clippy::too_many_arguments)] -pub(crate) unsafe fn priceset_range_nonabort_simd128( +pub(crate) unsafe fn priceset_range_nonabort_simd128( node_prices: &mut [u32], nodes: &mut [MaybeUninit], ml_cache: &mut [[u32; 2]], @@ -695,7 +696,7 @@ pub(crate) unsafe fn priceset_range_nonabort_simd128( reps: [u32; 3], last_pos: usize, ) -> usize { - priceset_range_vec::<4>( + priceset_range_vec::<4, ACCURATE>( node_prices, nodes, ml_cache, diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index a647c1813..56097db81 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -950,10 +950,12 @@ fn btultra2_profile_disables_small_offset_handicap() { !profile.favor_small_offsets, "btultra2 should match upstream zstd opt2 offset pricing" ); - assert!( - profile.accurate, - "btultra2 should use upstream zstd opt2 accurate pricing" - ); + const { + assert!( + ::ACCURATE_PRICE, + "btultra2 should use upstream zstd opt2 accurate pricing" + ); + } } #[test] @@ -1030,7 +1032,7 @@ fn dictionary_entropy_seed_initializes_opt_state_from_tables() { hc.backend.bt_mut().opt_state.rescale_freqs( b"abcd", - HcOptimalCostProfile::const_for_strategy::(), + ::ACCURATE_PRICE, ); let base_ll_freqs: [u32; HC_MAX_LL + 1] = [ @@ -1075,7 +1077,7 @@ fn dictionary_fse_seed_applies_without_huffman_seed() { hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); hc.backend.bt_mut().opt_state.rescale_freqs( b"abcd", - HcOptimalCostProfile::const_for_strategy::(), + ::ACCURATE_PRICE, ); let base_ll_freqs: [u32; HC_MAX_LL + 1] = [ @@ -1119,7 +1121,7 @@ fn dictionary_seed_overrides_predef_price_mode_on_tiny_input() { hc.seed_dictionary_entropy(None, Some(&*ll), Some(&*ml), Some(&*of)); hc.backend.bt_mut().opt_state.rescale_freqs( b"abc", - HcOptimalCostProfile::const_for_strategy::(), + ::ACCURATE_PRICE, ); assert!( matches!( @@ -1136,9 +1138,9 @@ fn lit_length_price_blocksize_max_costs_one_extra_bit() { HcOptimalCostProfile::const_for_strategy::(); let mut stats_predef = HcOptState::new(); stats_predef.price_type = HcOptPriceType::Predefined; - let predef_max = profile_predef.lit_length_price(&stats_predef, HC_BLOCKSIZE_MAX); + let predef_max = profile_predef.lit_length_price::(&stats_predef, HC_BLOCKSIZE_MAX); let predef_prev = - profile_predef.lit_length_price(&stats_predef, HC_BLOCKSIZE_MAX.saturating_sub(1)); + profile_predef.lit_length_price::(&stats_predef, HC_BLOCKSIZE_MAX.saturating_sub(1)); assert_eq!( predef_max, predef_prev + HC_BITCOST_MULTIPLIER, @@ -1158,8 +1160,9 @@ fn lit_length_price_blocksize_max_costs_one_extra_bit() { stats_dyn.lit_freq.fill(1); stats_dyn.lit_sum = (HC_MAX_LIT + 1) as u32; stats_dyn.set_base_prices(true); - let dyn_max = profile_dyn.lit_length_price(&stats_dyn, HC_BLOCKSIZE_MAX); - let dyn_prev = profile_dyn.lit_length_price(&stats_dyn, HC_BLOCKSIZE_MAX.saturating_sub(1)); + let dyn_max = profile_dyn.lit_length_price::(&stats_dyn, HC_BLOCKSIZE_MAX); + let dyn_prev = + profile_dyn.lit_length_price::(&stats_dyn, HC_BLOCKSIZE_MAX.saturating_sub(1)); assert_eq!( dyn_max, dyn_prev + HC_BITCOST_MULTIPLIER, @@ -1289,7 +1292,7 @@ fn literal_price_uses_eight_bits_when_literals_uncompressed() { stats.set_literals_compressed_for_tests(false); stats.price_type = HcOptPriceType::Predefined; assert_eq!( - profile.literal_price(&stats, b'a'), + profile.literal_price::(&stats, b'a'), 8 * HC_BITCOST_MULTIPLIER, "uncompressed literals should cost 8 bits regardless of price mode" ); @@ -1337,7 +1340,7 @@ fn dictionary_huffman_seed_ignored_when_literals_uncompressed() { stats.seed_dictionary_entropy(Some(&huff), Some(&*ll), Some(&*ml), Some(&*of)); stats.rescale_freqs( b"abcd", - HcOptimalCostProfile::const_for_strategy::(), + ::ACCURATE_PRICE, ); assert_eq!( stats.lit_sum, 0, From 32120aead2fbb29b4fcdd9353eb127c415d0a420 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 13:07:27 +0300 Subject: [PATCH 15/36] perf(encoding): compute integer-weight prices in place Under the integer weight (btopt) a literal, literal-length or match-length price is one bit scan on a frequency. The per-call price caches stood in for it with a stamp probe, a branch and, on a miss, two stores, which costs as much as the price itself. Upstream computes these prices in place at optLevel 0. The caches now serve only the fractional weight of the btultra strategies. Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary. runner1 (x86, task-clock), 50 frames of z000033[..200000], interleaved with the previous build and libzstd, three rounds of perf stat -r 3: L13 1096-1102 -> 1083-1087 ms (-1.2%; libzstd 875-881) L15 1184-1187 -> 1179-1183 ms (-0.4%; libzstd 1194-1198) Controls: L12 (no optimal parser) and L19 (fractional weight, which this does not touch) flat. Part of #128 --- zstd/src/encoding/bt/mod.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index ef5f7e5d2..6aea5b17e 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -657,6 +657,12 @@ impl BtMatcher { generations: &mut [u32; HC_MAX_LIT + 1], stamp: u32, ) -> u32 { + // The integer weight is one bit scan on a frequency, no dearer than the + // probe that would stand in for it, so it is computed in place, as + // upstream does at `optLevel` 0. Only the fractional weight is cached. + if !ACCURATE { + return profile.literal_price::(stats, byte); + } // SAFETY: `byte as usize` is `0..256` and the fixed-size arrays are // `[u32; HC_MAX_LIT + 1 = 257]`, so the index is statically in bounds. // Each cached_*_price call sits inside the optimal parser per-byte @@ -681,7 +687,8 @@ impl BtMatcher { cache: &mut [[u32; 2]], stamp: u32, ) -> u32 { - if lit_len >= cache.len() { + // Computed in place under the integer weight; see `cached_literal_price`. + if !ACCURATE || lit_len >= cache.len() { return profile.lit_length_price::(stats, lit_len); } // SAFETY: the early-return above proves `lit_len < cache.len()`. @@ -730,7 +737,8 @@ impl BtMatcher { cache: &mut [[u32; 2]], stamp: u32, ) -> u32 { - if match_len >= cache.len() { + // Computed in place under the integer weight; see `cached_literal_price`. + if !ACCURATE || match_len >= cache.len() { return profile.match_length_price::(stats, match_len); } // SAFETY: see `cached_lit_length_price` — paired `[price, generation]` From f3ae1876da83e49ac47f81d2060cfd24b2748662 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 13:37:20 +0300 Subject: [PATCH 16/36] perf(encoding): walk the insertion tree on the stored index The tree insertion walk decoded every node's stored index into an absolute position through four tests (empty slot, below the shift, below the window, past the position) and re-read the table's position base, shift and history start on each node. The collect walk over the same tree already carries the stored index alone: one unsigned range test bounds the window and ends on an empty slot, and the position, history index and pair slot are each one add of a bias taken before the walk. The insertion walk now has the same shape. A slot below the shift cannot reach the range test in an armed block, since a rebase rewrites every slot; that is now a debug assertion rather than a release test. Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary. Fewer operations, no measurable time change: callgrind at L13 (3 frames of z000033[..200000]) puts the insertion function at 48,148,515 -> 47,889,189 instructions; runner1 task-clock, interleaved, three rounds: L13 1080.6-1081.9 -> 1081.1-1082.0 ms, L19 2746-2762 -> 2734-2742 ms, control L12 flat. Part of #128 --- zstd/src/encoding/hc/generator.rs | 61 +++++++++++++++++-------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index 3d2d19844..4a47b18c5 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -183,6 +183,28 @@ macro_rules! bt_insert_step_no_rebase_body { // sentinel that ALWAYS triggers. let bt_low = $abs_pos.saturating_sub(bt_mask); let window_low = $table.window_low_abs_for_target($target_abs); + // The walk carries one coordinate, the stored index, as the collect body + // does and as upstream carries `matchIndex`: the window bound becomes one + // unsigned range test on it (HC_EMPTY, 0, decodes below the window and + // ends the walk), and the absolute position, the history index and the + // pair slot are each one add of a bias taken here, instead of reloading + // `position_base` / `index_shift` / `history_abs_start` through the + // table on every node. See the collect body for the derivation. + let abs_bias = $table + .position_base + .wrapping_sub(1) + .wrapping_sub($table.index_shift); + let win_off = abs_bias.wrapping_sub(window_low); + // The window floor follows the tree update's target, which may lie past + // this position by more than the window; nothing is in range then, as + // upstream's `matchIndex >= windowLow` finds on its first test. + let win_range = if window_low < $abs_pos { + $abs_pos - window_low + } else { + 0 + }; + let idx_bias = abs_bias.wrapping_sub($table.history_abs_start); + let bt_bias = $table.position_base.wrapping_sub(1); // `abs_pos + 9` is safe in raw form: `MatchTable::add_data` caps // total input at `usize::MAX - STREAM_ABS_HEADROOM` (where // `STREAM_ABS_HEADROOM = HC_OPT_NUM + 16`), so every @@ -206,40 +228,25 @@ macro_rules! bt_insert_step_no_rebase_body { let mut match_stored = unsafe { *hash_ptr.add(hash) }; unsafe { *hash_ptr.add(hash) = stored }; - while compares_left > 0 { - if match_stored == $crate::encoding::match_table::storage::HC_EMPTY { - break; - } - // Reject stale post-rebase slots whose pre-shift position is below - // `index_shift` explicitly. A `wrapping_sub` maps such a slot to a - // near-`usize::MAX` value that the `>= abs_pos` test only rejects - // while `abs_pos` is far from the integer ceiling; on a - // long-running rebased stream (reachable on 32-bit) `abs_pos` can - // approach the ceiling and the wrapped value can land back inside - // `[window_low, abs_pos)`. Ending the walk on the underflow avoids - // that. `match_stored != HC_EMPTY` here, so the `- 1` cannot - // underflow. The shift is taken off the stored index before the - // floor is added, because on a 32-bit word the floor plus a stored - // index need not fit; a slot under the shift decodes below - // `position_base`, which the window floor rejects anyway. - let match_relative = match_stored as usize - 1; - if match_relative < $table.index_shift { - break; - } - let candidate_abs = $table.position_base + (match_relative - $table.index_shift); - if candidate_abs < window_low || candidate_abs >= $abs_pos { - break; - } + while compares_left > 0 && (match_stored as usize).wrapping_add(win_off) < win_range { compares_left -= 1; - - let next_pair_idx = $table.bt_pair_index_for_abs(candidate_abs); + let stored = match_stored as usize; + // A slot written under an earlier encoding would sit below the + // shift; the block was armed, and a rebase rewrites every slot, so + // none reaches an in-window test. + debug_assert!(stored > $table.index_shift); + let candidate_abs = stored.wrapping_add(abs_bias); + debug_assert!(candidate_abs >= window_low && candidate_abs < $abs_pos); + // `2*((candidate_abs + index_shift) & bt_mask)`, with the shift + // folded: `candidate_abs + index_shift == stored + bt_bias`. + let next_pair_idx = 2 * (stored.wrapping_add(bt_bias) & bt_mask); // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)` // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base, // table not realloc'd during the walk. let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) }; let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) }; let seed_len = common_length_smaller.min(common_length_larger); - let candidate_idx = candidate_abs - $table.history_abs_start; + let candidate_idx = stored.wrapping_add(idx_bias); // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤ // concat.len()` since the candidate is within // `[history_abs_start, abs_pos)` and `tail_limit ≤ From 342c49dde6bad7325b25267a585fa6823427a6e7 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 15:04:38 +0300 Subject: [PATCH 17/36] perf(encoding): run the optimal parser as one pass per block The optimal parser entered its DP function once per segment, about 16 thousand times per 200 KB frame at level 13, and after each segment walked the segment's plan a second time to replay it into the statistics, re-deriving every offset base through the repeat history. Upstream runs a block as one loop (ZSTD_compressBlock_opt_generic) and records each sequence into the statistics from its traceback. The parser now has the same shape: - one function per kernel runs the whole block: the segment loop around the forward pass and traceback, with the frame, the buffer set-up and the const dispatch paid once per block - the traceback records each settled sequence into the statistics (ZSTD_updateStats) and refreshes the base prices once per segment, so the replay pass is gone; the btultra2 seed pass no longer builds a plan at all - what the pass carries between segments (cursor, pending literals, repeat history, the statistics' literal cursor) lives in the plan buffers rather than in locals, so none of it is live across the segment body. Held as locals it cost the btultra DP about 4.5% more instructions in spills and reloads and measured 1.6% slower at level 16 Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary. callgrind, 3 frames of z000033[..200000]: L13 398,671,808 -> 396,245,242 (-0.61%) L16 635,870,845 -> 636,903,867 (+0.16%) L19 948,388,977 -> 946,328,078 (-0.22%) runner1 task-clock, 50 frames, interleaved with the previous build and libzstd, every arm pinned to one core, two runs: L19 2748-2767 -> 2714-2734 ms (-1.3%, both runs), L13 and L16 unchanged, control L12 flat. Under the 1.5% floor, so not claimed as a speed change. Part of #128 --- zstd/src/encoding/bt/mod.rs | 65 ++- zstd/src/encoding/hc/optimal.rs | 804 ++++++++++++++++++-------------- zstd/src/encoding/opt/types.rs | 20 + zstd/src/encoding/strategy.rs | 7 +- 4 files changed, 510 insertions(+), 386 deletions(-) diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index 6aea5b17e..2579b034c 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -379,6 +379,7 @@ impl BtMatcher { store, price_arena, candidates_searched_at: _, + pass: _, } = buffers; candidates.clear(); self.opt_nodes_scratch = nodes; @@ -572,47 +573,41 @@ impl BtMatcher { } } - /// Upstream zstd parity: replay an already-emitted plan segment through the - /// `optStatePtr_t` stats updater so the next parse pass sees frozen - /// counts. Pure static helper — only mutates the caller-owned - /// `opt_state` / `reps` / `literals_start`. - pub(crate) fn update_plan_stats_segment( - current: &[u8], - current_len: usize, - plan: &[HcOptimalSequence], + /// Upstream zstd parity: `ZSTD_updateStats` for one sequence the parser + /// has just settled, called from its traceback so the next segment prices + /// against counts that include it. `literals_start` is the block offset + /// the previous sequence ended at and `reps` the history before this one; + /// both advance. The caller refreshes the base prices once the segment's + /// sequences are in. + #[inline] + pub(crate) fn record_sequence_stats( + block: &[u8], + sequence: HcOptimalSequence, literals_start: &mut usize, reps: &mut [u32; 3], opt_state: &mut HcOptState, - accurate: bool, ) { - if plan.is_empty() { + let lit_len = sequence.lit_len as usize; + let match_len = sequence.match_len as usize; + // `checked_add` on both edges so a malformed sequence can't overflow + // `usize` arithmetic before the bounds guard fires. `saturating_add` + // would have masked overflow as "clamp to usize::MAX" which then + // bypasses the `> block.len()` check. + let Some(start) = literals_start.checked_add(lit_len) else { + return; + }; + let Some(end) = start.checked_add(match_len) else { + return; + }; + if end > block.len() { return; } - for item in plan { - let lit_len = item.lit_len as usize; - let match_len = item.match_len as usize; - // `checked_add` on both edges so a malformed / partially-built - // plan can't overflow `usize` arithmetic before the - // bounds guard fires. `saturating_add` would have masked - // overflow as "clamp to usize::MAX" which then bypasses the - // `> current_len` check. - let Some(start) = literals_start.checked_add(lit_len) else { - continue; - }; - let Some(end) = start.checked_add(match_len) else { - continue; - }; - if end > current_len { - continue; - } - let literals = ¤t[*literals_start..start]; - let (off_base, next_reps) = - Self::encode_offset_with_reps(item.offset, literals.len(), *reps); - opt_state.update_stats(literals.len(), literals, off_base, match_len); - *reps = next_reps; - *literals_start = end; - } - opt_state.set_base_prices(accurate); + let literals = &block[*literals_start..start]; + let (off_base, next_reps) = + Self::encode_offset_with_reps(sequence.offset, literals.len(), *reps); + opt_state.update_stats(literals.len(), literals, off_base, match_len); + *reps = next_reps; + *literals_start = end; } /// Brings cells `start..=end` into the frontier as unreached: price `MAX`. diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index 976cd7d42..bb8e9f693 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -37,7 +37,7 @@ use crate::encoding::{ match_table::storage::HC3_HASH_LOG, opt::ldm::{HcOptLdmState, HcRawSeqStore}, opt::types::{ - HcCandidateQuery, HcOptimalNode, HcOptimalPlanBuffers, HcOptimalPlanState, + HcBlockPass, HcCandidateQuery, HcOptimalNode, HcOptimalPlanBuffers, HcOptimalPlanState, HcOptimalSequence, MatchCandidate, }, }; @@ -54,7 +54,17 @@ macro_rules! build_optimal_plan_impl_body { $out:ident, $buffers:expr, $collect:ident, - $priceset:path $(,)? + $priceset:path, + // The whole block and its statistics: the traceback records each + // sequence it settles into them (upstream `ZSTD_updateStats`). + $block:ident, + $opt_state:ident, + // False for the btultra2 seed pass, which keeps only the statistics. + $keep_plan:expr, + // The block the caller wraps this body in: every exit is a `break` out + // of it with the segment's result, so the body runs inside the caller's + // segment loop rather than as a function of its own. + $seg:lifetime $(,)? ) => {{ let current_abs_end = $current_abs_start + $current_len; let min_match_len = HC_OPT_MIN_MATCH_LEN; @@ -107,6 +117,7 @@ macro_rules! build_optimal_plan_impl_body { store, price_arena, candidates_searched_at: searched_at, + pass, } = &mut *$buffers; // The node arenas are indexed through base pointers resolved once, the // way upstream zstd indexes `opt[]`: nothing in this body resizes them, @@ -225,7 +236,7 @@ macro_rules! build_optimal_plan_impl_body { // Literals only: no sequence was emitted and the repcodes are // untouched, exactly as the per-literal returns left them. The // price is discarded by the caller. - return ( + break $seg ( 0u32, initial_reps, initial_litlen + skipped_literals, @@ -874,7 +885,7 @@ macro_rules! build_optimal_plan_impl_body { if last_pos == 0 { if $current_len == 0 { let price = 0u32; // deferred: node_prices[0] unset on no-match; caller discards price - return (price, initial_reps, initial_litlen, 0); + break $seg (price, initial_reps, initial_litlen, 0); } // No match at this position: it is a single literal (upstream zstd // `ZSTD_compressBlock_opt_generic` `if (!nbMatches) { ip++; }`). The @@ -891,7 +902,7 @@ macro_rules! build_optimal_plan_impl_body { // node_prices[0] is unset on a no-match seed (deferred); the caller // discards this price anyway. let price = 0u32; - return (price, initial_reps, next_litlen, 1); + break $seg (price, initial_reps, next_litlen, 1); } let target_pos = forced_end.unwrap_or(last_pos.min(frontier_limit)); @@ -907,11 +918,11 @@ macro_rules! build_optimal_plan_impl_body { unsafe { (*nodes.add(target_pos), *node_prices.add(target_pos)) } }; if last_stretch_price == u32::MAX { - return (u32::MAX, initial_reps, initial_litlen, $current_len); + break $seg (u32::MAX, initial_reps, initial_litlen, $current_len); } if last_stretch.mlen == 0 { - return ( + break $seg ( last_stretch_price, last_stretch.reps, last_stretch.litlen as usize, @@ -933,7 +944,7 @@ macro_rules! build_optimal_plan_impl_body { } else { let tail_literals = last_stretch.litlen as usize; if cur < tail_literals { - return ( + break $seg ( last_stretch_price, last_stretch.reps, tail_literals, @@ -996,6 +1007,12 @@ macro_rules! build_optimal_plan_impl_body { let mut tail_literals = initial_litlen; let mut store_pos = store_start; + // Each settled sequence goes into the statistics here, as upstream's + // traceback calls `ZSTD_updateStats` per stored sequence, instead of in + // a second pass over the plan after the segment. The repeat history + // runs from the segment's own. + let mut stats_reps = initial_reps; + let mut recorded = false; while store_pos <= store_end { let stretch = store[store_pos]; let llen = stretch.litlen as usize; @@ -1005,14 +1022,29 @@ macro_rules! build_optimal_plan_impl_body { store_pos += 1; continue; } - $out.push(HcOptimalSequence { + let sequence = HcOptimalSequence { offset: stretch.off, match_len: mlen as u32, lit_len: llen as u32, - }); + }; + if $keep_plan { + $out.push(sequence); + } + BtMatcher::record_sequence_stats( + $block, + sequence, + &mut pass.literals_cursor, + &mut stats_reps, + &mut *$opt_state, + ); + recorded = true; tail_literals = 0; store_pos += 1; } + if recorded { + $opt_state + .set_base_prices(<$strategy_ty as crate::encoding::strategy::Strategy>::ACCURATE_PRICE); + } let result = ( last_stretch_price, end_reps, @@ -1027,6 +1059,79 @@ macro_rules! build_optimal_plan_impl_body { }}; } +/// The optimal parser's pass over one block, upstream zstd +/// `ZSTD_compressBlock_opt_generic`: the segment loop around each segment's +/// forward pass and traceback ([`build_optimal_plan_impl_body!`]), whose +/// traceback also records the segment's sequences into the statistics. The +/// frame, the buffer set-up and the arguments are paid once per block rather +/// than once per segment, which on input with few matches is once per literal +/// run. `$keep_plan` is false for the btultra2 seed pass, which keeps only the +/// statistics. +macro_rules! optimal_block_body { + ( + $self:expr, + $strategy_ty:ty, + $current:ident, + $current_abs_start:ident, + $cursor:ident, + $litlen:ident, + $reps:ident, + $profile:ident, + $opt_state:ident, + $plan:ident, + $buffers:ident, + $keep_plan:expr, + $collect:ident, + $priceset:path $(,)? + ) => {{ + // Everything the pass carries between segments lives in `$buffers.pass`, + // read at the top of a segment and written at its end, so none of it is + // live across the segment body (see `HcOptimalPlanBuffers::pass`). + $buffers.pass = HcBlockPass { + cursor: $cursor, + litlen: $litlen, + reps: $reps, + literals_cursor: 0, + }; + while $buffers.pass.cursor < $current.len().saturating_sub(8) { + let cursor = $buffers.pass.cursor; + let segment = &$current[cursor..]; + let segment_abs_start = $current_abs_start + cursor; + let segment_len = $current.len() - cursor; + let segment_state = HcOptimalPlanState { + block_offset: cursor, + reps: $buffers.pass.reps, + litlen: $buffers.pass.litlen, + profile: $profile, + }; + let segment_stats: &HcOptState = &*$opt_state; + let (_, end_reps, end_litlen, consumed_len) = 'segment: { + build_optimal_plan_impl_body!( + $self, + $strategy_ty, + segment, + segment_abs_start, + segment_len, + segment_state, + segment_stats, + $plan, + $buffers, + $collect, + $priceset, + $current, + $opt_state, + $keep_plan, + 'segment, + ) + }; + let pass = &mut $buffers.pass; + pass.reps = end_reps; + pass.litlen = end_litlen; + pass.cursor += consumed_len; + } + }}; +} + /// `collect_optimal_candidates_initialized` body parameterized over the per-CPU /// kernel: the `$cpl` path is the kernel's `common_prefix_len_ptr` (used in /// the HC chain walk fallback), and the four method-name substitutions @@ -1248,140 +1353,122 @@ impl HcMatchGenerator { opt_state.rescale_freqs(current, S::ACCURATE_PRICE); let mut best_plan = core::mem::take(&mut self.backend.bt_mut().opt_segment_plan_scratch); best_plan.clear(); - let mut plan_reps = self.table.offset_hist; - let (mut cursor, mut plan_litlen) = - self.table.opt_start_cursor_and_litlen(current_abs_start); - let mut plan_literals_cursor = 0usize; - let match_loop_limit = current_len.saturating_sub(8); + let plan_reps = self.table.offset_hist; + let (cursor, plan_litlen) = self.table.opt_start_cursor_and_litlen(current_abs_start); // Frame-constant LDM presence, resolved once per block (not per segment // and not in the DP hot loop): drives the HAS_LDM const-generic dispatch. let has_ldm = !self.backend.bt_mut().ldm_sequences.is_empty(); - // Resolve the SIMD tier ONCE here, never per segment. The per-literal - // hot loop then runs under a single kernel-monomorphized expansion - // (calling build_optimal_plan_impl_ directly) instead of - // hitting select_kernel()'s OnceLock atomic + a CPU-tier match on every - // build_optimal_plan call. Mirrors the Fast matcher dispatch shape. + // Resolve the SIMD tier ONCE here: the whole block then runs under one + // kernel-monomorphized pass (`run_optimal_block_`) instead of + // hitting select_kernel()'s OnceLock atomic + a CPU-tier match per + // segment. Mirrors the Fast matcher dispatch shape. macro_rules! run_main_loop { - ($impl_wrapper:ident) => {{ - while cursor < match_loop_limit { - let remaining_len = current_len - cursor; - let segment_abs_start = current_abs_start + cursor; - let segment_start = best_plan.len(); - let state = HcOptimalPlanState { - block_offset: cursor, - reps: plan_reps, - litlen: plan_litlen, - profile, - }; - let (_, end_reps, end_litlen, consumed_len) = - match (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS, has_ldm) { - (true, false, false) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut best_plan, - &mut plan_buffers, - ) - }, - (true, false, true) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut best_plan, - &mut plan_buffers, - ) - }, - (true, true, false) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut best_plan, - &mut plan_buffers, - ) - }, - (true, true, true) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut best_plan, - &mut plan_buffers, - ) - }, - (false, false, false) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut best_plan, - &mut plan_buffers, - ) - }, - (false, false, true) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut best_plan, - &mut plan_buffers, - ) - }, - (false, true, false) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut best_plan, - &mut plan_buffers, - ) - }, - (false, true, true) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut best_plan, - &mut plan_buffers, - ) - }, - }; - // On a no-match segment (the per-literal case that dominates - // near-random input) nothing was emitted, so the stats - // update is a guaranteed no-op (it early-returns on an empty - // plan slice). Skip the per-literal call + its marshalling. - if best_plan.len() > segment_start { - BtMatcher::update_plan_stats_segment( + ($block:ident) => {{ + match (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS, has_ldm) { + (true, false, false) => unsafe { + self.$block::( current, - current_len, - &best_plan[segment_start..], - &mut plan_literals_cursor, - &mut plan_reps, + current_abs_start, + cursor, + plan_litlen, + plan_reps, + profile, &mut opt_state, - S::ACCURATE_PRICE, - ); - } - plan_reps = end_reps; - plan_litlen = end_litlen; - cursor += consumed_len; + &mut best_plan, + &mut plan_buffers, + ) + }, + (true, false, true) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + plan_litlen, + plan_reps, + profile, + &mut opt_state, + &mut best_plan, + &mut plan_buffers, + ) + }, + (true, true, false) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + plan_litlen, + plan_reps, + profile, + &mut opt_state, + &mut best_plan, + &mut plan_buffers, + ) + }, + (true, true, true) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + plan_litlen, + plan_reps, + profile, + &mut opt_state, + &mut best_plan, + &mut plan_buffers, + ) + }, + (false, false, false) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + plan_litlen, + plan_reps, + profile, + &mut opt_state, + &mut best_plan, + &mut plan_buffers, + ) + }, + (false, false, true) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + plan_litlen, + plan_reps, + profile, + &mut opt_state, + &mut best_plan, + &mut plan_buffers, + ) + }, + (false, true, false) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + plan_litlen, + plan_reps, + profile, + &mut opt_state, + &mut best_plan, + &mut plan_buffers, + ) + }, + (false, true, true) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + plan_litlen, + plan_reps, + profile, + &mut opt_state, + &mut best_plan, + &mut plan_buffers, + ) + }, } }}; } @@ -1391,19 +1478,19 @@ impl HcMatchGenerator { feature = "kernel-neon" ))] unsafe { - run_main_loop!(build_optimal_plan_impl_neon); + run_main_loop!(run_optimal_block_neon); } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; match self.table.kernel { #[cfg(feature = "kernel-avx2")] - FastpathKernel::Avx2Bmi2 => run_main_loop!(build_optimal_plan_impl_avx2_bmi2), + FastpathKernel::Avx2Bmi2 => run_main_loop!(run_optimal_block_avx2_bmi2), #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse2 => run_main_loop!(build_optimal_plan_impl_sse2), + FastpathKernel::Sse2 => run_main_loop!(run_optimal_block_sse2), #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse42 => run_main_loop!(build_optimal_plan_impl_sse42), - FastpathKernel::Scalar => run_main_loop!(build_optimal_plan_impl_scalar), + FastpathKernel::Sse42 => run_main_loop!(run_optimal_block_sse42), + FastpathKernel::Scalar => run_main_loop!(run_optimal_block_scalar), } } #[cfg(all( @@ -1412,7 +1499,7 @@ impl HcMatchGenerator { feature = "kernel-simd128" ))] unsafe { - run_main_loop!(build_optimal_plan_impl_simd128); + run_main_loop!(run_optimal_block_simd128); } #[cfg(not(any( all( @@ -1429,7 +1516,7 @@ impl HcMatchGenerator { ) )))] { - run_main_loop!(build_optimal_plan_impl_scalar); + run_main_loop!(run_optimal_block_scalar); } self.table @@ -1468,137 +1555,121 @@ impl HcMatchGenerator { let mut opt_state = core::mem::replace(&mut self.backend.bt_mut().opt_state, HcOptState::new()); opt_state.rescale_freqs(current, S::ACCURATE_PRICE); - let mut seed_reps = self.table.offset_hist; - let (mut cursor, mut seed_litlen) = - self.table.opt_start_cursor_and_litlen(current_abs_start); - let mut seed_literals_cursor = 0usize; + let seed_reps = self.table.offset_hist; + let (cursor, seed_litlen) = self.table.opt_start_cursor_and_litlen(current_abs_start); let mut seed_plan = core::mem::take(&mut self.backend.bt_mut().opt_seed_plan_scratch); seed_plan.clear(); - let match_loop_limit = current_len.saturating_sub(8); let has_ldm = !self.backend.bt_mut().ldm_sequences.is_empty(); - // SIMD tier resolved ONCE (see start_matching_optimal): the per-literal - // seed loop runs under a single kernel-monomorphized expansion, never - // re-entering select_kernel() per segment. + // SIMD tier resolved ONCE (see start_matching_optimal): the seed pass + // runs as one kernel-monomorphized block pass. It keeps the statistics + // and drops the plan (`KEEP_PLAN = false`). macro_rules! run_seed_loop { - ($impl_wrapper:ident) => {{ - while cursor < match_loop_limit { - let remaining_len = current_len - cursor; - let segment_abs_start = current_abs_start + cursor; - let segment_start = seed_plan.len(); - let state = HcOptimalPlanState { - block_offset: cursor, - reps: seed_reps, - litlen: seed_litlen, - profile: seed_profile, - }; - let (_, end_reps, end_litlen, consumed_len) = - match (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS, has_ldm) { - (true, false, false) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut seed_plan, - &mut *plan_buffers, - ) - }, - (true, false, true) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut seed_plan, - &mut *plan_buffers, - ) - }, - (true, true, false) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut seed_plan, - &mut *plan_buffers, - ) - }, - (true, true, true) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut seed_plan, - &mut *plan_buffers, - ) - }, - (false, false, false) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut seed_plan, - &mut *plan_buffers, - ) - }, - (false, false, true) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut seed_plan, - &mut *plan_buffers, - ) - }, - (false, true, false) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut seed_plan, - &mut *plan_buffers, - ) - }, - (false, true, true) => unsafe { - self.$impl_wrapper::( - ¤t[cursor..], - segment_abs_start, - remaining_len, - state, - &opt_state, - &mut seed_plan, - &mut *plan_buffers, - ) - }, - }; - // No-match segment: stats update no-ops on the empty slice - // and the truncate has nothing to drop; skip both. - if seed_plan.len() > segment_start { - BtMatcher::update_plan_stats_segment( + ($block:ident) => {{ + match (S::ACCURATE_PRICE, S::FAVOR_SMALL_OFFSETS, has_ldm) { + (true, false, false) => unsafe { + self.$block::( current, - current_len, - &seed_plan[segment_start..], - &mut seed_literals_cursor, - &mut seed_reps, + current_abs_start, + cursor, + seed_litlen, + seed_reps, + seed_profile, &mut opt_state, - S::ACCURATE_PRICE, - ); - seed_plan.truncate(segment_start); - } - seed_reps = end_reps; - seed_litlen = end_litlen; - cursor += consumed_len; + &mut seed_plan, + &mut *plan_buffers, + ) + }, + (true, false, true) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + seed_litlen, + seed_reps, + seed_profile, + &mut opt_state, + &mut seed_plan, + &mut *plan_buffers, + ) + }, + (true, true, false) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + seed_litlen, + seed_reps, + seed_profile, + &mut opt_state, + &mut seed_plan, + &mut *plan_buffers, + ) + }, + (true, true, true) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + seed_litlen, + seed_reps, + seed_profile, + &mut opt_state, + &mut seed_plan, + &mut *plan_buffers, + ) + }, + (false, false, false) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + seed_litlen, + seed_reps, + seed_profile, + &mut opt_state, + &mut seed_plan, + &mut *plan_buffers, + ) + }, + (false, false, true) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + seed_litlen, + seed_reps, + seed_profile, + &mut opt_state, + &mut seed_plan, + &mut *plan_buffers, + ) + }, + (false, true, false) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + seed_litlen, + seed_reps, + seed_profile, + &mut opt_state, + &mut seed_plan, + &mut *plan_buffers, + ) + }, + (false, true, true) => unsafe { + self.$block::( + current, + current_abs_start, + cursor, + seed_litlen, + seed_reps, + seed_profile, + &mut opt_state, + &mut seed_plan, + &mut *plan_buffers, + ) + }, } }}; } @@ -1608,19 +1679,19 @@ impl HcMatchGenerator { feature = "kernel-neon" ))] unsafe { - run_seed_loop!(build_optimal_plan_impl_neon); + run_seed_loop!(run_optimal_block_neon); } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use crate::encoding::fastpath::FastpathKernel; match self.table.kernel { #[cfg(feature = "kernel-avx2")] - FastpathKernel::Avx2Bmi2 => run_seed_loop!(build_optimal_plan_impl_avx2_bmi2), + FastpathKernel::Avx2Bmi2 => run_seed_loop!(run_optimal_block_avx2_bmi2), #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse2 => run_seed_loop!(build_optimal_plan_impl_sse2), + FastpathKernel::Sse2 => run_seed_loop!(run_optimal_block_sse2), #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse42 => run_seed_loop!(build_optimal_plan_impl_sse42), - FastpathKernel::Scalar => run_seed_loop!(build_optimal_plan_impl_scalar), + FastpathKernel::Sse42 => run_seed_loop!(run_optimal_block_sse42), + FastpathKernel::Scalar => run_seed_loop!(run_optimal_block_scalar), } } #[cfg(all( @@ -1629,7 +1700,7 @@ impl HcMatchGenerator { feature = "kernel-simd128" ))] unsafe { - run_seed_loop!(build_optimal_plan_impl_simd128); + run_seed_loop!(run_optimal_block_simd128); } #[cfg(not(any( all( @@ -1646,7 +1717,7 @@ impl HcMatchGenerator { ) )))] { - run_seed_loop!(build_optimal_plan_impl_scalar); + run_seed_loop!(run_optimal_block_scalar); } seed_plan.clear(); self.backend.bt_mut().opt_seed_plan_scratch = seed_plan; @@ -1711,12 +1782,13 @@ impl HcMatchGenerator { // Nothing in the buffer answers a query yet: the block that filled // it is over, and the parser is about to start another. candidates_searched_at: None, + pass: HcBlockPass::default(), } } - /// NEON-umbrella DP body. Inlines - /// `collect_optimal_candidates_initialized_neon` (and its entire - /// per-position pipeline) directly into the DP loop. + /// NEON-umbrella pass over one block (see [`optimal_block_body!`]). + /// `collect_optimal_candidates_initialized_neon` shares the umbrella, so + /// the per-position pipeline needs no feature re-dispatch. #[cfg(all( target_arch = "aarch64", target_endian = "little", @@ -1724,31 +1796,37 @@ impl HcMatchGenerator { ))] #[target_feature(enable = "neon")] #[allow(clippy::too_many_arguments)] - unsafe fn build_optimal_plan_impl_neon< + unsafe fn run_optimal_block_neon< S: crate::encoding::strategy::Strategy, const ACCURATE_PRICE: bool, const FAVOR_SMALL_OFFSETS: bool, const HAS_LDM: bool, + const KEEP_PLAN: bool, >( &mut self, current: &[u8], current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, + cursor: usize, + litlen: usize, + reps: [u32; 3], + profile: HcOptimalCostProfile, + opt_state: &mut HcOptState, + plan: &mut Vec, buffers: &mut HcOptimalPlanBuffers, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( + ) { + optimal_block_body!( self, S, current, current_abs_start, - current_len, - initial_state, - stats, - out, + cursor, + litlen, + reps, + profile, + opt_state, + plan, buffers, + KEEP_PLAN, collect_optimal_candidates_initialized_neon, crate::encoding::hc::priceset::priceset_range_nonabort_neon::, ) @@ -1763,37 +1841,43 @@ impl HcMatchGenerator { ))] #[target_feature(enable = "sse4.2")] #[allow(clippy::too_many_arguments)] - unsafe fn build_optimal_plan_impl_sse42< + unsafe fn run_optimal_block_sse42< S: crate::encoding::strategy::Strategy, const ACCURATE_PRICE: bool, const FAVOR_SMALL_OFFSETS: bool, const HAS_LDM: bool, + const KEEP_PLAN: bool, >( &mut self, current: &[u8], current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, + cursor: usize, + litlen: usize, + reps: [u32; 3], + profile: HcOptimalCostProfile, + opt_state: &mut HcOptState, + plan: &mut Vec, buffers: &mut HcOptimalPlanBuffers, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( + ) { + optimal_block_body!( self, S, current, current_abs_start, - current_len, - initial_state, - stats, - out, + cursor, + litlen, + reps, + profile, + opt_state, + plan, buffers, + KEEP_PLAN, collect_optimal_candidates_initialized_sse42, crate::encoding::hc::priceset::priceset_range_nonabort_sse41::, ) } - /// SSE2 twin of [`Self::build_optimal_plan_impl_sse42`] for x86 CPUs + /// SSE2 twin of [`Self::run_optimal_block_sse42`] for x86 CPUs /// without SSE4.1/4.2: same 128-bit pipeline, with the price set using /// the SSE2 unsigned-compare emulation. #[cfg(all( @@ -1802,31 +1886,37 @@ impl HcMatchGenerator { ))] #[target_feature(enable = "sse2")] #[allow(clippy::too_many_arguments)] - unsafe fn build_optimal_plan_impl_sse2< + unsafe fn run_optimal_block_sse2< S: crate::encoding::strategy::Strategy, const ACCURATE_PRICE: bool, const FAVOR_SMALL_OFFSETS: bool, const HAS_LDM: bool, + const KEEP_PLAN: bool, >( &mut self, current: &[u8], current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, + cursor: usize, + litlen: usize, + reps: [u32; 3], + profile: HcOptimalCostProfile, + opt_state: &mut HcOptState, + plan: &mut Vec, buffers: &mut HcOptimalPlanBuffers, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( + ) { + optimal_block_body!( self, S, current, current_abs_start, - current_len, - initial_state, - stats, - out, + cursor, + litlen, + reps, + profile, + opt_state, + plan, buffers, + KEEP_PLAN, collect_optimal_candidates_initialized_sse2, crate::encoding::hc::priceset::priceset_range_nonabort_sse2::, ) @@ -1838,31 +1928,37 @@ impl HcMatchGenerator { ))] #[target_feature(enable = "avx2,bmi2")] #[allow(clippy::too_many_arguments)] - unsafe fn build_optimal_plan_impl_avx2_bmi2< + unsafe fn run_optimal_block_avx2_bmi2< S: crate::encoding::strategy::Strategy, const ACCURATE_PRICE: bool, const FAVOR_SMALL_OFFSETS: bool, const HAS_LDM: bool, + const KEEP_PLAN: bool, >( &mut self, current: &[u8], current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, + cursor: usize, + litlen: usize, + reps: [u32; 3], + profile: HcOptimalCostProfile, + opt_state: &mut HcOptState, + plan: &mut Vec, buffers: &mut HcOptimalPlanBuffers, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( + ) { + optimal_block_body!( self, S, current, current_abs_start, - current_len, - initial_state, - stats, - out, + cursor, + litlen, + reps, + profile, + opt_state, + plan, buffers, + KEEP_PLAN, collect_optimal_candidates_initialized_avx2_bmi2, crate::encoding::hc::priceset::priceset_range_nonabort_avx2::, ) @@ -1889,31 +1985,37 @@ impl HcMatchGenerator { allow(dead_code) )] #[allow(clippy::too_many_arguments)] - fn build_optimal_plan_impl_scalar< + fn run_optimal_block_scalar< S: crate::encoding::strategy::Strategy, const ACCURATE_PRICE: bool, const FAVOR_SMALL_OFFSETS: bool, const HAS_LDM: bool, + const KEEP_PLAN: bool, >( &mut self, current: &[u8], current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, + cursor: usize, + litlen: usize, + reps: [u32; 3], + profile: HcOptimalCostProfile, + opt_state: &mut HcOptState, + plan: &mut Vec, buffers: &mut HcOptimalPlanBuffers, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( + ) { + optimal_block_body!( self, S, current, current_abs_start, - current_len, - initial_state, - stats, - out, + cursor, + litlen, + reps, + profile, + opt_state, + plan, buffers, + KEEP_PLAN, collect_optimal_candidates_initialized_scalar, crate::encoding::hc::priceset::priceset_range_nonabort_scalar::, ) @@ -1932,31 +2034,37 @@ impl HcMatchGenerator { // target_feature fn. #[allow(unused_unsafe)] #[allow(clippy::too_many_arguments)] - unsafe fn build_optimal_plan_impl_simd128< + unsafe fn run_optimal_block_simd128< S: crate::encoding::strategy::Strategy, const ACCURATE_PRICE: bool, const FAVOR_SMALL_OFFSETS: bool, const HAS_LDM: bool, + const KEEP_PLAN: bool, >( &mut self, current: &[u8], current_abs_start: usize, - current_len: usize, - initial_state: HcOptimalPlanState, - stats: &HcOptState, - out: &mut Vec, + cursor: usize, + litlen: usize, + reps: [u32; 3], + profile: HcOptimalCostProfile, + opt_state: &mut HcOptState, + plan: &mut Vec, buffers: &mut HcOptimalPlanBuffers, - ) -> (u32, [u32; 3], usize, usize) { - build_optimal_plan_impl_body!( + ) { + optimal_block_body!( self, S, current, current_abs_start, - current_len, - initial_state, - stats, - out, + cursor, + litlen, + reps, + profile, + opt_state, + plan, buffers, + KEEP_PLAN, collect_optimal_candidates_initialized_simd128, crate::encoding::hc::priceset::priceset_range_nonabort_simd128::, ) @@ -2030,7 +2138,7 @@ impl HcMatchGenerator { /// collect / HC chain walk) runs inside a single `target_feature` /// umbrella — all inner SIMD probes inline without ABI barriers. /// - /// The on-encode hot path bypasses this dispatcher: `build_optimal_plan_impl_` + /// The on-encode hot path bypasses this dispatcher: `run_optimal_block_` /// calls the matching `_` variant directly. This entry is kept /// for the cfg(test)-only `collect_optimal_candidates` shim and any /// future caller that isn't already inside a kernel umbrella. diff --git a/zstd/src/encoding/opt/types.rs b/zstd/src/encoding/opt/types.rs index be074596d..dbca5fe98 100644 --- a/zstd/src/encoding/opt/types.rs +++ b/zstd/src/encoding/opt/types.rs @@ -131,4 +131,24 @@ pub(crate) struct HcOptimalPlanBuffers { /// searched and the re-entry reads the answer instead of asking again. /// `None` whenever the buffer's contents do not answer any query. pub(crate) candidates_searched_at: Option<(usize, usize)>, + /// Where the block pass stands between segments. Held here, in memory, + /// rather than in locals of the pass: the pass needs it only at segment + /// boundaries, and as locals it stayed live across the whole segment body, + /// taking registers from the DP's own loops. Upstream keeps the same state + /// behind pointers (`rep`, `ms->opt`) for the same reason. + pub(crate) pass: HcBlockPass, +} + +/// The block pass's position between segments: see +/// [`HcOptimalPlanBuffers::pass`]. +#[derive(Copy, Clone, Default)] +pub(crate) struct HcBlockPass { + /// Block offset of the next segment. + pub(crate) cursor: usize, + /// Literals pending before `cursor`. + pub(crate) litlen: usize, + /// Repeat offsets at `cursor`. + pub(crate) reps: [u32; 3], + /// Block offset the statistics update has consumed literals up to. + pub(crate) literals_cursor: usize, } diff --git a/zstd/src/encoding/strategy.rs b/zstd/src/encoding/strategy.rs index 1d9cc3e78..32ef81d3d 100644 --- a/zstd/src/encoding/strategy.rs +++ b/zstd/src/encoding/strategy.rs @@ -17,9 +17,10 @@ //! └─ S::USE_BT == true → start_matching_optimal:: //! ├─ HcOptimalCostProfile::const_for_strategy::() //! ├─ should_run_btultra2_seed_pass:: // const false unless S = BtUltra2 -//! └─ select_kernel() once, then per-segment loop: -//! └─ build_optimal_plan_impl_:: -//! └─ build_optimal_plan_impl_body!(S) +//! └─ select_kernel() once, then one pass over the block: +//! └─ run_optimal_block_:: +//! └─ optimal_block_body!: per segment +//! build_optimal_plan_impl_body!(S) //! ├─ S::OPT_LEVEL == 0 → abort_on_worse_match //! ├─ S::OPT_LEVEL >= 2 → opt_level (refined) //! └─ $collect:: From 21282a84a4fba48570df7efc77adac0855bd6a53 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 15:38:31 +0300 Subject: [PATCH 18/36] perf(encoding): search on the repeat history in place The optimal parser handed its per-position search a query struct: the three repeat offsets, the pending literal count and an optional long-distance candidate, 48 bytes marshalled through memory on every position, and the search merged that candidate into its own result at three exits. Upstream's opt loop calls ZSTD_btGetAllMatches with a pointer to opt[cur].rep and an ll0 flag, and adds the long-distance candidate afterwards (ZSTD_optLdm_processMatchCandidate). The parser now has the same shape: - the search takes `&[u32; 3]` pointing at the node's own repeat history and `ll0: bool`, so nothing is copied per position - the long-distance candidate joins after the search in the parser, through `BtMatcher::push_ldm_candidate` (ZSTD_optLdm_maybeAddMatch); `HcCandidateQuery` survives only as the test entry's argument - the unused `bt_insert_and_collect_matches` dispatcher and its five kernel wrappers are gone (the search body is expanded inside collect), and with them the cost profile's `max_chain_depth` field, which only they read Output changes only with long-distance matching. The search returns early when a repeat or hash3 match reaches the sufficient length or the block end, and the candidate merged inside it was dropped on every such exit; upstream adds it whatever the search did. A regression test covers that case (red on the previous code, green now). --long=27 on 8 MiB of source text, frame bytes before -> after (libzstd), every frame round-trips through libzstd: L16 2,119,499 -> 2,119,471 (2,090,198) L17 2,099,752 -> 2,099,612 (2,065,913) L18 2,087,944 -> 2,087,718 (2,050,924) L19 2,083,481 -> 2,083,409 (2,046,434) L21 2,082,749 -> 2,082,733 (2,045,763) L22 unchanged Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary. callgrind, 3 frames of z000033[..200000]: L13 396,245,242 -> 387,035,914 (-2.32%) L16 636,903,867 -> 625,216,188 (-1.84%) L19 946,328,078 -> 929,011,865 (-1.83%) runner1 task-clock, 50 frames, interleaved with the previous build and libzstd, every arm pinned to one core, three rounds, ranges disjoint: L16 1837-1848 -> 1796-1799 ms (-2.3%), L19 2724-2743 -> 2686-2698 ms (-1.5%), L13 1082-1091 -> 1072-1079 ms (-1.1%, under the 1.5% floor), control L12 flat (483-490 -> 483-484). Part of #128 --- zstd/src/encoding/bt/mod.rs | 15 + zstd/src/encoding/cost_model/mod.rs | 6 +- zstd/src/encoding/hc/generator.rs | 4 +- zstd/src/encoding/hc/optimal.rs | 177 +++++------ zstd/src/encoding/match_generator/tests.rs | 57 +++- zstd/src/encoding/match_table/storage.rs | 340 +-------------------- zstd/src/encoding/opt/types.rs | 7 +- 7 files changed, 165 insertions(+), 441 deletions(-) diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index 2579b034c..b75eade6c 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -155,6 +155,21 @@ impl BtMatcher { false } + /// Append the long-distance candidate after the search when it is at + /// least `min_match_len` and longer than every candidate the search found + /// (upstream zstd `ZSTD_optLdm_maybeAddMatch`, zstd_opt.c). The search + /// ladder keeps `out` sorted by strictly increasing length, so its last + /// entry is the bar. + #[inline(always)] + pub(crate) fn push_ldm_candidate( + out: &mut Vec, + ldm: MatchCandidate, + min_match_len: usize, + ) { + let mut best_len = out.last().map_or(0, |c| c.match_len); + let _ = Self::push_candidate_ladder(out, &mut best_len, ldm, min_match_len); + } + pub(crate) fn new() -> Self { Self { opt_state: HcOptState::new(), diff --git a/zstd/src/encoding/cost_model/mod.rs b/zstd/src/encoding/cost_model/mod.rs index 2f14d5e85..bb9d01947 100644 --- a/zstd/src/encoding/cost_model/mod.rs +++ b/zstd/src/encoding/cost_model/mod.rs @@ -442,7 +442,6 @@ impl HcOptState { #[derive(Copy, Clone)] pub(crate) struct HcOptimalCostProfile { - pub(crate) max_chain_depth: usize, pub(crate) sufficient_match_len: usize, pub(crate) favor_small_offsets: bool, } @@ -455,10 +454,10 @@ impl HcOptimalCostProfile { /// this entry — there is no runtime peer. /// /// The `debug_assert!(S::USE_BT, …)` enforces that - /// `MAX_CHAIN_DEPTH` / `SUFFICIENT_MATCH_LEN` are only consulted + /// `SUFFICIENT_MATCH_LEN` is only consulted /// for BT-walking strategies, since non-BT strategies /// (`Fast` / `Dfast` / `Greedy` / `Lazy`) carry placeholder - /// values for those consts — see the `MAX_CHAIN_DEPTH` doc + /// values for the BT consts; see the `MAX_CHAIN_DEPTH` doc /// comment on each of those strategy types. #[inline] pub(crate) fn const_for_strategy() -> Self { @@ -469,7 +468,6 @@ impl HcOptimalCostProfile { profile is only meaningful when the BT walker is active.", ); Self { - max_chain_depth: S::MAX_CHAIN_DEPTH, sufficient_match_len: S::SUFFICIENT_MATCH_LEN, favor_small_offsets: S::FAVOR_SMALL_OFFSETS, } diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index 4a47b18c5..d88f10cc1 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -588,7 +588,7 @@ macro_rules! bt_insert_and_collect_matches_body { $best_len_for_skip:ident, $out:ident, $reps:ident, - $lit_len:ident, + $ll0:ident, $use_hash3:expr, $cpl:path, $cmf:path $(,)? @@ -693,7 +693,7 @@ macro_rules! bt_insert_and_collect_matches_body { ) }; } - if $lit_len == 0 { + if $ll0 { probe!($reps[1] as usize); probe!($reps[2] as usize); probe!(($reps[0] as usize).wrapping_sub(1)); diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index bb8e9f693..aeab54212 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -24,6 +24,8 @@ use alloc::vec::Vec; // does not count macro-body references, so it reports every one of these as // "unused" even though each macro expansion requires them (gating or removing // any one breaks the lib build). Suppress the false positive on the group. +#[cfg(test)] +use crate::encoding::opt::types::HcCandidateQuery; #[allow(unused_imports)] use crate::encoding::{ bt::BtMatcher, @@ -37,8 +39,8 @@ use crate::encoding::{ match_table::storage::HC3_HASH_LOG, opt::ldm::{HcOptLdmState, HcRawSeqStore}, opt::types::{ - HcBlockPass, HcCandidateQuery, HcOptimalNode, HcOptimalPlanBuffers, HcOptimalPlanState, - HcOptimalSequence, MatchCandidate, + HcBlockPass, HcOptimalNode, HcOptimalPlanBuffers, HcOptimalPlanState, HcOptimalSequence, + MatchCandidate, }, }; @@ -210,11 +212,8 @@ macro_rules! build_optimal_plan_impl_body { $current_abs_start + skipped_literals, current_abs_end, profile.sufficient_match_len, - HcCandidateQuery { - reps: initial_reps, - lit_len: initial_litlen + skipped_literals, - ldm_candidate: None, - }, + &initial_reps, + initial_litlen + skipped_literals == 0, &mut *candidates, ) }; @@ -318,14 +317,16 @@ macro_rules! build_optimal_plan_impl_body { $current_abs_start, current_abs_end, profile.sufficient_match_len, - HcCandidateQuery { - reps: initial_reps, - lit_len: initial_litlen, - ldm_candidate: seed_ldm, - }, + &initial_reps, + initial_litlen == 0, &mut *candidates, ) }; + // The long-distance candidate joins after the search, as + // upstream's `ZSTD_optLdm_processMatchCandidate` does. + if let Some(ldm) = seed_ldm { + BtMatcher::push_ldm_candidate(&mut *candidates, ldm, min_match_len); + } } if !candidates.is_empty() { // Deferred price-cache setup: the arena slices are two disjoint @@ -687,34 +688,35 @@ macro_rules! build_optimal_plan_impl_body { } let abs_pos = $current_abs_start + pos; - let ldm_candidate = if HAS_LDM { - $self.backend.bt_mut().ldm_process_match_candidate( - &mut opt_ldm, - pos, - $current_len - pos, - min_match_len, - ) - } else { - None - }; candidates.clear(); - // SAFETY: same umbrella as `$collect`. Query fields are read - // fresh here (consumed into the call's argument) so they do not - // stay live across the call; the post-call reads below are a - // separate, fresh load of the same stable `nodes[pos]`. + // SAFETY: same umbrella as `$collect`. The search reads the repeat + // history in place through `nodes[pos]`, as upstream passes + // `opt[cur].rep`, so nothing is copied into the call and nothing + // stays live across it; the post-call reads below are a fresh load + // of the same stable `nodes[pos]`. unsafe { $self.$collect::<$strategy_ty>( abs_pos, current_abs_end, profile.sufficient_match_len, - HcCandidateQuery { - reps: (*nodes.add(pos)).reps, - lit_len: (*nodes.add(pos)).litlen as usize, - ldm_candidate, - }, + &(*nodes.add(pos)).reps, + (*nodes.add(pos)).litlen == 0, &mut *candidates, ) }; + if HAS_LDM { + // Upstream `ZSTD_optLdm_processMatchCandidate`: the producer + // advances past this position and its candidate joins after + // the search. + if let Some(ldm) = $self.backend.bt_mut().ldm_process_match_candidate( + &mut opt_ldm, + pos, + $current_len - pos, + min_match_len, + ) { + BtMatcher::push_ldm_candidate(&mut *candidates, ldm, min_match_len); + } + } // Post-call reads of opt[cur]: fresh, born after `$collect`, so // never part of the cross-call live set (see memory-resident note // above). `nodes[pos]` is untouched by `$collect`. @@ -1147,7 +1149,8 @@ macro_rules! collect_optimal_candidates_initialized_body { $abs_pos:ident, $current_abs_end:ident, $sufficient_match_len:ident, - $query:ident, + $reps:ident, + $ll0:ident, $out:ident, $bt_insert_step:ident, $cpl:path, @@ -1170,20 +1173,8 @@ macro_rules! collect_optimal_candidates_initialized_body { ); debug_assert!(!$self.table.chain_table().is_empty()); let min_match_len = HC_OPT_MIN_MATCH_LEN; - let reps = $query.reps; - let lit_len = $query.lit_len; - let ldm_candidate = $query.ldm_candidate; $out.clear(); if $abs_pos < $self.table.skip_insert_until_abs { - if let Some(ldm) = ldm_candidate { - let mut best_len_for_skip = 0usize; - let _ = crate::encoding::bt::BtMatcher::push_candidate_ladder( - $out, - &mut best_len_for_skip, - ldm, - min_match_len, - ); - } return; } { @@ -1214,15 +1205,6 @@ macro_rules! collect_optimal_candidates_initialized_body { } let current_idx = $abs_pos - $self.table.history_abs_start; if current_idx + 4 > $self.table.live_history().len() { - if let Some(ldm) = ldm_candidate { - let mut best_len_for_skip = 0usize; - let _ = crate::encoding::bt::BtMatcher::push_candidate_ladder( - $out, - &mut best_len_for_skip, - ldm, - min_match_len, - ); - } return; } let mut best_len_for_skip = 0usize; @@ -1248,21 +1230,13 @@ macro_rules! collect_optimal_candidates_initialized_body { min_match_len, best_len_ref, $out, - reps, - lit_len, + $reps, + $ll0, use_hash3, $cpl, $cmf, ); } - if let Some(ldm) = ldm_candidate { - let _ = crate::encoding::bt::BtMatcher::push_candidate_ladder( - $out, - &mut best_len_for_skip, - ldm, - min_match_len, - ); - } }}; } impl HcMatchGenerator { @@ -2081,6 +2055,8 @@ impl HcMatchGenerator { ) { use crate::encoding::strategy::{self, StrategyTag}; self.table.ensure_tables(); + let reps = &query.reps; + let ll0 = query.lit_len == 0; // Dispatch purely from `self.strategy_tag` (set by // `configure()`). Tests must configure the matcher the same // way production does — wiring up `table.hash3_log` directly @@ -2092,7 +2068,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ), StrategyTag::BtUltra => self @@ -2100,7 +2077,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ), StrategyTag::Btlazy2 => self @@ -2108,14 +2086,16 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ), StrategyTag::BtOpt => self.collect_optimal_candidates_initialized::( abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ), StrategyTag::Fast | StrategyTag::Dfast | StrategyTag::Greedy | StrategyTag::Lazy => { @@ -2131,6 +2111,9 @@ impl HcMatchGenerator { ) } } + if let Some(ldm) = query.ldm_candidate { + BtMatcher::push_ldm_candidate(out, ldm, HC_OPT_MIN_MATCH_LEN); + } } /// Cross-platform entry. Picks the kernel-specific variant so the per- @@ -2159,7 +2142,8 @@ impl HcMatchGenerator { abs_pos: usize, current_abs_end: usize, sufficient_match_len: usize, - query: HcCandidateQuery, + reps: &[u32; 3], + ll0: bool, out: &mut Vec, ) { #[cfg(all( @@ -2172,7 +2156,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ) } @@ -2186,7 +2171,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ) }, @@ -2196,7 +2182,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ) }, @@ -2206,7 +2193,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ) }, @@ -2214,7 +2202,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ), } @@ -2233,7 +2222,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ) } @@ -2256,7 +2246,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, ) } @@ -2281,7 +2272,8 @@ impl HcMatchGenerator { abs_pos: usize, current_abs_end: usize, sufficient_match_len: usize, - query: HcCandidateQuery, + reps: &[u32; 3], + ll0: bool, out: &mut Vec, ) { collect_optimal_candidates_initialized_body!( @@ -2290,7 +2282,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, bt_insert_step_no_rebase_neon, crate::encoding::fastpath::neon::common_prefix_len_ptr, @@ -2311,7 +2304,8 @@ impl HcMatchGenerator { abs_pos: usize, current_abs_end: usize, sufficient_match_len: usize, - query: HcCandidateQuery, + reps: &[u32; 3], + ll0: bool, out: &mut Vec, ) { collect_optimal_candidates_initialized_body!( @@ -2320,7 +2314,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, bt_insert_step_no_rebase_sse2, crate::encoding::fastpath::sse2::common_prefix_len_ptr, @@ -2347,7 +2342,8 @@ impl HcMatchGenerator { abs_pos: usize, current_abs_end: usize, sufficient_match_len: usize, - query: HcCandidateQuery, + reps: &[u32; 3], + ll0: bool, out: &mut Vec, ) { collect_optimal_candidates_initialized_body!( @@ -2356,7 +2352,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, bt_insert_step_no_rebase_sse2, crate::encoding::fastpath::sse2::common_prefix_len_ptr, @@ -2377,7 +2374,8 @@ impl HcMatchGenerator { abs_pos: usize, current_abs_end: usize, sufficient_match_len: usize, - query: HcCandidateQuery, + reps: &[u32; 3], + ll0: bool, out: &mut Vec, ) { collect_optimal_candidates_initialized_body!( @@ -2386,7 +2384,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, bt_insert_step_no_rebase_avx2_bmi2, crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, @@ -2414,7 +2413,8 @@ impl HcMatchGenerator { abs_pos: usize, current_abs_end: usize, sufficient_match_len: usize, - query: HcCandidateQuery, + reps: &[u32; 3], + ll0: bool, out: &mut Vec, ) { collect_optimal_candidates_initialized_body!( @@ -2423,7 +2423,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, bt_insert_step_no_rebase_simd128, crate::encoding::fastpath::simd128::common_prefix_len_ptr, @@ -2447,7 +2448,8 @@ impl HcMatchGenerator { abs_pos: usize, current_abs_end: usize, sufficient_match_len: usize, - query: HcCandidateQuery, + reps: &[u32; 3], + ll0: bool, out: &mut Vec, ) { collect_optimal_candidates_initialized_body!( @@ -2456,7 +2458,8 @@ impl HcMatchGenerator { abs_pos, current_abs_end, sufficient_match_len, - query, + reps, + ll0, out, bt_insert_step_no_rebase_scalar, crate::encoding::fastpath::scalar::common_prefix_len_ptr, diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index 56097db81..be70eb96d 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -959,19 +959,19 @@ fn btultra2_profile_disables_small_offset_handicap() { } #[test] -fn btultra_profile_keeps_search_depth_budget() { - let p = HcOptimalCostProfile::const_for_strategy::(); +fn btultra_keeps_search_depth_budget() { assert_eq!( - p.max_chain_depth, 64, + ::MAX_CHAIN_DEPTH, + 64, "btultra chain-depth budget must match clevels.h level 18 searchLog 6 (1 << 6 = 64)" ); } #[test] -fn btopt_profile_keeps_search_depth_budget() { - let p = HcOptimalCostProfile::const_for_strategy::(); +fn btopt_keeps_search_depth_budget() { assert_eq!( - p.max_chain_depth, 32, + ::MAX_CHAIN_DEPTH, + 32, "btopt should not cap chain depth below upstream zstd btopt search budget" ); } @@ -1670,6 +1670,51 @@ fn hc_ldm_candidates_are_merged_into_optimal_candidates() { ); } +/// A repeat match past the sufficient length ends the search early, and the +/// long-distance candidate must still join afterwards: upstream adds it after +/// `ZSTD_btGetAllMatches` whatever the search did (zstd_opt.c, +/// `ZSTD_optLdm_processMatchCandidate`). Merged inside the search, it was lost +/// on every early exit. +#[test] +fn hc_ldm_candidate_survives_the_search_early_exit() { + let mut hc = HcMatchGenerator::new(512); + hc.strategy_tag = crate::encoding::strategy::StrategyTag::BtOpt; + // rep0 = 10 matches `abcde` at position 10, then `Y` meets `X`: length 5. + hc.table.history = b"abcdeXXXXXabcdeYYYYYYYYYYYYYYYYYYYY".to_vec(); + hc.table.history_start = 0; + hc.table.history_abs_start = 0; + hc.table.search_depth = 32; + + let abs_pos = 10usize; + let ldm = MatchCandidate { + start: abs_pos, + offset: 7, + match_len: 12, + }; + let mut out = Vec::new(); + hc.collect_optimal_candidates( + abs_pos, + hc.table.history.len(), + // Below the repeat match's length, so the repeat probe ends the search. + 4, + HcCandidateQuery { + reps: [10, 20, 30], + lit_len: 1, + ldm_candidate: Some(ldm), + }, + &mut out, + ); + assert!( + out.iter().any(|c| c.offset == 10 && c.match_len == 5), + "the repeat match that ends the search is kept" + ); + assert_eq!( + out.last().map(|c| (c.offset, c.match_len)), + Some((ldm.offset, ldm.match_len)), + "the longer long-distance candidate joins after the early exit" + ); +} + #[test] fn btultra_and_btultra2_both_keep_dictionary_candidates() { // Routes the BtUltra2 / BtUltra fixture through the production diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index bc979e72f..f9965d3e9 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -20,7 +20,7 @@ use alloc::vec::Vec; use super::super::Sequence; use super::super::blocks::encode_offset_with_history; -use super::super::cost_model::{HC_OPT_NUM, HcOptimalCostProfile}; +use super::super::cost_model::HC_OPT_NUM; use super::super::dict_attach::DictAttach; use super::super::hc::HC_MIN_MATCH_LEN; use super::super::opt::types::{HcOptimalSequence, MatchCandidate}; @@ -1880,344 +1880,6 @@ impl MatchTable { ) } - /// Stage D: cross-platform dispatcher for the BT collect-matches walker. - /// External / test entry — the hot path bypasses this and calls the - /// per-kernel variant from inside the surrounding - /// `collect_optimal_candidates_initialized_` umbrella. - #[allow(dead_code)] - #[allow(clippy::too_many_arguments)] - #[inline(always)] - pub(crate) fn bt_insert_and_collect_matches( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: &HcOptimalCostProfile, - min_match_len: usize, - best_len_for_skip: &mut usize, - out: &mut Vec, - reps: [u32; 3], - lit_len: usize, - use_hash3: bool, - ) { - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - unsafe { - self.bt_insert_and_collect_matches_neon( - abs_pos, - current_abs_end, - profile, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - ) - } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - use crate::encoding::fastpath::FastpathKernel; - match self.kernel { - #[cfg(feature = "kernel-avx2")] - FastpathKernel::Avx2Bmi2 => unsafe { - self.bt_insert_and_collect_matches_avx2_bmi2( - abs_pos, - current_abs_end, - profile, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - ) - }, - #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - self.bt_insert_and_collect_matches_sse2( - abs_pos, - current_abs_end, - profile, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - ) - }, - FastpathKernel::Scalar => self.bt_insert_and_collect_matches_scalar( - abs_pos, - current_abs_end, - profile, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - ), - } - } - // wasm resolves `simd128` at compile time (no runtime detection), so - // the tier comes from `cfg`, not from `self.kernel`. - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - // SAFETY: the `cfg` above establishes `simd128` at compile time, which - // is exactly the umbrella the callee declares. - unsafe { - self.bt_insert_and_collect_matches_simd128( - abs_pos, - current_abs_end, - profile, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - ) - } - #[cfg(not(any( - all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ), - target_arch = "x86", - target_arch = "x86_64", - all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ) - )))] - { - self.bt_insert_and_collect_matches_scalar( - abs_pos, - current_abs_end, - profile, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - ) - } - } - - /// NEON-umbrella variant of `bt_insert_and_collect_matches`. - /// - /// # Safety - /// AArch64 with NEON (baseline). - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - #[target_feature(enable = "neon")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn bt_insert_and_collect_matches_neon( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: &HcOptimalCostProfile, - min_match_len: usize, - best_len_for_skip: &mut usize, - out: &mut Vec, - reps: [u32; 3], - lit_len: usize, - use_hash3: bool, - ) { - let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_and_collect_matches_body!( - self, - search_depth, - abs_pos, - current_abs_end, - profile.sufficient_match_len, - profile.max_chain_depth, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - crate::encoding::fastpath::neon::common_prefix_len_ptr, - crate::encoding::fastpath::neon::count_match_from_indices, - ) - } - - /// SSE2 umbrella variant of `bt_insert_and_collect_matches`. - /// - /// # Safety - /// x86/x86_64 with SSE2. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - #[target_feature(enable = "sse2")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn bt_insert_and_collect_matches_sse2( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: &HcOptimalCostProfile, - min_match_len: usize, - best_len_for_skip: &mut usize, - out: &mut Vec, - reps: [u32; 3], - lit_len: usize, - use_hash3: bool, - ) { - let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_and_collect_matches_body!( - self, - search_depth, - abs_pos, - current_abs_end, - profile.sufficient_match_len, - profile.max_chain_depth, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - crate::encoding::fastpath::sse2::common_prefix_len_ptr, - crate::encoding::fastpath::sse2::count_match_from_indices, - ) - } - - /// AVX2+BMI2 umbrella variant of `bt_insert_and_collect_matches`. - /// - /// # Safety - /// x86/x86_64 with AVX2 + BMI2. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - #[target_feature(enable = "avx2,bmi2")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn bt_insert_and_collect_matches_avx2_bmi2( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: &HcOptimalCostProfile, - min_match_len: usize, - best_len_for_skip: &mut usize, - out: &mut Vec, - reps: [u32; 3], - lit_len: usize, - use_hash3: bool, - ) { - let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_and_collect_matches_body!( - self, - search_depth, - abs_pos, - current_abs_end, - profile.sufficient_match_len, - profile.max_chain_depth, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, - crate::encoding::fastpath::avx2_bmi2::count_match_from_indices, - ) - } - - /// WebAssembly `simd128` umbrella BT collect-matches walker. - /// - /// # Safety - /// wasm32 with `simd128` enabled at compile time. - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - #[target_feature(enable = "simd128")] - #[allow(clippy::too_many_arguments)] - pub(crate) unsafe fn bt_insert_and_collect_matches_simd128( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: &HcOptimalCostProfile, - min_match_len: usize, - best_len_for_skip: &mut usize, - out: &mut Vec, - reps: [u32; 3], - lit_len: usize, - use_hash3: bool, - ) { - let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_and_collect_matches_body!( - self, - search_depth, - abs_pos, - current_abs_end, - profile.sufficient_match_len, - profile.max_chain_depth, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - crate::encoding::fastpath::simd128::common_prefix_len_ptr, - crate::encoding::fastpath::simd128::count_match_from_indices, - ) - } - - /// Scalar fallback BT collect-matches walker. Compiled unless the NEON - /// tier covers this target. - #[cfg(not(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - )))] - #[allow(clippy::too_many_arguments)] - pub(crate) fn bt_insert_and_collect_matches_scalar( - &mut self, - abs_pos: usize, - current_abs_end: usize, - profile: &HcOptimalCostProfile, - min_match_len: usize, - best_len_for_skip: &mut usize, - out: &mut Vec, - reps: [u32; 3], - lit_len: usize, - use_hash3: bool, - ) { - let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_and_collect_matches_body!( - self, - search_depth, - abs_pos, - current_abs_end, - profile.sufficient_match_len, - profile.max_chain_depth, - min_match_len, - best_len_for_skip, - out, - reps, - lit_len, - use_hash3, - crate::encoding::fastpath::scalar::common_prefix_len_ptr, - crate::encoding::fastpath::scalar::count_match_from_indices, - ) - } - /// BT-side history replay after [`Self::begin_rebase`]. Re-walks /// `history_start..abs_pos` through the BT step so the pointer-pair /// table is consistent with the freshly reset `position_base`. diff --git a/zstd/src/encoding/opt/types.rs b/zstd/src/encoding/opt/types.rs index dbca5fe98..c8265e9ca 100644 --- a/zstd/src/encoding/opt/types.rs +++ b/zstd/src/encoding/opt/types.rs @@ -61,9 +61,10 @@ pub(crate) struct HcOptimalSequence { pub(crate) lit_len: u32, } -/// Inputs to the per-position candidate collection step. Bundled so the -/// `collect_optimal_candidates_initialized_body!` macro can hand-roll the -/// argument list once. +/// One position's search as the test entry `collect_optimal_candidates` takes +/// it: the repeat history, the pending literal run, and an optional +/// long-distance candidate merged after the search. +#[cfg(test)] #[derive(Copy, Clone)] pub(crate) struct HcCandidateQuery { pub(crate) reps: [u32; 3], From e20d1bb610485ac61f52fc889afe17269e5a8845 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 16:40:02 +0300 Subject: [PATCH 19/36] perf(encoding): insert a tree catch-up run in one call Before each search the optimal parser inserts the positions it skipped into the binary tree, and it did so with one call per position. Each call re-derived everything that does not depend on the position: the live history slice, the hash and pair table bases with their bounds checks, the pair mask from the chain log, the window floor for the run's target and the coordinate biases, then spilled most of them to the stack before walking a node. At level 13 that prologue was about 170 of the 366 instructions per inserted position, and a catch-up run averages 2.4 positions (130,713 insertions in 53,922 runs over 3 frames of z000033[..200000]). Upstream runs the whole catch-up in ZSTD_updateTree_internal with ZSTD_insertBt1 inlined into its loop, so those values are resolved once per run. The insertion now takes a range, `bt_insert_range(from, stop, end, target)`, and returns the cursor after the last insertion (a long match still carries it past the target, as upstream's `idx += ZSTD_insertBt1(...)` does). Everything position-independent is resolved once per run. Every caller moved over: - the parser's catch-up makes one call per run - `bt_update_tree_until` is one function over the dispatcher instead of five per-kernel copies that existed only to inline the per-position step; a run that may need a rebase still goes one position at a time - the rebase replay inserts its prefix in one call - the sparse incompressible path inserts one position per call, as before Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary, and with --long=27 at L16-L22 on 8 MiB of source text. callgrind, 3 frames of z000033[..200000]: L13 387,035,914 -> 378,521,611 (-2.20%) L16 625,216,188 -> 625,866,312 (+0.10%) L19 929,011,865 -> 929,718,179 (+0.08%) At level 13 the insertion falls 47.9M -> 44.8M and the search 130.6M -> 125.2M. btultra searches nearly every position, so it has almost nothing to catch up (the insertion is 2.2M -> 1.8M at level 16), and its search moves +1.0M from code generation around the call. runner1 task-clock, 50 frames, interleaved with the previous build and libzstd, every arm pinned to one core, two runs of three rounds: L13 1068-1086 -> 1049-1058 ms (-2.0%, disjoint in both runs), L16 1799-1835 -> 1821-1834 ms (overlapping, not established), L19 flat, control L12 flat. Part of #128 --- zstd/src/encoding/bt/mod.rs | 2 +- zstd/src/encoding/fastpath/mod.rs | 4 +- zstd/src/encoding/hc/generator.rs | 448 +++++++++--------- zstd/src/encoding/hc/optimal.rs | 46 +- zstd/src/encoding/match_table/storage.rs | 365 +++----------- .../match_table/storage/storage_tests.rs | 2 +- 6 files changed, 339 insertions(+), 528 deletions(-) diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index b75eade6c..bd218287a 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -4,7 +4,7 @@ //! model (`opt_state`), the optimal-parser scratch buffers //! (`opt_*_scratch` / `opt_*_generation` / `opt_*_stamp`), and the //! LDM long-distance match buffer (`ldm_sequences`). Method bodies -//! (BT walk, `bt_insert_step_no_rebase`, `bt_update_tree_until`, +//! (BT walk, `bt_insert_range`, `bt_update_tree_until`, //! `build_optimal_plan*`, `collect_optimal_candidates*`, //! `emit_optimal_plan`, …) still live on `HcMatchGenerator` and will //! move onto `impl BtMatcher` once Stage 3b threads diff --git a/zstd/src/encoding/fastpath/mod.rs b/zstd/src/encoding/fastpath/mod.rs index 2d75a506e..b8bd7c915 100644 --- a/zstd/src/encoding/fastpath/mod.rs +++ b/zstd/src/encoding/fastpath/mod.rs @@ -51,8 +51,8 @@ //! //! Week 1 (this commit): module scaffold + dispatcher skeleton. //! Week 2a: match-length / common-prefix-len + `count_match_from_indices`. -//! Week 3a: BT walk (`bt_insert_step_no_rebase`, -//! `bt_insert_and_collect_matches`) + HC chain walk. +//! Week 3a: BT walk (`bt_insert_range`, the collect-matches body) + HC +//! chain walk. //! Week 3b: optimal parser DP (`build_optimal_plan_impl` + price helpers). //! Week 4: entropy encoders (FSE `encode_interleaved`, Huff0 `encode_stream`). //! Week 5-6: bench vs `perf/pre-intrinsics-refactor-baseline` tag, profile, diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index d88f10cc1..0abd1f45a 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -78,19 +78,25 @@ pub(crate) struct HcMatchGenerator { // the top of this file bring them back into scope so the existing // methods on `HcMatchGenerator` compile unchanged. -/// `bt_insert_step_no_rebase` body parameterized over the per-CPU -/// `count_match_from_indices` symbol. Each kernel-specific wrapper invokes -/// the macro with its own `fastpath::::count_match_from_indices` -/// path so the call resolves inside the wrapper's `#[target_feature]` -/// umbrella and inlines instead of paying the function-call ABI per BT walk -/// iteration. Used only by `HcMatchGenerator` BT walk wrappers below. +/// Binary-tree insertion of the positions `[$from, $stop)`, parameterized over +/// the per-CPU `count_match_from_indices` symbol so the compare inlines under +/// each kernel wrapper's `#[target_feature]` umbrella. Evaluates to the cursor +/// after the last insertion, which a long match can carry past `$stop`: the +/// positions it covers stay out of the tree, as upstream's `ZSTD_updateTree` +/// leaves them (`idx += ZSTD_insertBt1(...)`, no clamp to the target). +/// +/// One call inserts a whole run, the shape of upstream's +/// `ZSTD_updateTree_internal` with `ZSTD_insertBt1` inlined into its loop: +/// the tables, the mask, the window floor (a function of `$target_abs`, fixed +/// for the run) and the coordinate biases are resolved once per run, not once +/// per position. The parser's catch-up inserts a couple of positions per run, +/// and resolving all of that per position cost more than the tree walk. /// /// Crate-private: the macro body references private `encoding::*` /// modules via `$crate::...`, so it is unusable downstream and is /// re-exported only inside this crate via `pub(crate) use` below. -macro_rules! bt_insert_step_no_rebase_body { - ($table:expr, $search_depth:expr, $abs_pos:ident, $current_abs_end:ident, $target_abs:ident, $cmf:path) => {{ - let idx = $abs_pos - $table.history_abs_start; +macro_rules! bt_insert_range_body { + ($table:expr, $search_depth:expr, $from:ident, $stop:ident, $current_abs_end:ident, $target_abs:ident, $cmf:path) => {{ // Borrowed-aware live region (owned: `history[history_start..]`; // borrowed: the in-place input `[0, block_end)`). Reborrow-then-raw-ptr // so the slice holds NO borrow and coexists with the `&mut $table` @@ -99,20 +105,12 @@ macro_rules! bt_insert_step_no_rebase_body { let lh = $table.live_history(); core::slice::from_raw_parts(lh.as_ptr(), lh.len()) }; - if idx + 8 > concat.len() { - return 1; - } - debug_assert!( - $abs_pos <= $current_abs_end, - "BT walker called past current block end" - ); - let tail_limit = $current_abs_end - $abs_pos; - let hash = $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx, - $table.hash_log, - $table.search_mls, - ); + let hist_start = $table.history_abs_start; + let hash_log = $table.hash_log; + let search_mls = $table.search_mls; + let position_base = $table.position_base; + let index_shift = $table.index_shift; + let search_depth = $search_depth; // Upstream holds `U32* const hashTable = ms->hashTable` for the whole // body (zstd_opt.c:449). Ours re-derived it from the shared table // buffer at every use, and each re-derivation is a bounds-checked @@ -125,63 +123,13 @@ macro_rules! bt_insert_step_no_rebase_body { // walk). Both bases come out of ONE split borrow: taking the second // through its own `&mut` reslice reborrows the whole buffer, which // invalidates a pointer already taken from the first. - debug_assert_eq!($table.hash_table().len(), 1usize << $table.hash_log); + debug_assert_eq!($table.hash_table().len(), 1usize << hash_log); debug_assert_eq!($table.chain_table().len(), 2 << $table.bt_log()); - debug_assert!(hash < 1usize << $table.hash_log); let (hash_ptr, chain_ptr) = { let (hash_table, chain_table) = $table.hash_and_chain_mut(); (hash_table.as_mut_ptr(), chain_table.as_mut_ptr()) }; - // Prefetch the hash bucket now. For the large L16+ hash table over - // high-entropy input the bucket is L3/DRAM-cold, and unlike upstream's - // monolithic ZSTD_btGetAllMatches (which overlaps this miss with its - // inline rep/hash3 prologue) the read+write of `hash_table[hash]` - // below is reached with nothing to hide it behind — it stalled a large - // share of this function's cycles. Issuing the hint here lets the miss - // overlap the address setup that follows. - #[cfg(all( - target_feature = "sse", - any(target_arch = "x86", target_arch = "x86_64") - ))] - { - #[cfg(target_arch = "x86")] - use core::arch::x86::{_MM_HINT_T0, _mm_prefetch}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch}; - // SAFETY: prefetch is a hint that never faults; `hash` indexes - // `hash_table` directly below, so it is in bounds. - unsafe { - _mm_prefetch(hash_ptr.add(hash).cast(), _MM_HINT_T0); - } - // Prefetch the NEXT position's bucket too. The optimal-parser DP - // advances one position per iteration, so this miss is issued a - // full BT walk plus the next iteration's pre-collect work ahead of - // the collect that will read it — far more lead than the same-call - // hint above, enough to hide the full DRAM latency. - if idx + 1 + 8 <= concat.len() { - let hash_next = - $crate::encoding::match_table::storage::MatchTable::hash_position_at( - concat, - idx + 1, - $table.hash_log, - $table.search_mls, - ); - // SAFETY: prefetch never faults; an out-of-range index is a - // harmless no-op hint. - unsafe { - _mm_prefetch(hash_ptr.add(hash_next).cast(), _MM_HINT_T0); - } - } - } - // Total, not tested: the block was armed before the parse began. - let stored = $table.relative_position_armed($abs_pos) + 1; let bt_mask = $table.bt_mask(); - // `abs_pos < bt_mask` legitimately happens for the first BT walk of - // a fresh frame (bt_low effectively "no floor"). Saturating keeps - // the floor at 0 so the `candidate_abs <= bt_low` check never - // triggers early; raw subtraction would underflow into a huge - // sentinel that ALWAYS triggers. - let bt_low = $abs_pos.saturating_sub(bt_mask); let window_low = $table.window_low_abs_for_target($target_abs); // The walk carries one coordinate, the stored index, as the collect body // does and as upstream carries `matchIndex`: the window bound becomes one @@ -190,152 +138,229 @@ macro_rules! bt_insert_step_no_rebase_body { // pair slot are each one add of a bias taken here, instead of reloading // `position_base` / `index_shift` / `history_abs_start` through the // table on every node. See the collect body for the derivation. - let abs_bias = $table - .position_base - .wrapping_sub(1) - .wrapping_sub($table.index_shift); + let abs_bias = position_base.wrapping_sub(1).wrapping_sub(index_shift); let win_off = abs_bias.wrapping_sub(window_low); - // The window floor follows the tree update's target, which may lie past - // this position by more than the window; nothing is in range then, as - // upstream's `matchIndex >= windowLow` finds on its first test. - let win_range = if window_low < $abs_pos { - $abs_pos - window_low - } else { - 0 - }; - let idx_bias = abs_bias.wrapping_sub($table.history_abs_start); - let bt_bias = $table.position_base.wrapping_sub(1); - // `abs_pos + 9` is safe in raw form: `MatchTable::add_data` caps - // total input at `usize::MAX - STREAM_ABS_HEADROOM` (where - // `STREAM_ABS_HEADROOM = HC_OPT_NUM + 16`), so every - // frame-lifetime absolute cursor passed to the BT walker stays - // below `usize::MAX - 9` regardless of stream length or - // pointer width. The guard is hoisted to the data-ingest - // boundary so this per-position site pays zero arithmetic - // overhead in the hot loop. - let mut match_end_abs = $abs_pos + 9; - let mut best_len = 8usize; - let mut compares_left = $search_depth; - let mut common_length_smaller = 0usize; - let mut common_length_larger = 0usize; - let pair_idx = $table.bt_pair_index_for_abs($abs_pos); - let mut smaller_slot = pair_idx; - let mut larger_slot = pair_idx + 1; - // SAFETY: `hash` is masked to `hash_log` bits and the table is - // `1 << hash_log` slots wide (both asserted at `hash_ptr`), so the slot - // is in range by construction. Upstream reads and writes the same slot - // through its own raw `hashTable`. - let mut match_stored = unsafe { *hash_ptr.add(hash) }; - unsafe { *hash_ptr.add(hash) = stored }; + let idx_bias = abs_bias.wrapping_sub(hist_start); + let bt_bias = position_base.wrapping_sub(1); + let mut pos = $from; + while pos < $stop { + let forward: usize = 'insert: { + let abs_pos = pos; + let idx = abs_pos - hist_start; + if idx + 8 > concat.len() { + break 'insert 1; + } + debug_assert!( + abs_pos <= $current_abs_end, + "BT walker called past current block end" + ); + let tail_limit = $current_abs_end - abs_pos; + let hash = $crate::encoding::match_table::storage::MatchTable::hash_position_at( + concat, idx, hash_log, search_mls, + ); + debug_assert!(hash < 1usize << hash_log); + // Prefetch the hash bucket now. For the large L16+ hash table over + // high-entropy input the bucket is L3/DRAM-cold, and unlike upstream's + // monolithic ZSTD_btGetAllMatches (which overlaps this miss with its + // inline rep/hash3 prologue) the read+write of `hash_table[hash]` + // below is reached with nothing to hide it behind — it stalled a large + // share of this function's cycles. Issuing the hint here lets the miss + // overlap the address setup that follows. + #[cfg(all( + target_feature = "sse", + any(target_arch = "x86", target_arch = "x86_64") + ))] + { + #[cfg(target_arch = "x86")] + use core::arch::x86::{_MM_HINT_T0, _mm_prefetch}; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{_MM_HINT_T0, _mm_prefetch}; + // SAFETY: prefetch is a hint that never faults; `hash` indexes + // `hash_table` directly below, so it is in bounds. + unsafe { + _mm_prefetch(hash_ptr.add(hash).cast(), _MM_HINT_T0); + } + // Prefetch the NEXT position's bucket too. The optimal-parser DP + // advances one position per iteration, so this miss is issued a + // full BT walk plus the next iteration's pre-collect work ahead of + // the collect that will read it — far more lead than the same-call + // hint above, enough to hide the full DRAM latency. + if idx + 1 + 8 <= concat.len() { + let hash_next = + $crate::encoding::match_table::storage::MatchTable::hash_position_at( + concat, + idx + 1, + hash_log, + search_mls, + ); + // SAFETY: prefetch never faults; an out-of-range index is a + // harmless no-op hint. + unsafe { + _mm_prefetch(hash_ptr.add(hash_next).cast(), _MM_HINT_T0); + } + } + } + // Total, not tested: the block was armed before the parse began + // (`MatchTable::relative_position_armed`, with its fields hoisted). + debug_assert!($table.can_skip_rebase_check(abs_pos)); + let stored = ((abs_pos - position_base + index_shift) as u32) + 1; + // `abs_pos < bt_mask` legitimately happens for the first BT walk of + // a fresh frame (bt_low effectively "no floor"). Saturating keeps + // the floor at 0 so the `candidate_abs <= bt_low` check never + // triggers early; raw subtraction would underflow into a huge + // sentinel that ALWAYS triggers. + let bt_low = abs_pos.saturating_sub(bt_mask); + // The window floor follows the tree update's target, which may lie past + // this position by more than the window; nothing is in range then, as + // upstream's `matchIndex >= windowLow` finds on its first test. + let win_range = if window_low < abs_pos { + abs_pos - window_low + } else { + 0 + }; + // `abs_pos + 9` is safe in raw form: `MatchTable::add_data` caps + // total input at `usize::MAX - STREAM_ABS_HEADROOM` (where + // `STREAM_ABS_HEADROOM = HC_OPT_NUM + 16`), so every + // frame-lifetime absolute cursor passed to the BT walker stays + // below `usize::MAX - 9` regardless of stream length or + // pointer width. The guard is hoisted to the data-ingest + // boundary so this per-position site pays zero arithmetic + // overhead in the hot loop. + let mut match_end_abs = abs_pos + 9; + let mut best_len = 8usize; + let mut compares_left = search_depth; + let mut common_length_smaller = 0usize; + let mut common_length_larger = 0usize; + // `MatchTable::bt_pair_index_for_abs` with the shift and mask hoisted. + let pair_idx = 2 * (abs_pos.wrapping_add(index_shift) & bt_mask); + let mut smaller_slot = pair_idx; + let mut larger_slot = pair_idx + 1; + // SAFETY: `hash` is masked to `hash_log` bits and the table is + // `1 << hash_log` slots wide (both asserted at `hash_ptr`), so the slot + // is in range by construction. Upstream reads and writes the same slot + // through its own raw `hashTable`. + let mut match_stored = unsafe { *hash_ptr.add(hash) }; + unsafe { *hash_ptr.add(hash) = stored }; - while compares_left > 0 && (match_stored as usize).wrapping_add(win_off) < win_range { - compares_left -= 1; - let stored = match_stored as usize; - // A slot written under an earlier encoding would sit below the - // shift; the block was armed, and a rebase rewrites every slot, so - // none reaches an in-window test. - debug_assert!(stored > $table.index_shift); - let candidate_abs = stored.wrapping_add(abs_bias); - debug_assert!(candidate_abs >= window_low && candidate_abs < $abs_pos); - // `2*((candidate_abs + index_shift) & bt_mask)`, with the shift - // folded: `candidate_abs + index_shift == stored + bt_bias`. - let next_pair_idx = 2 * (stored.wrapping_add(bt_bias) & bt_mask); - // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)` - // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base, - // table not realloc'd during the walk. - let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) }; - let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) }; - let seed_len = common_length_smaller.min(common_length_larger); - let candidate_idx = stored.wrapping_add(idx_bias); - // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤ - // concat.len()` since the candidate is within - // `[history_abs_start, abs_pos)` and `tail_limit ≤ - // current_abs_end - abs_pos`. - let match_len = unsafe { $cmf(concat, idx, candidate_idx, tail_limit, seed_len) }; + while compares_left > 0 && (match_stored as usize).wrapping_add(win_off) < win_range + { + compares_left -= 1; + let stored = match_stored as usize; + // A slot written under an earlier encoding would sit below the + // shift; the block was armed, and a rebase rewrites every slot, so + // none reaches an in-window test. + debug_assert!(stored > index_shift); + let candidate_abs = stored.wrapping_add(abs_bias); + debug_assert!(candidate_abs >= window_low && candidate_abs < abs_pos); + // `2*((candidate_abs + index_shift) & bt_mask)`, with the shift + // folded: `candidate_abs + index_shift == stored + bt_bias`. + let next_pair_idx = 2 * (stored.wrapping_add(bt_bias) & bt_mask); + // SAFETY: `next_pair_idx (+1)` = `2*(candidate_abs & bt_mask) (+1)` + // ≤ `chain_table.len()-1`; `chain_ptr` is the hoisted live base, + // table not realloc'd during the walk. + let next_smaller = unsafe { *chain_ptr.add(next_pair_idx) }; + let next_larger = unsafe { *chain_ptr.add(next_pair_idx + 1) }; + let seed_len = common_length_smaller.min(common_length_larger); + let candidate_idx = stored.wrapping_add(idx_bias); + // SAFETY: BT walk invariant — `candidate_idx + tail_limit ≤ + // concat.len()` since the candidate is within + // `[history_abs_start, abs_pos)` and `tail_limit ≤ + // current_abs_end - abs_pos`. + let match_len = + unsafe { $cmf(concat, idx, candidate_idx, tail_limit, seed_len) }; - if match_len > best_len { - best_len = match_len; - // `candidate_abs + match_len <= current_abs_end` by BT walk - // invariant — `match_len <= tail_limit = current_abs_end - - // abs_pos` and `candidate_abs < abs_pos`. - let candidate_end = candidate_abs + match_len; - if candidate_end > match_end_abs { - match_end_abs = candidate_end; - } - } + if match_len > best_len { + best_len = match_len; + // `candidate_abs + match_len <= current_abs_end` by BT walk + // invariant — `match_len <= tail_limit = current_abs_end - + // abs_pos` and `candidate_abs < abs_pos`. + let candidate_end = candidate_abs + match_len; + if candidate_end > match_end_abs { + match_end_abs = candidate_end; + } + } - if match_len >= tail_limit { - break; - } + if match_len >= tail_limit { + break; + } - let candidate_next = candidate_idx + match_len; - let current_next = idx + match_len; - // SAFETY: first-differing positions after a match_len-long prefix; - // match_len < tail_limit (break above) + BT-walk bound - // idx/candidate_idx + tail_limit <= concat.len() keep both in range. - if unsafe { - *concat.get_unchecked(candidate_next) < *concat.get_unchecked(current_next) - } { - // SAFETY: `smaller_slot` holds a valid pair index (init - // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX` - // sentinel is set only just before `break`, never written here. - unsafe { *chain_ptr.add(smaller_slot) = match_stored }; - common_length_smaller = match_len; - if candidate_abs <= bt_low { - smaller_slot = usize::MAX; - break; + let candidate_next = candidate_idx + match_len; + let current_next = idx + match_len; + // SAFETY: first-differing positions after a match_len-long prefix; + // match_len < tail_limit (break above) + BT-walk bound + // idx/candidate_idx + tail_limit <= concat.len() keep both in range. + if unsafe { + *concat.get_unchecked(candidate_next) < *concat.get_unchecked(current_next) + } { + // SAFETY: `smaller_slot` holds a valid pair index (init + // `pair_idx`, updated to `next_pair_idx + 1`); the `usize::MAX` + // sentinel is set only just before `break`, never written here. + unsafe { *chain_ptr.add(smaller_slot) = match_stored }; + common_length_smaller = match_len; + if candidate_abs <= bt_low { + smaller_slot = usize::MAX; + break; + } + smaller_slot = next_pair_idx + 1; + match_stored = next_larger; + } else { + // SAFETY: as above for `larger_slot`. + unsafe { *chain_ptr.add(larger_slot) = match_stored }; + common_length_larger = match_len; + if candidate_abs <= bt_low { + larger_slot = usize::MAX; + break; + } + larger_slot = next_pair_idx; + match_stored = next_smaller; + } } - smaller_slot = next_pair_idx + 1; - match_stored = next_larger; - } else { - // SAFETY: as above for `larger_slot`. - unsafe { *chain_ptr.add(larger_slot) = match_stored }; - common_length_larger = match_len; - if candidate_abs <= bt_low { - larger_slot = usize::MAX; - break; + + // SAFETY: both slots, when not the `usize::MAX` sentinel, hold valid + // pair indices into the hoisted `chain_table` base. + if smaller_slot != usize::MAX { + unsafe { + *chain_ptr.add(smaller_slot) = + $crate::encoding::match_table::storage::HC_EMPTY + }; + } + if larger_slot != usize::MAX { + unsafe { + *chain_ptr.add(larger_slot) = + $crate::encoding::match_table::storage::HC_EMPTY + }; } - larger_slot = next_pair_idx; - match_stored = next_smaller; - } - } - // SAFETY: both slots, when not the `usize::MAX` sentinel, hold valid - // pair indices into the hoisted `chain_table` base. - if smaller_slot != usize::MAX { - unsafe { - *chain_ptr.add(smaller_slot) = $crate::encoding::match_table::storage::HC_EMPTY - }; - } - if larger_slot != usize::MAX { - unsafe { - *chain_ptr.add(larger_slot) = $crate::encoding::match_table::storage::HC_EMPTY + let speed_positions = if best_len > 384 { + (best_len - 384).min(192) + } else { + 0 + }; + // `match_end_abs` is initialized to `abs_pos + 9` and is only + // reassigned inside the `candidate_end > match_end_abs` branch + // above. So even though an individual `candidate_end = + // candidate_abs + match_len` can land below `abs_pos` (the + // candidate sits earlier in history and the match runs short), + // the variable itself never drops below its initial value. + // That gives `match_end_abs ≥ abs_pos + 9 > abs_pos + 8` as a + // loop-wide invariant, so the raw subtraction below cannot + // underflow. + speed_positions.max(match_end_abs - (abs_pos + 8)) }; + // Both terms of `forward` end inside the block (`match_end_abs <= + // current_abs_end`, and a speed skip needs a match over 384 bytes), so + // the cursor cannot overflow. + pos += forward.max(1); } - - let speed_positions = if best_len > 384 { - (best_len - 384).min(192) - } else { - 0 - }; - // `match_end_abs` is initialized to `abs_pos + 9` and is only - // reassigned inside the `candidate_end > match_end_abs` branch - // above. So even though an individual `candidate_end = - // candidate_abs + match_len` can land below `abs_pos` (the - // candidate sits earlier in history and the match runs short), - // the variable itself never drops below its initial value. - // That gives `match_end_abs ≥ abs_pos + 9 > abs_pos + 8` as a - // loop-wide invariant, so the raw subtraction below cannot - // underflow. - speed_positions.max(match_end_abs - ($abs_pos + 8)) + pos }}; } -pub(crate) use bt_insert_step_no_rebase_body; +pub(crate) use bt_insert_range_body; /// `hash3_candidate` body parameterized over the per-CPU /// `common_prefix_len_ptr` symbol. The hash3 probe checks one candidate per /// position when invoked, so the per-call ABI savings compound across the -/// segment. Crate-private (see `bt_insert_step_no_rebase_body!`). +/// segment. Crate-private (see `bt_insert_range_body!`). macro_rules! hash3_candidate_body { ( $table:expr, @@ -399,7 +424,7 @@ pub(crate) use hash3_candidate_body; /// /// The callback `f` runs in the wrapper's umbrella context too, so closures /// that capture mutable state still work (FnMut). Crate-private -/// (see `bt_insert_step_no_rebase_body!`). +/// (see `bt_insert_range_body!`). macro_rules! for_each_repcode_candidate_body { ( $table:expr, @@ -487,13 +512,8 @@ macro_rules! for_each_repcode_candidate_body { } pub(crate) use for_each_repcode_candidate_body; -/// `bt_insert_and_collect_matches` body parameterized over the per-CPU -/// `count_match_from_indices` symbol. Same shape as -/// [`bt_insert_step_no_rebase_body`] — picks up the matching kernel through -/// `$cmf` so the per-iteration vector probe inlines under the wrapper's -/// `target_feature` umbrella. Returns nothing (matches the original method). -/// Crate-private (see `bt_insert_step_no_rebase_body!`). -/// One repeat-offset probe, expanded per slot. +/// One repeat-offset probe, expanded per slot. Crate-private (see +/// `bt_insert_range_body!`). /// /// The repeat loop runs three or four times with the slot known at each /// expansion, so it is unrolled rather than counted: the counter, its bound and @@ -886,7 +906,7 @@ macro_rules! bt_insert_and_collect_matches_body { // Total, not tested: the block was armed before the parse began. let stored = $table.relative_position_armed($abs_pos) + 1; let bt_mask = $table.bt_mask(); - // See `bt_insert_step_no_rebase_body!`: saturating is needed for the + // See `bt_insert_range_body!`: saturating is needed for the // first BT walk of a fresh frame where `abs_pos < bt_mask`. let bt_low = $abs_pos.saturating_sub(bt_mask); let window_low = $table.window_low_abs_for_target($abs_pos); @@ -924,7 +944,7 @@ macro_rules! bt_insert_and_collect_matches_body { .wrapping_sub($table.index_shift); let idx_bias = abs_bias.wrapping_sub($table.history_abs_start); let bt_bias = $table.position_base.wrapping_sub(1); - // Raw `+ 9` is safe here — see `bt_insert_step_no_rebase_body!` + // Raw `+ 9` is safe here — see `bt_insert_range_body!` // for the full discussion of the upstream `STREAM_ABS_HEADROOM` // cap in `MatchTable::add_data`. let mut match_end_abs = $abs_pos + 9; diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index aeab54212..3842ae341 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -1152,7 +1152,7 @@ macro_rules! collect_optimal_candidates_initialized_body { $reps:ident, $ll0:ident, $out:ident, - $bt_insert_step:ident, + $bt_insert_range:ident, $cpl:path, $cmf:path $(,)? ) => {{ @@ -1178,28 +1178,27 @@ macro_rules! collect_optimal_candidates_initialized_body { return; } { - // BT tree catch-up folded inline (was a per-position call to - // bt_update_tree_until): insert the positions the parser skipped into - // the binary tree before this position's search. Upstream zstd - // ZSTD_updateTree shape; `$bt_insert_step` overshoots the target with - // no clamp (forward skips match-covered positions), exactly as C. - // SAFETY: caller is in the same target_feature umbrella as - // `$bt_insert_step`; the runtime kernel detector already gated entry. + // BT tree catch-up: insert the positions the parser skipped into the + // binary tree before this position's search, as one run (upstream + // zstd ZSTD_updateTree). `$bt_insert_range` overshoots the target + // with no clamp (a long match skips the positions it covers), + // exactly as upstream. if $self.table.skip_insert_until_abs < $self.table.history_abs_start { $self.table.skip_insert_until_abs = $self.table.history_abs_start; } - let mut update_abs = $self.table.skip_insert_until_abs; + let update_abs = $self.table.skip_insert_until_abs; // No rebase guard: `arm_block_positions` cleared every position in // this block before the parse began, and the catch-up only inserts // positions below `$abs_pos`, which is in it. debug_assert!($self.table.can_skip_rebase_check($abs_pos)); - while update_abs < $abs_pos { - let forward = unsafe { + if update_abs < $abs_pos { + // SAFETY: caller is in the same target_feature umbrella as + // `$bt_insert_range`; the runtime kernel detector gated entry. + let _ = unsafe { $self .table - .$bt_insert_step(update_abs, $current_abs_end, $abs_pos) + .$bt_insert_range(update_abs, $abs_pos, $current_abs_end, $abs_pos) }; - update_abs += forward.max(1); } $self.table.skip_insert_until_abs = $abs_pos; } @@ -2253,11 +2252,10 @@ impl HcMatchGenerator { } } - /// NEON-umbrella variant. Every inner helper (`bt_update_tree_until_neon`, - /// `for_each_repcode_candidate_with_reps_neon`, `hash3_candidate_neon`, - /// `bt_insert_and_collect_matches_neon`, `fastpath::neon:: - /// common_prefix_len_ptr`) shares the NEON umbrella so the per-position - /// pipeline executes as a single straight-line inline sequence. + /// NEON-umbrella variant: the repeat and hash3 probes, the tree walk and + /// `fastpath::neon::common_prefix_len_ptr` share the NEON umbrella so the + /// per-position search runs as one straight-line sequence, and the tree + /// catch-up (`bt_insert_range_neon`) runs under the same tier. #[cfg(all( target_arch = "aarch64", target_endian = "little", @@ -2285,7 +2283,7 @@ impl HcMatchGenerator { reps, ll0, out, - bt_insert_step_no_rebase_neon, + bt_insert_range_neon, crate::encoding::fastpath::neon::common_prefix_len_ptr, crate::encoding::fastpath::neon::count_match_from_indices, ) @@ -2317,7 +2315,7 @@ impl HcMatchGenerator { reps, ll0, out, - bt_insert_step_no_rebase_sse2, + bt_insert_range_sse2, crate::encoding::fastpath::sse2::common_prefix_len_ptr, crate::encoding::fastpath::sse2::count_match_from_indices, ) @@ -2355,7 +2353,7 @@ impl HcMatchGenerator { reps, ll0, out, - bt_insert_step_no_rebase_sse2, + bt_insert_range_sse2, crate::encoding::fastpath::sse2::common_prefix_len_ptr, crate::encoding::fastpath::sse2::count_match_from_indices, ) @@ -2387,7 +2385,7 @@ impl HcMatchGenerator { reps, ll0, out, - bt_insert_step_no_rebase_avx2_bmi2, + bt_insert_range_avx2_bmi2, crate::encoding::fastpath::avx2_bmi2::common_prefix_len_ptr, crate::encoding::fastpath::avx2_bmi2::count_match_from_indices, ) @@ -2426,7 +2424,7 @@ impl HcMatchGenerator { reps, ll0, out, - bt_insert_step_no_rebase_simd128, + bt_insert_range_simd128, crate::encoding::fastpath::simd128::common_prefix_len_ptr, crate::encoding::fastpath::simd128::count_match_from_indices, ) @@ -2461,7 +2459,7 @@ impl HcMatchGenerator { reps, ll0, out, - bt_insert_step_no_rebase_scalar, + bt_insert_range_scalar, crate::encoding::fastpath::scalar::common_prefix_len_ptr, crate::encoding::fastpath::scalar::count_match_from_indices, ) diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index f9965d3e9..c2c060aa2 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -1679,17 +1679,17 @@ impl MatchTable { (start_cursor, start_cursor) } - /// Stage D: BT walker step. Cross-platform dispatcher that picks - /// the per-kernel variant so the per-iteration - /// `count_match_from_indices` symbol inlines under the kernel's - /// `target_feature` umbrella. Previously lived on `BtMatcher` - /// but the body uses only table state plus `self.search_depth`, - /// so it migrates onto `MatchTable` and clears the cross-struct - /// borrow that blocked the rest of the BT update chain. + /// Insert the positions `[from, stop)` into the binary tree, with the + /// window floor taken at `target_abs`, and return the cursor after the + /// last insertion (a long match can carry it past `stop`). Cross-platform + /// dispatcher: the per-kernel variant runs the whole range under its + /// `target_feature` umbrella. Every position in the range must already be + /// representable (see [`Self::arm_block_positions`]). #[inline(always)] - pub(crate) fn bt_insert_step_no_rebase( + pub(crate) fn bt_insert_range( &mut self, - abs_pos: usize, + from: usize, + stop: usize, current_abs_end: usize, target_abs: usize, ) -> usize { @@ -1699,7 +1699,7 @@ impl MatchTable { feature = "kernel-neon" ))] unsafe { - self.bt_insert_step_no_rebase_neon(abs_pos, current_abs_end, target_abs) + self.bt_insert_range_neon(from, stop, current_abs_end, target_abs) } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { @@ -1707,14 +1707,14 @@ impl MatchTable { match self.kernel { #[cfg(feature = "kernel-avx2")] FastpathKernel::Avx2Bmi2 => unsafe { - self.bt_insert_step_no_rebase_avx2_bmi2(abs_pos, current_abs_end, target_abs) + self.bt_insert_range_avx2_bmi2(from, stop, current_abs_end, target_abs) }, #[cfg(feature = "kernel-sse")] FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - self.bt_insert_step_no_rebase_sse2(abs_pos, current_abs_end, target_abs) + self.bt_insert_range_sse2(from, stop, current_abs_end, target_abs) }, FastpathKernel::Scalar => { - self.bt_insert_step_no_rebase_scalar(abs_pos, current_abs_end, target_abs) + self.bt_insert_range_scalar(from, stop, current_abs_end, target_abs) } } } @@ -1728,7 +1728,7 @@ impl MatchTable { // SAFETY: the `cfg` above establishes `simd128` at compile time, which // is exactly the umbrella the callee declares. unsafe { - self.bt_insert_step_no_rebase_simd128(abs_pos, current_abs_end, target_abs) + self.bt_insert_range_simd128(from, stop, current_abs_end, target_abs) } #[cfg(not(any( all( @@ -1745,11 +1745,11 @@ impl MatchTable { ) )))] { - self.bt_insert_step_no_rebase_scalar(abs_pos, current_abs_end, target_abs) + self.bt_insert_range_scalar(from, stop, current_abs_end, target_abs) } } - /// NEON umbrella BT walker step. + /// NEON umbrella binary-tree range insertion. /// /// # Safety /// AArch64 with NEON (baseline). @@ -1759,24 +1759,26 @@ impl MatchTable { feature = "kernel-neon" ))] #[target_feature(enable = "neon")] - pub(crate) unsafe fn bt_insert_step_no_rebase_neon( + pub(crate) unsafe fn bt_insert_range_neon( &mut self, - abs_pos: usize, + from: usize, + stop: usize, current_abs_end: usize, target_abs: usize, ) -> usize { let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_step_no_rebase_body!( + super::super::hc::generator::bt_insert_range_body!( self, search_depth, - abs_pos, + from, + stop, current_abs_end, target_abs, crate::encoding::fastpath::neon::count_match_from_indices ) } - /// SSE2 umbrella BT walker step. + /// SSE2 umbrella binary-tree range insertion. /// /// # Safety /// x86/x86_64 with SSE2. @@ -1785,24 +1787,26 @@ impl MatchTable { feature = "kernel-sse" ))] #[target_feature(enable = "sse2")] - pub(crate) unsafe fn bt_insert_step_no_rebase_sse2( + pub(crate) unsafe fn bt_insert_range_sse2( &mut self, - abs_pos: usize, + from: usize, + stop: usize, current_abs_end: usize, target_abs: usize, ) -> usize { let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_step_no_rebase_body!( + super::super::hc::generator::bt_insert_range_body!( self, search_depth, - abs_pos, + from, + stop, current_abs_end, target_abs, crate::encoding::fastpath::sse2::count_match_from_indices ) } - /// AVX2+BMI2 umbrella BT walker step. + /// AVX2+BMI2 umbrella binary-tree range insertion. /// /// # Safety /// x86/x86_64 with AVX2 + BMI2. @@ -1811,24 +1815,26 @@ impl MatchTable { feature = "kernel-avx2" ))] #[target_feature(enable = "avx2,bmi2")] - pub(crate) unsafe fn bt_insert_step_no_rebase_avx2_bmi2( + pub(crate) unsafe fn bt_insert_range_avx2_bmi2( &mut self, - abs_pos: usize, + from: usize, + stop: usize, current_abs_end: usize, target_abs: usize, ) -> usize { let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_step_no_rebase_body!( + super::super::hc::generator::bt_insert_range_body!( self, search_depth, - abs_pos, + from, + stop, current_abs_end, target_abs, crate::encoding::fastpath::avx2_bmi2::count_match_from_indices ) } - /// WebAssembly `simd128` umbrella BT walker step. + /// WebAssembly `simd128` umbrella binary-tree range insertion. /// /// # Safety /// wasm32 with `simd128` enabled at compile time. @@ -1838,42 +1844,46 @@ impl MatchTable { feature = "kernel-simd128" ))] #[target_feature(enable = "simd128")] - pub(crate) unsafe fn bt_insert_step_no_rebase_simd128( + pub(crate) unsafe fn bt_insert_range_simd128( &mut self, - abs_pos: usize, + from: usize, + stop: usize, current_abs_end: usize, target_abs: usize, ) -> usize { let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_step_no_rebase_body!( + super::super::hc::generator::bt_insert_range_body!( self, search_depth, - abs_pos, + from, + stop, current_abs_end, target_abs, crate::encoding::fastpath::simd128::count_match_from_indices ) } - /// Scalar fallback BT walker step. Compiled unless the NEON tier covers - /// this target, i.e. on every non-AArch64 target and on AArch64 when - /// `kernel-neon` is off. + /// Scalar fallback binary-tree range insertion. Compiled unless the NEON + /// tier covers this target, i.e. on every non-AArch64 target and on + /// AArch64 when `kernel-neon` is off. #[cfg(not(all( target_arch = "aarch64", target_endian = "little", feature = "kernel-neon" )))] - pub(crate) fn bt_insert_step_no_rebase_scalar( + pub(crate) fn bt_insert_range_scalar( &mut self, - abs_pos: usize, + from: usize, + stop: usize, current_abs_end: usize, target_abs: usize, ) -> usize { let search_depth = self.search_depth; - super::super::hc::generator::bt_insert_step_no_rebase_body!( + super::super::hc::generator::bt_insert_range_body!( self, search_depth, - abs_pos, + from, + stop, current_abs_end, target_abs, crate::encoding::fastpath::scalar::count_match_from_indices @@ -1881,259 +1891,42 @@ impl MatchTable { } /// BT-side history replay after [`Self::begin_rebase`]. Re-walks - /// `history_start..abs_pos` through the BT step so the pointer-pair + /// `history_start..abs_pos` through the tree insertion so the pointer-pair /// table is consistent with the freshly reset `position_base`. pub(crate) fn replay_history_for_rebase_bt(&mut self, history_start: usize, abs_pos: usize) { let rebuild_end = self.history_abs_end(); - let mut pos = history_start; - while pos < abs_pos { - let forward = self.bt_insert_step_no_rebase(pos, rebuild_end, abs_pos); - // `pos` is a frame-lifetime absolute cursor that can approach - // `usize::MAX` on long 32-bit streams. Cap the step at the - // remaining distance to `abs_pos` so the addition stays - // within `usize` even when the BT walker returns a large - // `forward` near the stream end. - let step = forward.max(1).min(abs_pos - pos); - pos += step; - } - } - - /// Stage D: BT-tree update dispatcher. Picks the kernel-specific - /// variant so the per-iteration BT walker inlines under the - /// surrounding `target_feature` umbrella. - #[inline(always)] - pub(crate) fn bt_update_tree_until(&mut self, abs_pos: usize, current_abs_end: usize) { - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - unsafe { - self.bt_update_tree_until_neon(abs_pos, current_abs_end) - } - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - use crate::encoding::fastpath::FastpathKernel; - match self.kernel { - #[cfg(feature = "kernel-avx2")] - FastpathKernel::Avx2Bmi2 => unsafe { - self.bt_update_tree_until_avx2_bmi2(abs_pos, current_abs_end) - }, - #[cfg(feature = "kernel-sse")] - FastpathKernel::Sse2 | FastpathKernel::Sse42 => unsafe { - self.bt_update_tree_until_sse2(abs_pos, current_abs_end) - }, - FastpathKernel::Scalar => { - self.bt_update_tree_until_scalar(abs_pos, current_abs_end) - } - } - } - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - unsafe { - self.bt_update_tree_until_simd128(abs_pos, current_abs_end) - } - #[cfg(not(any( - all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ), - all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ), - target_arch = "x86", - target_arch = "x86_64" - )))] - { - self.bt_update_tree_until_scalar(abs_pos, current_abs_end) - } - } - - /// WebAssembly `simd128` umbrella variant: the per-iteration - /// `bt_insert_step_no_rebase_simd128` inlines into the body because both - /// share the `target_feature = "simd128"` umbrella, so the tree walk runs - /// the same tier as the insert step it drives. - /// - /// # Safety - /// wasm32 with `simd128` enabled at compile time. - #[cfg(all( - target_arch = "wasm32", - target_feature = "simd128", - feature = "kernel-simd128" - ))] - #[target_feature(enable = "simd128")] - pub(crate) unsafe fn bt_update_tree_until_simd128( - &mut self, - abs_pos: usize, - current_abs_end: usize, - ) { - if self.skip_insert_until_abs < self.history_abs_start { - self.skip_insert_until_abs = self.history_abs_start; - } - let mut update_abs = self.skip_insert_until_abs; - while update_abs < abs_pos { - if !self.can_skip_rebase_check(abs_pos) { - self.maybe_rebase_positions(update_abs); - } - let forward = unsafe { - self.bt_insert_step_no_rebase_simd128(update_abs, current_abs_end, abs_pos) - }; - // Upstream zstd `ZSTD_updateTree`: no clamp to the target, so a long - // match's covered positions stay out of the tree exactly as C leaves - // them. - update_abs += forward.max(1); - } - self.skip_insert_until_abs = abs_pos; - } - - /// NEON-umbrella variant: per-iteration `bt_insert_step_no_rebase_neon` - /// inlines into the body because both share the - /// `target_feature = "neon"` umbrella. - /// - /// # Safety - /// AArch64 with NEON (baseline). - #[cfg(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - ))] - #[target_feature(enable = "neon")] - pub(crate) unsafe fn bt_update_tree_until_neon( - &mut self, - abs_pos: usize, - current_abs_end: usize, - ) { - if self.skip_insert_until_abs < self.history_abs_start { - self.skip_insert_until_abs = self.history_abs_start; - } - let mut update_abs = self.skip_insert_until_abs; - while update_abs < abs_pos { - if !self.can_skip_rebase_check(abs_pos) { - self.maybe_rebase_positions(update_abs); - } - // SAFETY: same NEON umbrella; direct call inlines the BT-walk body. - let forward = - unsafe { self.bt_insert_step_no_rebase_neon(update_abs, current_abs_end, abs_pos) }; - // Upstream zstd `ZSTD_updateTree`: `idx += ZSTD_insertBt1(...)` with - // NO clamp to the target. The insert step's `forward` skips the - // positions a long match already covers, so letting it overshoot - // `abs_pos` leaves those positions OUT of the tree exactly as C does - // (`nextToUpdate = target` afterwards). Clamping to `abs_pos` inserted - // those covered positions, giving our tree extra candidates C never - // surfaces (e.g. a longer-but-farther match the optimal parser then - // wrongly forces over a cheaper closer one). - update_abs += forward.max(1); - } - self.skip_insert_until_abs = abs_pos; + let _ = self.bt_insert_range(history_start, abs_pos, rebuild_end, abs_pos); } - /// SSE4.2 umbrella variant. + /// Insert every position from the insertion frontier up to `abs_pos` + /// into the binary tree (upstream zstd `ZSTD_updateTree`). /// - /// # Safety - /// x86/x86_64 with SSE2. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-sse" - ))] - #[target_feature(enable = "sse2")] - pub(crate) unsafe fn bt_update_tree_until_sse2( - &mut self, - abs_pos: usize, - current_abs_end: usize, - ) { - if self.skip_insert_until_abs < self.history_abs_start { - self.skip_insert_until_abs = self.history_abs_start; - } - let mut update_abs = self.skip_insert_until_abs; - while update_abs < abs_pos { - if !self.can_skip_rebase_check(abs_pos) { - self.maybe_rebase_positions(update_abs); - } - let forward = - unsafe { self.bt_insert_step_no_rebase_sse2(update_abs, current_abs_end, abs_pos) }; - // Upstream zstd `ZSTD_updateTree`: `idx += ZSTD_insertBt1(...)` with - // NO clamp to the target. The insert step's `forward` skips the - // positions a long match already covers, so letting it overshoot - // `abs_pos` leaves those positions OUT of the tree exactly as C does - // (`nextToUpdate = target` afterwards). Clamping to `abs_pos` inserted - // those covered positions, giving our tree extra candidates C never - // surfaces (e.g. a longer-but-farther match the optimal parser then - // wrongly forces over a cheaper closer one). - update_abs += forward.max(1); - } - self.skip_insert_until_abs = abs_pos; - } - - /// AVX2+BMI2 umbrella variant. - /// - /// # Safety - /// x86/x86_64 with AVX2 + BMI2. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - feature = "kernel-avx2" - ))] - #[target_feature(enable = "avx2,bmi2")] - pub(crate) unsafe fn bt_update_tree_until_avx2_bmi2( - &mut self, - abs_pos: usize, - current_abs_end: usize, - ) { - if self.skip_insert_until_abs < self.history_abs_start { - self.skip_insert_until_abs = self.history_abs_start; - } - let mut update_abs = self.skip_insert_until_abs; - while update_abs < abs_pos { - if !self.can_skip_rebase_check(abs_pos) { - self.maybe_rebase_positions(update_abs); - } - let forward = unsafe { - self.bt_insert_step_no_rebase_avx2_bmi2(update_abs, current_abs_end, abs_pos) - }; - // Upstream zstd `ZSTD_updateTree`: `idx += ZSTD_insertBt1(...)` with - // NO clamp to the target. The insert step's `forward` skips the - // positions a long match already covers, so letting it overshoot - // `abs_pos` leaves those positions OUT of the tree exactly as C does - // (`nextToUpdate = target` afterwards). Clamping to `abs_pos` inserted - // those covered positions, giving our tree extra candidates C never - // surfaces (e.g. a longer-but-farther match the optimal parser then - // wrongly forces over a cheaper closer one). - update_abs += forward.max(1); - } - self.skip_insert_until_abs = abs_pos; - } - - /// Scalar fallback, compiled unless the NEON tier covers this target. - #[cfg(not(all( - target_arch = "aarch64", - target_endian = "little", - feature = "kernel-neon" - )))] - pub(crate) fn bt_update_tree_until_scalar(&mut self, abs_pos: usize, current_abs_end: usize) { + /// A long match carries the cursor past `abs_pos` with no clamp, as + /// upstream's `idx += ZSTD_insertBt1(...)` does, and the frontier is then + /// set to `abs_pos` (`nextToUpdate = target`): the positions the match + /// covered stay out of the tree. Clamping to `abs_pos` inserted them and + /// gave the tree candidates upstream never surfaces (a longer but farther + /// match the optimal parser then forced over a cheaper closer one). + pub(crate) fn bt_update_tree_until(&mut self, abs_pos: usize, current_abs_end: usize) { if self.skip_insert_until_abs < self.history_abs_start { self.skip_insert_until_abs = self.history_abs_start; } let mut update_abs = self.skip_insert_until_abs; - while update_abs < abs_pos { - if !self.can_skip_rebase_check(abs_pos) { - self.maybe_rebase_positions(update_abs); + if update_abs < abs_pos { + if self.can_skip_rebase_check(abs_pos) { + // Every position up to the target is representable: one run. + let _ = self.bt_insert_range(update_abs, abs_pos, current_abs_end, abs_pos); + } else { + // A rebase may be due inside the run, and it moves the + // coordinates the range hoists, so go one position at a time. + while update_abs < abs_pos { + if !self.can_skip_rebase_check(abs_pos) { + self.maybe_rebase_positions(update_abs); + } + update_abs = + self.bt_insert_range(update_abs, update_abs + 1, current_abs_end, abs_pos); + } } - let forward = - self.bt_insert_step_no_rebase_scalar(update_abs, current_abs_end, abs_pos); - // Upstream zstd `ZSTD_updateTree`: `idx += ZSTD_insertBt1(...)` with - // NO clamp to the target. The insert step's `forward` skips the - // positions a long match already covers, so letting it overshoot - // `abs_pos` leaves those positions OUT of the tree exactly as C does - // (`nextToUpdate = target` afterwards). Clamping to `abs_pos` inserted - // those covered positions, giving our tree extra candidates C never - // surfaces (e.g. a longer-but-farther match the optimal parser then - // wrongly forces over a cheaper closer one). - update_abs += forward.max(1); } self.skip_insert_until_abs = abs_pos; } @@ -2514,7 +2307,7 @@ impl MatchTable { let mut pos = current_abs_start; while pos < current_abs_end { self.maybe_rebase_positions(pos); - let _ = self.bt_insert_step_no_rebase(pos, current_abs_end, current_abs_end); + let _ = self.bt_insert_range(pos, pos + 1, current_abs_end, current_abs_end); self.insert_hash3_only_no_rebase(pos); let next = pos.saturating_add(INCOMPRESSIBLE_SKIP_STEP); if next <= pos { @@ -2533,7 +2326,7 @@ impl MatchTable { continue; } self.maybe_rebase_positions(pos); - let _ = self.bt_insert_step_no_rebase(pos, current_abs_end, current_abs_end); + let _ = self.bt_insert_range(pos, pos + 1, current_abs_end, current_abs_end); self.insert_hash3_only_no_rebase(pos); } diff --git a/zstd/src/encoding/match_table/storage/storage_tests.rs b/zstd/src/encoding/match_table/storage/storage_tests.rs index c795c9dca..b5d3906a9 100644 --- a/zstd/src/encoding/match_table/storage/storage_tests.rs +++ b/zstd/src/encoding/match_table/storage/storage_tests.rs @@ -109,7 +109,7 @@ fn skip_matching_bt_dense_routes_through_bt_update_tree() { fn replay_history_for_rebase_bt_walks_inserted_prefix() { let mut t = new_table(64); // Construct a contiguous mirror long enough for the BT walker - // (`bt_insert_step_no_rebase` reads 8-byte prefixes). + // (`bt_insert_range` reads 8-byte prefixes). t.history = vec![0u8; 64]; for (i, slot) in t.history.iter_mut().enumerate() { *slot = (i % 17) as u8; From ea02b1e89cf11076dd593f45596f943ea4491e28 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 16:51:54 +0300 Subject: [PATCH 20/36] test(miri): let the aarch64 prefetch hints fall back to no-ops Miri cannot execute inline assembly, and the aarch64 prefetch hints are inline assembly, so every test that reaches the dfast matcher (levels 3 and 4) or the decoder's prefetch aborted under Miri on aarch64. Under cfg(miri) the hints now take the portable no-op fallback; a hint has no observable effect, and every other build is unchanged. all_levels_tiny_input_with_hint now runs under Miri with the scalar kernel. --- zstd/src/decoding/prefetch.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/zstd/src/decoding/prefetch.rs b/zstd/src/decoding/prefetch.rs index 4d7525b7c..d497b5666 100644 --- a/zstd/src/decoding/prefetch.rs +++ b/zstd/src/decoding/prefetch.rs @@ -119,13 +119,15 @@ fn prefetch_stride_x86(slice: &[u8]) { } } -#[cfg(target_arch = "aarch64")] +// The aarch64 hints are inline assembly, which Miri cannot execute; under Miri +// they take the no-op fallback below, a hint having no observable effect. +#[cfg(all(target_arch = "aarch64", not(miri)))] #[inline(always)] fn prefetch_slice_impl_l1(slice: &[u8]) { prefetch_stride_aarch64::(slice); } -#[cfg(target_arch = "aarch64")] +#[cfg(all(target_arch = "aarch64", not(miri)))] #[inline(always)] fn prefetch_first_line_l1_impl(ptr: *const u8) { use core::arch::asm; @@ -138,13 +140,13 @@ fn prefetch_first_line_l1_impl(ptr: *const u8) { } } -#[cfg(target_arch = "aarch64")] +#[cfg(all(target_arch = "aarch64", not(miri)))] #[inline(always)] fn prefetch_slice_impl_t1(slice: &[u8]) { prefetch_stride_aarch64::(slice); } -#[cfg(target_arch = "aarch64")] +#[cfg(all(target_arch = "aarch64", not(miri)))] #[inline(always)] fn prefetch_stride_aarch64(slice: &[u8]) { use core::arch::asm; @@ -182,7 +184,7 @@ fn prefetch_stride_aarch64(slice: &[u8]) { #[cfg(not(any( target_arch = "x86_64", all(target_arch = "x86", target_feature = "sse"), - target_arch = "aarch64", + all(target_arch = "aarch64", not(miri)), )))] #[inline(always)] fn prefetch_slice_impl_l1(_slice: &[u8]) {} @@ -190,7 +192,7 @@ fn prefetch_slice_impl_l1(_slice: &[u8]) {} #[cfg(not(any( target_arch = "x86_64", all(target_arch = "x86", target_feature = "sse"), - target_arch = "aarch64", + all(target_arch = "aarch64", not(miri)), )))] #[inline(always)] fn prefetch_first_line_l1_impl(_ptr: *const u8) {} @@ -198,7 +200,7 @@ fn prefetch_first_line_l1_impl(_ptr: *const u8) {} #[cfg(not(any( target_arch = "x86_64", all(target_arch = "x86", target_feature = "sse"), - target_arch = "aarch64", + all(target_arch = "aarch64", not(miri)), )))] #[inline(always)] fn prefetch_slice_impl_t1(_slice: &[u8]) {} From d9107b8957d217e9cf0184ef09f8abc362f93380 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 16:59:57 +0300 Subject: [PATCH 21/36] perf(encoding): let a match cell carry offset and length only Every price improvement in the optimal parser's forward pass wrote the whole cell: offset, length, literal count and the three repeat offsets of the position the match started from, the last three reloaded from the stack for each write. Those repeat offsets were never read. When the pass reaches a match-ended cell it derives the history from the match's start (the ZSTD_newRep step), overwriting them, and nothing reads a cell ahead of the pass except its offset, length and literal count. Upstream stores the same three fields plus the price (ZSTD_compressBlock_opt_generic) and applies ZSTD_newRep at cur. - `HcOptimalNode::write_match_end` writes offset, length and a zero literal count through field places, leaving the history untouched; the btopt seed and forward loops and every price-set kernel use it, and the kernels lose their `reps` parameter - the two reads of a cell before the pass has derived its history (the match a literal replaces, and the cell the pass is on) read the three fields instead of copying the node, since the arena is uninitialised memory and the history of such a cell may never have been written Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary, and with --long=27 at L16-L22 on 8 MiB of source text. Miri (scalar kernel) passes all_levels_tiny_input_with_hint and btultra2_sparse_skip_matching_preserves_tail_cross_block_match. callgrind, 3 frames of z000033[..200000]: L13 378,521,611 -> 376,366,552 (-0.57%) L16 625,866,312 -> 615,970,215 (-1.58%) L19 929,718,179 -> 911,138,258 (-2.00%) runner1 task-clock, interleaved with the previous build, pinned, three rounds: L13 1045-1052 -> 1037-1041 ms, L19 2677-2711 -> 2659-2664 ms, L16 overlapping, while the L12 control, which never runs this parser, moved by up to 2% between rounds. Under the 1.5% floor: an operation reduction, not a claimed speed change. Part of #128 --- zstd/src/encoding/hc/optimal.rs | 75 ++++++++++++++++++-------------- zstd/src/encoding/hc/priceset.rs | 38 ++++------------ zstd/src/encoding/opt/types.rs | 21 +++++++++ 3 files changed, 71 insertions(+), 63 deletions(-) diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index 3842ae341..d87a82134 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -489,12 +489,11 @@ macro_rules! build_optimal_plan_impl_body { let node_price = unsafe { *node_prices.add(match_len) }; if match_len > last_pos || next_cost < node_price { unsafe { - *nodes.add(match_len) = HcOptimalNode { - off: candidate.offset as u32, - mlen: match_len as u32, - litlen: 0, - reps: initial_reps, - }; + HcOptimalNode::write_match_end( + nodes.add(match_len), + candidate.offset as u32, + match_len as u32, + ); *node_prices.add(match_len) = next_cost; } if match_len > last_pos { @@ -546,14 +545,20 @@ macro_rules! build_optimal_plan_impl_body { // below) — also the price of `prev_match`, the pre-overwrite copy. let node_pos_price = unsafe { *node_prices.add(pos) }; if lit_cost <= node_pos_price { - // An unreached cell has no node to read; the default node - // (`litlen != 0`) fails the end-of-match test below exactly - // as a reset node would. - let prev_match = if node_pos_price != u32::MAX { - unsafe { *nodes.add(pos) } - } else { - HcOptimalNode::default() - }; + // The cell being replaced, read field by field: a match + // written it without a repeat history (the pass derives that + // on arrival), so the node as a whole may not be initialised. + // An unreached cell has no node to read; a zero length fails + // the end-of-match test below exactly as a reset node would. + let (prev_match_off, prev_match_mlen, prev_match_litlen) = + if node_pos_price != u32::MAX { + unsafe { + let cell = nodes.add(pos); + ((*cell).off, (*cell).mlen, (*cell).litlen) + } + } else { + (0, 0, u32::MAX) + }; unsafe { *nodes.add(pos) = HcOptimalNode { litlen: lit_len as u32, @@ -563,8 +568,8 @@ macro_rules! build_optimal_plan_impl_body { } #[allow(clippy::collapsible_if)] if opt_level - && prev_match.mlen > 0 - && prev_match.litlen == 0 + && prev_match_mlen > 0 + && prev_match_litlen == 0 && pos < $current_len { if ll1_price < ll0_price { @@ -598,13 +603,13 @@ macro_rules! build_optimal_plan_impl_body { let next_price = unsafe { *node_prices.add(next) }; if with1literal < with_more_literals && with1literal < next_price { // Upstream zstd parity (zstd_opt.c:1232): `cur >= prevMatch.mlen`. - debug_assert!(pos >= prev_match.mlen as usize); - let prev_pos = pos - prev_match.mlen as usize; + debug_assert!(pos >= prev_match_mlen as usize); + let prev_pos = pos - prev_match_mlen as usize; { debug_assert!(unsafe { *node_prices.add(prev_pos) } != u32::MAX); let prev_state = unsafe { *nodes.add(prev_pos) }; let (_, reps_after_match) = BtMatcher::encode_offset_with_reps( - prev_match.off, + prev_match_off, prev_state.litlen as usize, prev_state.reps, ); @@ -613,9 +618,10 @@ macro_rules! build_optimal_plan_impl_body { // joins the frontier below. unsafe { *nodes.add(next) = HcOptimalNode { - reps: reps_after_match, + off: prev_match_off, + mlen: prev_match_mlen, litlen: 1, - ..prev_match + reps: reps_after_match, }; *node_prices.add(next) = with1literal; } @@ -649,15 +655,20 @@ macro_rules! build_optimal_plan_impl_body { continue; } { - let base_node = unsafe { *nodes.add(pos) }; - if base_node.mlen > 0 && base_node.litlen == 0 { + // Field by field: a match-written cell has no repeat history + // yet, which this block is about to derive. + let (base_off, base_mlen, base_litlen) = unsafe { + let cell = nodes.add(pos); + ((*cell).off, (*cell).mlen, (*cell).litlen) + }; + if base_mlen > 0 && base_litlen == 0 { // Upstream zstd parity (zstd_opt.c:1255): `cur >= opt[cur].mlen`. - debug_assert!(pos >= base_node.mlen as usize); - let prev_pos = pos - base_node.mlen as usize; + debug_assert!(pos >= base_mlen as usize); + let prev_pos = pos - base_mlen as usize; debug_assert!(unsafe { *node_prices.add(prev_pos) } != u32::MAX); let prev_state = unsafe { *nodes.add(prev_pos) }; let (_, reps_after_match) = BtMatcher::encode_offset_with_reps( - base_node.off, + base_off, prev_state.litlen as usize, prev_state.reps, ); @@ -815,12 +826,11 @@ macro_rules! build_optimal_plan_impl_body { let node_next_price = unsafe { *node_prices.add(next) }; if next > last_pos || next_cost < node_next_price { unsafe { - *nodes.add(next) = HcOptimalNode { - off: candidate.offset as u32, - mlen: match_len as u32, - litlen: 0, - reps: base_reps, - }; + HcOptimalNode::write_match_end( + nodes.add(next), + candidate.offset as u32, + match_len as u32, + ); *node_prices.add(next) = next_cost; } if next > last_pos { @@ -869,7 +879,6 @@ macro_rules! build_optimal_plan_impl_body { off_price, base_cost, candidate.offset as u32, - base_reps, last_pos, ) }); diff --git a/zstd/src/encoding/hc/priceset.rs b/zstd/src/encoding/hc/priceset.rs index 5d493170a..0e29ddada 100644 --- a/zstd/src/encoding/hc/priceset.rs +++ b/zstd/src/encoding/hc/priceset.rs @@ -109,7 +109,6 @@ pub(crate) fn priceset_range_nonabort_scalar( off_price: u32, base_cost: u32, off: u32, - reps: [u32; 3], last_pos: usize, ) -> usize { let mut new_last = last_pos; @@ -120,12 +119,8 @@ pub(crate) fn priceset_range_nonabort_scalar( let next = pos + ml; if next_cost < node_prices[next] { node_prices[next] = next_cost; - nodes[next].write(HcOptimalNode { - off, - mlen: ml as u32, - litlen: 0, - reps, - }); + // SAFETY: `nodes[next]` is an in-bounds cell of the slice. + unsafe { HcOptimalNode::write_match_end(nodes[next].as_mut_ptr(), off, ml as u32) }; if next > new_last { new_last = next; } @@ -177,7 +172,6 @@ fn priceset_range_vec( off_price: u32, base_cost: u32, off: u32, - reps: [u32; 3], last_pos: usize, deint: impl Fn(&[[u32; 2]], u32) -> Option<[u32; W]>, mask: impl Fn(&[u32; W], &[u32]) -> u8, @@ -235,12 +229,10 @@ fn priceset_range_vec( bits &= bits - 1; let next = base_next + k; node_prices[next] = buf[k]; - nodes[next].write(HcOptimalNode { - off, - mlen: (ml + k) as u32, - litlen: 0, - reps, - }); + // SAFETY: `nodes[next]` is an in-bounds cell of the slice. + unsafe { + HcOptimalNode::write_match_end(nodes[next].as_mut_ptr(), off, (ml + k) as u32) + }; if next > new_last { new_last = next; } @@ -254,12 +246,8 @@ fn priceset_range_vec( let next = pos + ml; if next_cost < node_prices[next] { node_prices[next] = next_cost; - nodes[next].write(HcOptimalNode { - off, - mlen: ml as u32, - litlen: 0, - reps, - }); + // SAFETY: `nodes[next]` is an in-bounds cell of the slice. + unsafe { HcOptimalNode::write_match_end(nodes[next].as_mut_ptr(), off, ml as u32) }; if next > new_last { new_last = next; } @@ -337,7 +325,6 @@ pub(crate) unsafe fn priceset_range_nonabort_avx2( off_price: u32, base_cost: u32, off: u32, - reps: [u32; 3], last_pos: usize, ) -> usize { priceset_range_vec::<8, ACCURATE>( @@ -354,7 +341,6 @@ pub(crate) unsafe fn priceset_range_nonabort_avx2( off_price, base_cost, off, - reps, last_pos, // SAFETY: both closures run inside this fn's avx2 target_feature umbrella. |cells, stamp| unsafe { priceset_cached_prices8_avx2(cells, stamp) }, @@ -420,7 +406,6 @@ pub(crate) unsafe fn priceset_range_nonabort_neon( off_price: u32, base_cost: u32, off: u32, - reps: [u32; 3], last_pos: usize, ) -> usize { priceset_range_vec::<4, ACCURATE>( @@ -437,7 +422,6 @@ pub(crate) unsafe fn priceset_range_nonabort_neon( off_price, base_cost, off, - reps, last_pos, // SAFETY: both closures run inside this fn's neon target_feature umbrella. |cells, stamp| unsafe { priceset_cached_prices4_neon(cells, stamp) }, @@ -552,7 +536,6 @@ pub(crate) unsafe fn priceset_range_nonabort_sse41( off_price: u32, base_cost: u32, off: u32, - reps: [u32; 3], last_pos: usize, ) -> usize { priceset_range_vec::<4, ACCURATE>( @@ -569,7 +552,6 @@ pub(crate) unsafe fn priceset_range_nonabort_sse41( off_price, base_cost, off, - reps, last_pos, // SAFETY: both closures run inside this fn's sse4.2 target_feature // umbrella, which covers the SSE2 loader and the SSE4.1 mask. @@ -599,7 +581,6 @@ pub(crate) unsafe fn priceset_range_nonabort_sse2( off_price: u32, base_cost: u32, off: u32, - reps: [u32; 3], last_pos: usize, ) -> usize { priceset_range_vec::<4, ACCURATE>( @@ -616,7 +597,6 @@ pub(crate) unsafe fn priceset_range_nonabort_sse2( off_price, base_cost, off, - reps, last_pos, // SAFETY: both closures run inside this fn's sse2 target_feature umbrella. |cells, stamp| unsafe { priceset_cached_prices4_sse2(cells, stamp) }, @@ -693,7 +673,6 @@ pub(crate) unsafe fn priceset_range_nonabort_simd128( off_price: u32, base_cost: u32, off: u32, - reps: [u32; 3], last_pos: usize, ) -> usize { priceset_range_vec::<4, ACCURATE>( @@ -710,7 +689,6 @@ pub(crate) unsafe fn priceset_range_nonabort_simd128( off_price, base_cost, off, - reps, last_pos, // SAFETY: both closures run inside this fn's simd128 target_feature umbrella. |cells, stamp| unsafe { priceset_cached_prices4_simd128(cells, stamp) }, diff --git a/zstd/src/encoding/opt/types.rs b/zstd/src/encoding/opt/types.rs index c8265e9ca..243bf38e7 100644 --- a/zstd/src/encoding/opt/types.rs +++ b/zstd/src/encoding/opt/types.rs @@ -40,6 +40,27 @@ pub(crate) struct HcOptimalNode { pub(crate) reps: [u32; 3], } +impl HcOptimalNode { + /// Record a match ending at `cell`: its offset, its length and an empty + /// literal run. The repeat history is left as it was, possibly + /// uninitialised: the forward pass derives it when it reaches the cell, as + /// upstream's does (`ZSTD_compressBlock_opt_generic` stores `mlen`, `off`, + /// `litlen` and the price, and applies `ZSTD_newRep` at `cur`). + /// + /// # Safety + /// `cell` must be valid for writes. + #[inline(always)] + pub(crate) unsafe fn write_match_end(cell: *mut Self, off: u32, mlen: u32) { + // SAFETY: field places of a writable cell; no reference to the whole + // (possibly uninitialised) node is formed. + unsafe { + core::ptr::addr_of_mut!((*cell).off).write(off); + core::ptr::addr_of_mut!((*cell).mlen).write(mlen); + core::ptr::addr_of_mut!((*cell).litlen).write(0); + } + } +} + impl Default for HcOptimalNode { fn default() -> Self { Self { From 45132562ce61de421bb946407524f2cd2793ae95 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 17:46:17 +0300 Subject: [PATCH 22/36] perf(encoding): take the tree's coordinates once per block Every position the optimal parser searched re-derived the binary tree's coordinates from the table: the biases that map a stored index to its absolute position, its history index and its pair slot, the pair mask from the chain log, the position's stored index and its pair slot. That was about thirty instructions of the search's prologue, most of it spilled to the stack straight away, on 353 thousand calls per 3 frames at level 13. All of it follows from the position base, the index shift, the history start and the chain log, none of which moves while an armed block is parsed. Upstream resolves `base` and `btMask` once per call of its whole search. `MatchTable::capture_block_coords` takes them into a `BtCoords` at the start of each block pass, and the search reads the snapshot. Debug builds compare the snapshot with a fresh derivation on every position, and the stored index and pair slot with their table forms, so the whole debug suite checks that nothing moves them mid-block. Byte-identical at L1-L22 on z000033[..200000], z000033 and z000033 with dict_tests/dictionary, and with --long=27 at L16-L22 on 8 MiB of source text. callgrind, 3 frames of z000033[..200000]: L13 376,366,552 -> 369,209,950 (-1.90%) L16 615,970,215 -> 614,154,481 (-0.29%) L19 911,138,258 -> 908,885,890 (-0.25%) runner1 task-clock, interleaved with the previous build and libzstd, pinned, three rounds: L13 1037-1039 -> 1028-1031 ms, L16 1799-1809 -> 1787-1790 ms (both disjoint), L19 overlapping, L12 control 477-483 -> 476-490. Under the 1.5% floor: an operation reduction, not a claimed speed change. Part of #128 --- zstd/src/encoding/hc/generator.rs | 30 +++++++++--------- zstd/src/encoding/hc/optimal.rs | 3 ++ zstd/src/encoding/match_table/storage.rs | 40 ++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index 0abd1f45a..dcd390220 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -903,9 +903,15 @@ macro_rules! bt_insert_and_collect_matches_body { } } } + // The tree's coordinates are fixed for the block and were taken before + // its parse, so none is derived again per position. + let coords = $table.block_coords; + debug_assert_eq!(coords, $table.bt_coords(), "block coordinates are stale"); // Total, not tested: the block was armed before the parse began. - let stored = $table.relative_position_armed($abs_pos) + 1; - let bt_mask = $table.bt_mask(); + // `abs_pos - abs_bias` is `relative_position_armed(abs_pos) + 1`. + debug_assert!($table.can_skip_rebase_check($abs_pos)); + let stored = $abs_pos.wrapping_sub(coords.abs_bias) as u32; + let bt_mask = coords.bt_mask; // See `bt_insert_range_body!`: saturating is needed for the // first BT walk of a fresh frame where `abs_pos < bt_mask`. let bt_low = $abs_pos.saturating_sub(bt_mask); @@ -920,11 +926,7 @@ macro_rules! bt_insert_and_collect_matches_body { // abs_pos - window_low ⟺ s.wrapping_add(win_off) < win_range. // HC_EMPTY (s = 0) maps to base = (lowest representable abs) - 1 < // window_low, so it falls out of range and ends the walk. - let win_off = $table - .position_base - .wrapping_sub(1) - .wrapping_sub($table.index_shift) - .wrapping_sub(window_low); + let win_off = coords.abs_bias.wrapping_sub(window_low); let win_range = $abs_pos - window_low; // Decode biases: fold the per-node coordinate conversions into // loop-invariant additions. The gate-validated chain entry @@ -938,12 +940,9 @@ macro_rules! bt_insert_and_collect_matches_body { // single-coordinate equivalent. Wrapping throughout: the window gate // already proved `match_stored ∈ [window_low, abs_pos)` before decode, // mirroring the `win_off` form above. - let abs_bias = $table - .position_base - .wrapping_sub(1) - .wrapping_sub($table.index_shift); - let idx_bias = abs_bias.wrapping_sub($table.history_abs_start); - let bt_bias = $table.position_base.wrapping_sub(1); + let abs_bias = coords.abs_bias; + let idx_bias = coords.idx_bias; + let bt_bias = coords.bt_bias; // Raw `+ 9` is safe here — see `bt_insert_range_body!` // for the full discussion of the upstream `STREAM_ABS_HEADROOM` // cap in `MatchTable::add_data`. @@ -959,7 +958,10 @@ macro_rules! bt_insert_and_collect_matches_body { let mut compares_left = ($max_chain_depth).min($search_depth); let mut common_length_smaller = 0usize; let mut common_length_larger = 0usize; - let pair_idx = $table.bt_pair_index_for_abs($abs_pos); + // `bt_pair_index_for_abs(abs_pos)`: `stored + bt_bias` is + // `abs_pos + index_shift`. + let pair_idx = 2 * ((stored as usize).wrapping_add(bt_bias) & bt_mask); + debug_assert_eq!(pair_idx, $table.bt_pair_index_for_abs($abs_pos)); let mut smaller_slot = pair_idx; let mut larger_slot = pair_idx + 1; // SAFETY: `hash` is masked to `hash_log` bits and the table is diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index d87a82134..ffe87097f 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -1095,6 +1095,8 @@ macro_rules! optimal_block_body { $collect:ident, $priceset:path $(,)? ) => {{ + // The block is armed, so the tree's coordinates hold for the whole pass. + $self.table.capture_block_coords(); // Everything the pass carries between segments lives in `$buffers.pass`, // read at the top of a segment and written at its end, so none of it is // live across the segment body (see `HcOptimalPlanBuffers::pass`). @@ -2063,6 +2065,7 @@ impl HcMatchGenerator { ) { use crate::encoding::strategy::{self, StrategyTag}; self.table.ensure_tables(); + self.table.capture_block_coords(); let reps = &query.reps; let ll0 = query.lit_len == 0; // Dispatch purely from `self.strategy_tag` (set by diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index c2c060aa2..68de61256 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -160,6 +160,20 @@ pub(crate) const HC_CHAIN_LOG: usize = 19; /// modes leave it sized to zero. pub(crate) const HC3_HASH_LOG: usize = 17; +/// The binary tree's coordinates: the biases that map a stored index to its +/// absolute position (`abs_bias`), its history index (`idx_bias`) and its pair +/// slot (`bt_bias`, then `& bt_mask`, doubled). They follow from the position +/// base, the index shift, the history start and the chain log, none of which +/// moves while an armed block is parsed, so the parser takes them once per +/// block, as upstream resolves `base` and `btMask` once per call. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct BtCoords { + pub(crate) abs_bias: usize, + pub(crate) idx_bias: usize, + pub(crate) bt_bias: usize, + pub(crate) bt_mask: usize, +} + /// Shared storage backing every match finder. Holds the contiguous /// Immutable dictionary match structure (upstream zstd `ZSTD_dictMatchState`) for /// the binary-tree / optimal path. A hash + single-link chain over the @@ -283,6 +297,9 @@ pub(crate) struct MatchTable { /// guarantee 8 readable bytes); the HC `hash_position` stays 4-byte. /// Defaults to `4`. pub(crate) search_mls: usize, + /// Tree coordinates of the block being parsed, taken by + /// [`Self::capture_block_coords`] before the parse. + pub(crate) block_coords: BtCoords, /// Immutable dictionary match chain (upstream zstd `ZSTD_dictMatchState`), /// searched by the BT/optimal collect alongside the live tree. `Some` /// once primed from a non-empty dictionary on a BT level. @@ -340,6 +357,7 @@ impl Clone for MatchTable { is_btultra2: self.is_btultra2, uses_bt: self.uses_bt, search_mls: self.search_mls, + block_coords: self.block_coords, dms: self.dms.clone(), borrowed_input: self.borrowed_input, borrowed_block: self.borrowed_block, @@ -382,6 +400,7 @@ impl Clone for MatchTable { self.is_btultra2 = source.is_btultra2; self.uses_bt = source.uses_bt; self.search_mls = source.search_mls; + self.block_coords = source.block_coords; self.borrowed_input = source.borrowed_input; self.borrowed_block = source.borrowed_block; self.kernel = source.kernel; @@ -512,6 +531,7 @@ impl MatchTable { is_btultra2: false, uses_bt: false, search_mls: 4, + block_coords: BtCoords::default(), dms: DictAttach::new(), borrowed_input: None, borrowed_block: None, @@ -1426,6 +1446,26 @@ impl MatchTable { (1usize << self.bt_log()) - 1 } + /// The binary tree's coordinates as the table stands now. + pub(crate) fn bt_coords(&self) -> BtCoords { + let abs_bias = self + .position_base + .wrapping_sub(1) + .wrapping_sub(self.index_shift); + BtCoords { + abs_bias, + idx_bias: abs_bias.wrapping_sub(self.history_abs_start), + bt_bias: self.position_base.wrapping_sub(1), + bt_mask: self.bt_mask(), + } + } + + /// Take the tree's coordinates for the block about to be parsed; the + /// search reads them from [`Self::block_coords`] on every position. + pub(crate) fn capture_block_coords(&mut self) { + self.block_coords = self.bt_coords(); + } + /// Convert an absolute position into a BT pair index in /// `chain_table`. Each node occupies two consecutive slots /// (smaller, larger) so the result is doubled. Upstream zstd parity: From 4305e19a2ac8a8241e9525e023e763b14c2e3554 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 18:08:39 +0300 Subject: [PATCH 23/36] fix(dictionary): stop the legacy neighbour walks at the noise slots Every rank outside the suffix array reads the noise band, and the band is the same 32 bytes on every run. The neighbour walks in `analyze_position` ended only on a short match, so a corpus whose text repeats the band compared equal at every rank past either end and never stopped: `--train-legacy` and `create_legacy_dict_from_slice` hung on it. The reference reads past its two slots there, so it defines no result to keep. The forward walks now stop at the upper slot (the counting walk still counts it, as the reference does) and the backward walk at rank 0; on every other corpus the slot holds noise that ended the walk anyway. Carries neighbour_walks_stop_at_the_noise_slots, which runs a walk off the array over a corpus that is the band itself: it hung before the fix and passes after. The band's generator is split out for it. The dictionaries are unchanged: legacy_trainer_ffi matches libzstd's content in all its cases, and --train-legacy -B4096 over decodecorpus_files writes the same bytes. callgrind on 40 of those files: 12,959,833,409 -> 12,949,794,330 instructions. Part of #128 --- zstd/src/dictionary/legacy.rs | 49 +++++++++++++++++++++-------- zstd/src/dictionary/legacy/tests.rs | 18 +++++++++++ 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index d5a898b10..b880153cd 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -59,8 +59,24 @@ struct DictItem { savings: u32, } +/// The reference's noise band (`ZDICT_fillNoise`), the same bytes on every run. +fn noise_band() -> [u8; NOISE_LENGTH] { + let mut band = [0u8; NOISE_LENGTH]; + let mut acc: u32 = 2_654_435_761; + for byte in &mut band { + acc = acc.wrapping_mul(2_246_822_519); + *byte = (acc >> 21) as u8; + } + band +} + /// The corpus followed by its noise band, read through bounds-checked /// accessors that treat anything past the band as a mismatch. +/// +/// One contiguous copy, as the reference makes it +/// (`ZDICT_trainFromBuffer_legacy`). Borrowing the samples and keeping the +/// band apart saves the copy's memory, but joining the two in the accessors +/// measured 2.5-5% slower on the whole trainer at identical output. struct Corpus { bytes: Vec, } @@ -69,12 +85,7 @@ impl Corpus { fn new(samples: &[u8]) -> Self { let mut bytes = Vec::with_capacity(samples.len() + NOISE_LENGTH); bytes.extend_from_slice(samples); - // The reference's noise generator (`ZDICT_fillNoise`). - let mut acc: u32 = 2_654_435_761; - for _ in 0..NOISE_LENGTH { - acc = acc.wrapping_mul(2_246_822_519); - bytes.push((acc >> 21) as u8); - } + bytes.extend_from_slice(&noise_band()); Self { bytes } } @@ -133,10 +144,15 @@ impl Suffixes { Self { sa, noise: len } } - /// `suffix[at]`, where ranks `-1` and `len` (and any further out, which a - /// walk only reaches on a corpus whose tail matches the noise) are the - /// noise band. A negative rank wraps past every real one, so a single - /// compare tells the two apart. + /// Ranks in the array; rank `ranks()` is the upper noise slot. + #[inline] + fn ranks(&self) -> i64 { + self.sa.len() as i64 + } + + /// `suffix[at]`, where ranks `-1` and `len` are the noise band (the walks + /// stop at those two slots). A negative rank wraps past every real one, + /// so a single compare tells the two apart. #[inline] fn at(&self, at: i64) -> usize { match self.sa.get(at as u64 as usize) { @@ -315,13 +331,18 @@ fn analyze_position( } // The neighbours sharing at least the minimum length, forward then back. + // Each walk stops at its noise slot as well as on a short match: every + // rank past the array reads the same band, so a corpus that repeats the + // band would otherwise walk forever. The reference reads past its two + // slots there, so it has no result to keep; on every other corpus the + // slot holds noise that ends the walk anyway. loop { end += 1; - if corpus.common(pos, suffixes.at(end)) < MIN_MATCH_LENGTH { + if end >= suffixes.ranks() || corpus.common(pos, suffixes.at(end)) < MIN_MATCH_LENGTH { break; } } - while corpus.common(pos, suffixes.at(start - 1)) >= MIN_MATCH_LENGTH { + while start > 0 && corpus.common(pos, suffixes.at(start - 1)) >= MIN_MATCH_LENGTH { start -= 1; } @@ -377,7 +398,9 @@ fn analyze_position( end += 1; let length = corpus.common(pos, suffixes.at(end)).min(LENGTH_LIMIT - 1); lengths[length] += 1; - if length < MIN_MATCH_LENGTH { + // The upper noise slot is counted, as the reference counts it, and + // ends the walk (see the first walk above). + if length < MIN_MATCH_LENGTH || end >= suffixes.ranks() { break; } } diff --git a/zstd/src/dictionary/legacy/tests.rs b/zstd/src/dictionary/legacy/tests.rs index 8037736be..31c4984fa 100644 --- a/zstd/src/dictionary/legacy/tests.rs +++ b/zstd/src/dictionary/legacy/tests.rs @@ -2,6 +2,24 @@ use super::*; use alloc::format; use alloc::string::String; +/// Every rank past either end of the suffix array reads the noise band, and +/// the band is the same bytes on every run. A corpus whose text repeats the +/// band therefore compared equal at every rank past the end, and the +/// neighbour walks never stopped: training hung on such a corpus. The walks +/// stop at the two noise slots the reference allocates. +#[test] +fn neighbour_walks_stop_at_the_noise_slots() { + // The corpus is the band itself, and its one suffix is the only rank in + // the array: every walk leaves the array on its first step. + let band = noise_band(); + let corpus = Corpus::new(&band); + let suffixes = Suffixes::new(vec![0], band.len()); + let mut done = vec![false; band.len() + 16]; + let solution = analyze_position(&mut done, &suffixes, 0, &corpus, MIN_RATIO); + // One suffix cannot repeat `MIN_RATIO` times. + assert_eq!(solution.length, 0); +} + /// `count` log lines of a few shapes, each line a sample. fn log_samples(count: u32) -> (Vec, Vec) { const SHAPES: [&str; 4] = [ From ce2eeaf2211825adc39aaa5553addaff4018d32e Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 19:10:28 +0300 Subject: [PATCH 24/36] docs: -M on legacy training, dictionary parity tests in CI, private doc links - README: `-M` caps the samples `--train-legacy` loads, as the reference's command hands its memory limit to `DiB_trainFromFiles`; the claim that it describes nothing is narrowed to compressing and listing. The call site says why the limit is passed. - The dictionary parity targets against libzstd (`dictionary_ffi`, `legacy_trainer_ffi`) need `dict-builder` as well as `bench-internals`, and CI ran ffi-bench with `bench-internals` alone, so both compiled to nothing there. Both now require the two features and CI enables them. - Two intra-doc links in `compressed.rs` pointed at a renamed function and at a test-only one, failing rustdoc with private items. Part of #128 --- .github/workflows/ci.yml | 5 +++-- README.md | 10 ++++++---- ffi-bench/Cargo.toml | 4 ++-- zstd/src/bin/structured-zstd/main.rs | 4 ++++ zstd/src/encoding/blocks/compressed.rs | 4 ++-- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b14d0b63f..c7e44a33d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -348,9 +348,10 @@ jobs: - name: FFI parity + cross-validation tests # The libzstd-linking parity / cross-validation / corpus tests live in # `ffi-bench`; `bench-internals` activates the white-box-facade targets - # gated behind it. Corpus fixtures resolve from the manifest dir, so no + # gated behind it, and `dict-builder` the trainer parity targets. + # Corpus fixtures resolve from the manifest dir, so no # working-directory is needed. - run: cargo nextest run --profile ci -p ffi-bench --features bench-internals + run: cargo nextest run --profile ci -p ffi-bench --features bench-internals,dict-builder cross-i686: needs: lint diff --git a/README.md b/README.md index 38c536436..c9407b1b2 100644 --- a/README.md +++ b/README.md @@ -111,8 +111,9 @@ anything but zstd and `--rsyncable`, which needs the worker threads this build does not have. `-M` is treated as the safety promise it is: on the runs that decode, a limit covering the 128 MiB window, the decoder's buffers and the `-D` dictionary is kept and a tighter one is refused rather than -ignored. Compressing, listing and training allocate no decoder, so the flag is -accepted there and describes nothing, as upstream has it. +ignored. Compressing and listing allocate no decoder, so the flag is accepted +there and describes nothing, as upstream has it; for `--train-legacy` it caps +the samples loaded, as below. `--train` and `--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#]` train with FastCOVER, the algorithm upstream also defaults to (a knob set to zero @@ -121,8 +122,9 @@ with the COVER trainer. Its tuning, `--train-cover=...`, is refused rather than misread: the reference-side parameters name knobs this trainer does not have. `--train-legacy[=s=#]` (or `-s#`) runs upstream's original trainer, which counts samples: they are loaded as upstream loads them (each file one sample of -up to 128 KiB, or cut into `-B#` pieces), and for the same file list the -dictionary carries the same content as upstream's. `-D` takes either a dictionary produced by `--train` or any file at +up to 128 KiB, or cut into `-B#` pieces, whole samples up to 2 GiB or `-M` when +that is smaller), and for the same file list the dictionary carries the same +content as upstream's. `-D` takes either a dictionary produced by `--train` or any file at all, which is then used as raw content the way upstream does; such a dictionary has no ID, so the same bytes must be supplied when decoding. diff --git a/ffi-bench/Cargo.toml b/ffi-bench/Cargo.toml index 8f5107b43..fa4f22a4a 100644 --- a/ffi-bench/Cargo.toml +++ b/ffi-bench/Cargo.toml @@ -133,12 +133,12 @@ required-features = ["bench-internals"] [[test]] name = "dictionary_ffi" path = "tests/dictionary_ffi.rs" -required-features = ["bench-internals"] +required-features = ["bench-internals", "dict-builder"] [[test]] name = "legacy_trainer_ffi" path = "tests/legacy_trainer_ffi.rs" -required-features = ["bench-internals"] +required-features = ["bench-internals", "dict-builder"] [[test]] name = "encode_corpus_ffi" diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 162bf5ede..fb0a08474 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -3325,6 +3325,10 @@ fn train_dictionary(opts: &Options) -> Result<()> { // The legacy trainer counts samples, so they are loaded as the // reference's command loads them: shuffled, capped per file, and // cut by `-B`. The same files then yield the same content. + // `-M` caps what is loaded, as the reference's command passes its + // memory limit to `DiB_trainFromFiles` (zstdcli.c), which keeps + // whole samples up to it (dibio.c); dropping it would train on a + // different corpus than the reference for the same command line. let set = load_training_samples(&opts.inputs, opts.block_size, opts.memory_limit)?; create_legacy_dict_from_slice( &set.corpus, diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index a5a9fee8f..2cdc71344 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -296,7 +296,7 @@ impl SequencePrefixSums { /// One collected sequence. /// -/// `off_base` holds the offset the matcher found until [`fill_wire_offsets`] +/// `off_base` holds the offset the matcher found until [`fill_and_count`] /// runs over the sequence, and the wire code from then on: 1/2/3 for the repeat /// offsets, N+3 for an explicit N. It is one field rather than two because the /// found offset has no reader once its code exists, and a fourth word would @@ -989,7 +989,7 @@ enum HuffOutcome { New(huff0_encoder::HuffmanTable), } -/// [`estimate_block_parts_size`] for sequences whose length codes are already +/// `estimate_block_parts_size` for sequences whose length codes are already /// derived: the splitter's probes, which price many ranges of one block. The /// Huffman table the section may repeat is `previous`, and what the section /// does with it is returned beside the size; the FSE repeat tables and the From 15a106da015eab2db029e1b5e31baf31dc14dc85 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 19:36:53 +0300 Subject: [PATCH 25/36] perf(dictionary): borrow the legacy training corpus The legacy trainer copied every sample into one buffer followed by the noise band, as the reference does. The corpus now borrows the samples and keeps the band beside them; the accessors join the two, with the reads that cross into the band out of line. The word loop in `common` reads unaligned words through a raw pointer. Through a slice index each word paid a bound test, because the loop limit does not prove every index in range; the owned version paid an overflow test per word for the same reason. Those tests, not the borrow, were what made an earlier borrowed version measure slower. --train-legacy -B4096 over decodecorpus_files, runner1 (load 0.03), bench profile, three interleaved rounds pinned to one core: - wall time 15.92-15.97 s -> 8.92-9.14 s - max RSS 251 MB -> 235 MB - dictionary bytes identical (md5 857c43bb70a0abeaf5e1bc28e8479edd) - zstd 1.5.7 in the same session: 7.45-7.58 s, 181 MB, so 2.1x -> 1.2x legacy_trainer_ffi still matches libzstd's content in every case, and the legacy tests pass under Miri. Part of #128 --- zstd/src/dictionary/legacy.rs | 147 +++++++++++++++++++++++++--------- 1 file changed, 108 insertions(+), 39 deletions(-) diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index b880153cd..f388525e3 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -73,61 +73,128 @@ fn noise_band() -> [u8; NOISE_LENGTH] { /// The corpus followed by its noise band, read through bounds-checked /// accessors that treat anything past the band as a mismatch. /// -/// One contiguous copy, as the reference makes it -/// (`ZDICT_trainFromBuffer_legacy`). Borrowing the samples and keeping the -/// band apart saves the copy's memory, but joining the two in the accessors -/// measured 2.5-5% slower on the whole trainer at identical output. -struct Corpus { - bytes: Vec, +/// The samples are borrowed and the band kept apart, and the accessors join +/// the two. The reference copies the whole corpus into a buffer one band +/// longer (`ZDICT_trainFromBuffer_legacy`), a second copy of the samples while +/// the caller still holds them for the finalizer. +struct Corpus<'a> { + samples: &'a [u8], + noise: [u8; NOISE_LENGTH], } -impl Corpus { - fn new(samples: &[u8]) -> Self { - let mut bytes = Vec::with_capacity(samples.len() + NOISE_LENGTH); - bytes.extend_from_slice(samples); - bytes.extend_from_slice(&noise_band()); - Self { bytes } +impl<'a> Corpus<'a> { + fn new(samples: &'a [u8]) -> Self { + Self { + samples, + noise: noise_band(), + } + } + + /// Length of the samples and the band together. + #[inline] + fn len(&self) -> usize { + self.samples.len() + NOISE_LENGTH } #[inline] fn byte(&self, at: usize) -> Option { - self.bytes.get(at).copied() + match self.samples.get(at) { + Some(&byte) => Some(byte), + None => self.noise.get(at - self.samples.len()).copied(), + } + } + + /// `N` bytes from `at`, or `None` past the band. + #[inline(always)] + fn read(&self, at: usize) -> Option<[u8; N]> { + if let Some(bytes) = self.samples.get(at..at + N) { + return Some(bytes.try_into().expect("N bytes")); + } + self.read_across(at) + } + + /// [`Self::read`] for a read that reaches the band, which only the last + /// few positions of the corpus make. + #[cold] + #[inline(never)] + fn read_across(&self, at: usize) -> Option<[u8; N]> { + let mut out = [0u8; N]; + for (i, slot) in out.iter_mut().enumerate() { + *slot = self.byte(at + i)?; + } + Some(out) } #[inline] fn read16(&self, at: usize) -> Option { - let pair = self.bytes.get(at..at + 2)?; - Some(u16::from_le_bytes([pair[0], pair[1]])) + self.read::<2>(at).map(u16::from_le_bytes) } #[inline] fn read64(&self, at: usize) -> Option { - let word = self.bytes.get(at..at + 8)?; - Some(u64::from_le_bytes(word.try_into().expect("eight bytes"))) + self.read::<8>(at).map(u64::from_le_bytes) } /// Bytes `a` and `b` have in common (`ZDICT_count`), compared a word at - /// a time as the reference compares them. + /// a time as the reference compares them while both words lie in the + /// samples, then a byte at a time across into the band. #[inline] fn common(&self, a: usize, b: usize) -> usize { - let bytes = self.bytes.as_slice(); - let Some(limit) = bytes.len().checked_sub(a.max(b)) else { - return 0; + let samples = self.samples; + // A comparison that starts in the band has no word to read in the + // samples. + let Some(in_samples) = samples.len().checked_sub(a.max(b)) else { + return self.common_tail(a, b, 0); }; - let word = |at: usize| u64::from_le_bytes(bytes[at..at + 8].try_into().expect("8 bytes")); + let base = samples.as_ptr(); let mut n = 0; - while n + 8 <= limit { - let diff = word(a + n) ^ word(b + n); + while n + 8 <= in_samples { + // SAFETY: `a, b <= max(a, b)` and `n + 8 <= samples.len() - + // max(a, b)`, so both eight-byte reads end inside the samples. + // Read unchecked: through a slice index the bound is tested on + // every word, since the loop limit does not tell the optimiser + // that each index stays inside. + let (x, y) = unsafe { + ( + base.add(a + n).cast::().read_unaligned(), + base.add(b + n).cast::().read_unaligned(), + ) + }; + let diff = u64::from_le(x) ^ u64::from_le(y); if diff != 0 { return n + (diff.trailing_zeros() / 8) as usize; } n += 8; } - while n < limit && bytes[a + n] == bytes[b + n] { + self.common_tail(a, b, n) + } + + /// The last bytes of [`Self::common`] from `n` on, fewer than a word of + /// them in the samples, then on into the band. + #[cold] + #[inline(never)] + fn common_tail(&self, a: usize, b: usize, mut n: usize) -> usize { + let Some(limit) = self.len().checked_sub(a.max(b)) else { + return 0; + }; + while n < limit && self.byte(a + n) == self.byte(b + n) { n += 1; } n } + + /// Copy the `out.len()` bytes from `from` into `out`. + fn copy_to(&self, from: usize, out: &mut [u8]) { + if let Some(bytes) = self.samples.get(from..from + out.len()) { + out.copy_from_slice(bytes); + return; + } + for (i, slot) in out.iter_mut().enumerate() { + *slot = self + .byte(from + i) + .expect("a segment lies inside the corpus"); + } + } } /// The suffix array with one extra slot on each side, both pointing into the @@ -236,7 +303,7 @@ pub(crate) fn train_legacy_raw( let length = item.length as usize; let start = end - length; let from = item.pos as usize; - content[start..end].copy_from_slice(&corpus.bytes[from..from + length]); + corpus.copy_to(from, &mut content[start..end]); end = start; } debug_assert_eq!(end, 0, "the kept segments fill the content exactly"); @@ -245,9 +312,10 @@ pub(crate) fn train_legacy_raw( /// `ZDICT_trainBuffer_legacy`: walk every uncovered position of the corpus in /// text order and insert the segment its suffix neighbourhood yields. -fn find_segments(list: &mut [DictItem], corpus: &Corpus, len: usize, min_rep: u32) { +fn find_segments(list: &mut [DictItem], corpus: &Corpus<'_>, len: usize, min_rep: u32) { let min_ratio = min_rep.max(MIN_RATIO); - let sa = suffix_array(&corpus.bytes[..len]); + debug_assert_eq!(corpus.samples.len(), len); + let sa = suffix_array(corpus.samples); let mut rank = vec![0u32; len]; for (at, &pos) in sa.iter().enumerate() { rank[pos as usize] = at as u32; @@ -304,7 +372,7 @@ fn analyze_position( done: &mut [bool], suffixes: &Suffixes, mut start: i64, - corpus: &Corpus, + corpus: &Corpus<'_>, min_ratio: u32, ) -> DictItem { let mut pos = suffixes.at(start); @@ -462,15 +530,16 @@ fn analyze_position( solution } -/// Whether the `length` bytes at `a` equal those at `b` (`isIncluded`). -fn is_included(corpus: &Corpus, a: usize, b: usize, length: usize) -> bool { - match ( - corpus.bytes.get(a..a + length), - corpus.bytes.get(b..b + length), - ) { - (Some(x), Some(y)) => x == y, - _ => false, +/// Whether the `length` bytes at `a` equal those at `b` (`isIncluded`); both +/// runs must lie inside the corpus. Compares `length` bytes and no more, where +/// `common` would run on to the end of the shared prefix. +fn is_included(corpus: &Corpus<'_>, a: usize, b: usize, length: usize) -> bool { + let samples = corpus.samples; + if let (Some(x), Some(y)) = (samples.get(a..a + length), samples.get(b..b + length)) { + return x == y; } + a.max(b) + length <= corpus.len() + && (0..length).all(|i| corpus.byte(a + i) == corpus.byte(b + i)) } /// Move entry `at` towards the front while its savings beat its predecessor's. @@ -486,7 +555,7 @@ fn promote(list: &mut [DictItem], mut at: usize) -> usize { /// `ZDICT_tryMerge`: fold `elt` into an entry it overlaps, skipping entry /// `skip`. Returns the merged entry's index, or 0 when nothing merged. -fn try_merge(list: &mut [DictItem], elt: DictItem, skip: usize, corpus: &Corpus) -> usize { +fn try_merge(list: &mut [DictItem], elt: DictItem, skip: usize, corpus: &Corpus<'_>) -> usize { let size = list[0].pos as usize; let elt_end = elt.pos + elt.length; @@ -571,7 +640,7 @@ fn remove_item(list: &mut [DictItem], id: usize) { /// `ZDICT_insertDictItem`: merge `elt` into the table if it overlaps an entry, /// and keep merging while the merged entry overlaps another; otherwise insert /// it in savings order, dropping the last entry when the table is full. -fn insert_item(list: &mut [DictItem], max_size: u32, elt: DictItem, corpus: &Corpus) { +fn insert_item(list: &mut [DictItem], max_size: u32, elt: DictItem, corpus: &Corpus<'_>) { let mut merge_id = try_merge(list, elt, 0, corpus); if merge_id != 0 { loop { From 923f0c10e41882a01fb4f7dac06938e2882d6a37 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 20:23:50 +0300 Subject: [PATCH 26/36] perf(dictionary): build the suffix array in place The SA-IS construction held, beside the array itself, a byte per position for the suffix types, the LMS positions, a copy of the reduced string, the reduced string's own array and the sorted LMS positions, and four bucket tables per level. In the recursion the alphabet is the number of names, so the tables alone came to millions of entries per level. Under the legacy trainer this made the construction, not the analysis, the peak of the whole run. It is now laid out as Yuta Mori's sais.c lays it out: the reduced string is gathered at the back of the array and sorted recursively into the front, the LMS positions are recomputed from the text into the reduced string's slots once it is sorted, and placed at their buckets' ends in place. Two bucket tables per level (counts and a cursor), the types a bit per position, and no recursion at all when every name is distinct, since the names are then the order. --train-legacy -B4096 over decodecorpus_files, runner1 (load 0.56), bench profile, three interleaved rounds pinned to one core: - max RSS 235 MB -> 166 MB (zstd 1.5.7: 181 MB) - wall time 8.91-9.00 s -> 8.59-8.65 s - dictionary bytes identical (md5 857c43bb70a0abeaf5e1bc28e8479edd) massif before: peak 243 MB during the construction, 38 MB of it the first recursion level's bucket tables. 166 MB is what the analysis holds (samples, suffix array, rank array, marks: ten bytes a position), so the construction no longer sets the peak. legacy_trainer_ffi still matches libzstd's content in every case. Part of #128 --- zstd/src/dictionary/suffix_array.rs | 483 +++++++++++++++++----------- 1 file changed, 291 insertions(+), 192 deletions(-) diff --git a/zstd/src/dictionary/suffix_array.rs b/zstd/src/dictionary/suffix_array.rs index 593311e42..d309736ca 100644 --- a/zstd/src/dictionary/suffix_array.rs +++ b/zstd/src/dictionary/suffix_array.rs @@ -49,229 +49,328 @@ pub(crate) fn suffix_array(text: &[u8]) -> Vec { "a suffix array of {} bytes does not fit 31-bit positions", text.len() ); - sa_is(text, usize::from(u8::MAX)) + let mut sa = vec![0u32; text.len()]; + sa_is(text, &mut sa, usize::from(u8::MAX)); + sa +} + +/// Whether each suffix is S-type (smaller than the one after it), a bit per +/// position. The last is L-type against the virtual sentinel. +struct Types { + words: Vec, +} + +impl Types { + fn new(s: &[T]) -> Self { + let n = s.len(); + let mut words = vec![0u64; n.div_ceil(64)]; + let mut next = false; + for i in (0..n - 1).rev() { + let small = if s[i] == s[i + 1] { + next + } else { + s[i] < s[i + 1] + }; + words[i / 64] |= u64::from(small) << (i % 64); + next = small; + } + Self { words } + } + + #[inline] + fn is_s(&self, i: usize) -> bool { + (self.words[i / 64] >> (i % 64)) & 1 != 0 + } + + /// Whether `p` starts a leftmost S-type run. + #[inline] + fn is_lms(&self, p: usize, n: usize) -> bool { + p > 0 && p < n && self.is_s(p) && !self.is_s(p - 1) + } } -/// SA-IS over `s`, whose symbols all lie in `0..=upper`. The text is taken to -/// end in a virtual sentinel smaller than every symbol. -fn sa_is(s: &[T], upper: usize) -> Vec { +/// `buf[c]` = the first slot of symbol `c`'s bucket. +fn bucket_starts(counts: &[u32], buf: &mut [u32]) { + let mut sum = 0; + for (slot, &count) in buf.iter_mut().zip(counts) { + *slot = sum; + sum += count; + } +} + +/// `buf[c]` = one past the last slot of symbol `c`'s bucket. +fn bucket_ends(counts: &[u32], buf: &mut [u32]) { + let mut sum = 0; + for (slot, &count) in buf.iter_mut().zip(counts) { + sum += count; + *slot = sum; + } +} + +/// SA-IS over `s` into `sa` (as long as `s`), whose symbols all lie in +/// `0..=upper`. The text is taken to end in a virtual sentinel smaller than +/// every symbol. +/// +/// Laid out as Yuta Mori's `sais.c` lays it out, with nothing beside `sa` but +/// the bucket counters and a bit per position: the reduced string is gathered +/// at the back of `sa` and sorted recursively into the front, so no level +/// holds a copy of its input or of the array. +fn sa_is(s: &[T], sa: &mut [u32], upper: usize) { let n = s.len(); + debug_assert_eq!(sa.len(), n); match n { - 0 => return Vec::new(), - 1 => return vec![0], - 2 => return if s[0] < s[1] { vec![0, 1] } else { vec![1, 0] }, + 0 => return, + 1 => { + sa[0] = 0; + return; + } + 2 => { + sa.copy_from_slice(if s[0] < s[1] { &[0, 1] } else { &[1, 0] }); + return; + } _ => {} } if n < NAIVE_THRESHOLD { - let mut sa: Vec = (0..n as u32).collect(); + for (at, slot) in sa.iter_mut().enumerate() { + *slot = at as u32; + } sa.sort_unstable_by(|&a, &b| s[a as usize..].cmp(&s[b as usize..])); - return sa; + return; } - // `ls[i]`: the suffix at `i` is S-type (smaller than the one after it). - // The last is L-type against the virtual sentinel. - let mut ls = vec![false; n]; - for i in (0..n - 1).rev() { - ls[i] = if s[i] == s[i + 1] { - ls[i + 1] - } else { - s[i] < s[i + 1] - }; + let types = Types::new(s); + let mut counts = vec![0u32; upper + 1]; + for &c in s { + counts[c.index()] += 1; } - // Bucket starts: `sum_l[c]` where the L-type suffixes of `c` begin, - // `sum_s[c]` where its S-type ones do. - let mut sum_l = vec![0u32; upper + 1]; - let mut sum_s = vec![0u32; upper + 1]; + let mut buf = vec![0u32; upper + 1]; + + // Stage 1: the LMS positions at their buckets' ends, in any order, induce + // the order of the LMS substrings. + sa.fill(0); + bucket_ends(&counts, &mut buf); + let mut m = 0; + for p in (1..n).rev() { + if types.is_lms(p, n) { + let c = s[p].index(); + buf[c] -= 1; + sa[buf[c] as usize] = p as u32; + m += 1; + } + } + induce(s, sa, &counts, &mut buf); + if m == 0 { + // No LMS suffix: the induction from the last suffix alone is the + // whole order. + return; + } + + // Name each LMS substring by rank, equal substrings sharing a name. The + // sorted LMS positions are compacted to the front, each one's name goes + // to `m + p / 2` (LMS positions are at least two apart, so the slots are + // distinct and lie past the first `m`), then the names are gathered in + // text order at the back. A substring runs from its LMS position to the + // next one, or to the end of the text. + let mut k = 0; for i in 0..n { - if ls[i] { - // An S-type suffix is followed by a larger symbol, so its own is - // never the largest and the next bucket exists. - debug_assert!(s[i].index() < upper); - sum_l[s[i].index() + 1] += 1; - } else { - sum_s[s[i].index()] += 1; + let v = sa[i] as usize; + if types.is_lms(v, n) { + sa[k] = v as u32; + k += 1; } } - for c in 0..=upper { - sum_s[c] += sum_l[c]; - if c < upper { - sum_l[c + 1] += sum_s[c]; + debug_assert_eq!(k, m); + sa[m..].fill(EMPTY); + let substring_end = |p: usize| { + let mut end = p + 1; + while end < n && !types.is_lms(end, n) { + end += 1; + } + end + }; + let mut name = 0u32; + let mut prev = sa[0] as usize; + let mut prev_end = substring_end(prev); + debug_assert!(m + prev / 2 < n); + sa[m + prev / 2] = 0; + for i in 1..m { + let cur = sa[i] as usize; + let cur_end = substring_end(cur); + // Equal when as long and equal symbol for symbol, the symbol after + // included. One that runs into the end of the text ends at the virtual + // sentinel, which nothing else reaches, so it is unique. + let same = cur_end - cur == prev_end - prev + && cur_end < n + && prev_end < n + && s[cur..=cur_end] == s[prev..=prev_end]; + if !same { + name += 1; } + debug_assert!(m + cur / 2 < n); + sa[m + cur / 2] = name; + prev = cur; + prev_end = cur_end; } - - // Bucket ends: one past the last slot of each symbol's bucket. - let ends: Vec = (0..=upper) - .map(|c| if c < upper { sum_l[c + 1] } else { n as u32 }) - .collect(); - let mut sa = vec![0u32; n]; - let mut buf = vec![0u32; upper + 1]; - // The induction sweeps are the whole cost of the construction: one random - // write per suffix. They follow Yuta Mori's `sais.c` (`induceSA`): whether - // a suffix's predecessor is to be induced in the sweep that reads it is - // decided when the suffix is written, from the symbol before it, which is - // next to the one just read, and kept as the complement of the position - // (`!j`, the top bit set). The L sweep complements every entry it reads, - // which turns exactly the entries whose predecessor is S-type into the - // live ones for the S sweep; the S sweep restores the rest. The bucket - // cursor stays in a register while the symbol does not change. - // - // Every index below is in bounds by the bucket layout: a position is below - // `n`, a symbol lies in `0..=upper`, and each cursor stays inside the - // bucket its symbol's suffixes fill. - let live = |v: u32| (v as i32) > 0; - let induce = |sa: &mut [u32], buf: &mut [u32], lms: &[u32]| { - sa.fill(0); - buf.copy_from_slice(&sum_s); - for &d in lms { - let d = d as usize; - if d == n { - continue; - } - // An LMS suffix's predecessor is L-type: live for the L sweep. - let c = s[d].index(); - sa[buf[c] as usize] = d as u32; - buf[c] += 1; + let mut j = n; + for i in (m..n).rev() { + if sa[i] != EMPTY { + j -= 1; + sa[j] = sa[i]; } + } + debug_assert_eq!(j, n - m); - buf.copy_from_slice(&sum_l); - let last = n - 1; - let mut c1 = s[last].index(); - let mut b = buf[c1] as usize; - sa[b] = if s[last - 1] < s[last] { - !(last as u32) - } else { - last as u32 - }; - b += 1; - for i in 0..n { - // SAFETY: `i < n == sa.len()`. - let v = unsafe { *sa.get_unchecked(i) }; - unsafe { *sa.get_unchecked_mut(i) = !v }; - if live(v) { - let j = v as usize - 1; - // SAFETY: `j < n == s.len()`. - let c0 = unsafe { s.get_unchecked(j) }.index(); - if c0 != c1 { - debug_assert!(c0 < buf.len() && c1 < buf.len()); - // SAFETY: both symbols lie in `0..=upper`. - unsafe { - *buf.get_unchecked_mut(c1) = b as u32; - b = *buf.get_unchecked(c0) as usize; - } - c1 = c0; - } - // `j` is L-type; its predecessor is live for this sweep when it - // is L-type as well, which here means not smaller. - let dead = j > 0 && unsafe { s.get_unchecked(j - 1) }.index() < c1; - debug_assert!(b < n); - // SAFETY: `b` stays inside bucket `c1`. - unsafe { *sa.get_unchecked_mut(b) = if dead { !(j as u32) } else { j as u32 } }; - b += 1; + // Stage 2: sort the reduced string into the front of `sa`. With every + // name distinct its order is the names themselves, and no recursion is + // needed. + { + // `m <= n / 2`, so the front `m` slots and the back `m` are disjoint. + let (front, reduced) = sa.split_at_mut(n - m); + let order = &mut front[..m]; + if name as usize + 1 == m { + for (at, &rank) in reduced.iter().enumerate() { + order[rank as usize] = at as u32; } + } else { + sa_is(&*reduced, order, name as usize); } - - buf.copy_from_slice(&ends); - let mut c1 = 0usize; - let mut b = buf[0] as usize; - for i in (0..n).rev() { - // SAFETY: `i < n == sa.len()`. - let v = unsafe { *sa.get_unchecked(i) }; - if live(v) { - let j = v as usize - 1; - // SAFETY: `j < n == s.len()`. - let c0 = unsafe { s.get_unchecked(j) }.index(); - if c0 != c1 { - debug_assert!(c0 < buf.len() && c1 < buf.len()); - // SAFETY: both symbols lie in `0..=upper`. - unsafe { - *buf.get_unchecked_mut(c1) = b as u32; - b = *buf.get_unchecked(c0) as usize; - } - c1 = c0; - } - // `j` is S-type; its predecessor is live when it is S-type as - // well, which here means not larger. - let dead = j == 0 || unsafe { s.get_unchecked(j - 1) }.index() > c1; - debug_assert!(b > 0); - b -= 1; - // SAFETY: `b` stays inside bucket `c1`. - unsafe { *sa.get_unchecked_mut(b) = if dead { !(j as u32) } else { j as u32 } }; - } else { - unsafe { *sa.get_unchecked_mut(i) = !v }; + // The reduced string is done with: its slots take the LMS positions + // in text order, which turn the sorted indices into positions. + let mut at = 0; + for p in 1..n { + if types.is_lms(p, n) { + reduced[at] = p as u32; + at += 1; } } - }; - - // The leftmost S-type positions, in text order. - let is_lms = |p: usize| p > 0 && p < n && ls[p] && !ls[p - 1]; - let lms: Vec = (1..n).filter(|&i| is_lms(i)).map(|i| i as u32).collect(); - let m = lms.len(); - - induce(&mut sa, &mut buf, &lms); + debug_assert_eq!(at, m); + for slot in order.iter_mut() { + *slot = reduced[*slot as usize]; + } + } - if m > 0 { - // Name each LMS substring by rank, equal substrings sharing a name, - // and sort the string of names recursively. Laid out in `sa` itself as - // `sais.c` lays it out: the sorted LMS positions compacted to the - // front, each one's name at `m + p / 2` (LMS positions are at least two - // apart, so the slots are distinct and lie past the first `m`), then - // the names gathered in text order at the back. A substring runs from - // its LMS position to the next one, or to the end of the text. - let mut k = 0; - for i in 0..n { - let v = sa[i] as usize; - if is_lms(v) { - sa[k] = v as u32; - k += 1; - } + // Stage 3: the sorted LMS positions to their buckets' ends, keeping their + // order, from the back so that no entry is overwritten before it is read + // (`sais.c`, `sais_main` stage 3). A bucket's end is at least the number + // of LMS positions of its symbol and below, so the write cursor never + // passes the read one. + bucket_ends(&counts, &mut buf); + let mut i = m; + let mut j = n; + while i > 0 { + let mut p = sa[i - 1]; + let c = s[p as usize].index(); + let end = buf[c] as usize; + while end < j { + j -= 1; + sa[j] = 0; } - debug_assert_eq!(k, m); - sa[m..].fill(EMPTY); - let substring_end = |p: usize| { - let mut end = p + 1; - while end < n && !is_lms(end) { - end += 1; + loop { + debug_assert!(j >= i); + j -= 1; + sa[j] = p; + i -= 1; + if i == 0 { + break; } - end - }; - let mut name = 0u32; - let mut prev = sa[0] as usize; - let mut prev_end = substring_end(prev); - debug_assert!(m + prev / 2 < n); - sa[m + prev / 2] = 0; - for i in 1..m { - let cur = sa[i] as usize; - let cur_end = substring_end(cur); - // Equal when as long and equal symbol for symbol, the symbol after - // included. One that runs into the end of the text ends at the - // virtual sentinel, which nothing else reaches, so it is unique. - let same = cur_end - cur == prev_end - prev - && cur_end < n - && prev_end < n - && s[cur..=cur_end] == s[prev..=prev_end]; - if !same { - name += 1; + p = sa[i - 1]; + if s[p as usize].index() != c { + break; } - debug_assert!(m + cur / 2 < n); - sa[m + cur / 2] = name; - prev = cur; - prev_end = cur_end; } - let mut j = n; - for i in (m..n).rev() { - if sa[i] != EMPTY { - j -= 1; - sa[j] = sa[i]; + } + sa[..j].fill(0); + induce(s, sa, &counts, &mut buf); +} + +/// The two induction sweeps from the LMS positions already in `sa` (every +/// other slot zero): the L-type suffixes left to right from the bucket +/// starts, then the S-type ones right to left from the bucket ends. +/// +/// The sweeps are the whole cost of the construction: one random write per +/// suffix. They follow Yuta Mori's `sais.c` (`induceSA`): whether a suffix's +/// predecessor is to be induced in the sweep that reads it is decided when the +/// suffix is written, from the symbol before it, which is next to the one just +/// read, and kept as the complement of the position (`!j`, the top bit set). +/// The L sweep complements every entry it reads, which turns exactly the +/// entries whose predecessor is S-type into the live ones for the S sweep; the +/// S sweep restores the rest. The bucket cursor stays in a register while the +/// symbol does not change. +/// +/// Every index below is in bounds by the bucket layout: a position is below +/// `n`, a symbol lies in `0..=upper`, and each cursor stays inside the bucket +/// its symbol's suffixes fill. +fn induce(s: &[T], sa: &mut [u32], counts: &[u32], buf: &mut [u32]) { + let n = s.len(); + let live = |v: u32| (v as i32) > 0; + + bucket_starts(counts, buf); + let last = n - 1; + let mut c1 = s[last].index(); + let mut b = buf[c1] as usize; + sa[b] = if s[last - 1] < s[last] { + !(last as u32) + } else { + last as u32 + }; + b += 1; + for i in 0..n { + // SAFETY: `i < n == sa.len()`. + let v = unsafe { *sa.get_unchecked(i) }; + unsafe { *sa.get_unchecked_mut(i) = !v }; + if live(v) { + let j = v as usize - 1; + // SAFETY: `j < n == s.len()`. + let c0 = unsafe { s.get_unchecked(j) }.index(); + if c0 != c1 { + debug_assert!(c0 < buf.len() && c1 < buf.len()); + // SAFETY: both symbols lie in `0..=upper`. + unsafe { + *buf.get_unchecked_mut(c1) = b as u32; + b = *buf.get_unchecked(c0) as usize; + } + c1 = c0; } + // `j` is L-type; its predecessor is live for this sweep when it is + // L-type as well, which here means not smaller. + let dead = j > 0 && unsafe { s.get_unchecked(j - 1) }.index() < c1; + debug_assert!(b < n); + // SAFETY: `b` stays inside bucket `c1`. + unsafe { *sa.get_unchecked_mut(b) = if dead { !(j as u32) } else { j as u32 } }; + b += 1; } - debug_assert_eq!(j, n - m); - let rec_s = sa[n - m..].to_vec(); + } - let rec_sa = sa_is(&rec_s, name as usize); - let sorted_lms: Vec = rec_sa.iter().map(|&rank| lms[rank as usize]).collect(); - induce(&mut sa, &mut buf, &sorted_lms); + bucket_ends(counts, buf); + let mut c1 = 0usize; + let mut b = buf[0] as usize; + for i in (0..n).rev() { + // SAFETY: `i < n == sa.len()`. + let v = unsafe { *sa.get_unchecked(i) }; + if live(v) { + let j = v as usize - 1; + // SAFETY: `j < n == s.len()`. + let c0 = unsafe { s.get_unchecked(j) }.index(); + if c0 != c1 { + debug_assert!(c0 < buf.len() && c1 < buf.len()); + // SAFETY: both symbols lie in `0..=upper`. + unsafe { + *buf.get_unchecked_mut(c1) = b as u32; + b = *buf.get_unchecked(c0) as usize; + } + c1 = c0; + } + // `j` is S-type; its predecessor is live when it is S-type as + // well, which here means not larger. + let dead = j == 0 || unsafe { s.get_unchecked(j - 1) }.index() > c1; + debug_assert!(b > 0); + b -= 1; + // SAFETY: `b` stays inside bucket `c1`. + unsafe { *sa.get_unchecked_mut(b) = if dead { !(j as u32) } else { j as u32 } }; + } else { + unsafe { *sa.get_unchecked_mut(i) = !v }; + } } - sa } #[cfg(test)] From bdb10fd567cc6a23b5c5ebcd9171eb12d3046ed1 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 20:39:36 +0300 Subject: [PATCH 27/36] perf(dictionary): prefetch ahead of the induction sweeps Each step of an induction sweep reads the text before the suffix it takes, at a position the order of the array makes random, and in the recursion also the cursor of that symbol's bucket, whose table has as many entries as there are names. Both miss the cache on a large corpus, and the sweeps are most of the construction. The sweeps now warm them ahead, as libsais does: the text for the entry 64 slots ahead, and, where the symbols are the recursion's names, the bucket cursor for the entry 32 ahead, whose symbol that earlier prefetch has brought in. The symbol read is clamped, so an entry not yet written or not live costs a wasted hint, never an out-of-range read. --train-legacy -B4096 over decodecorpus_files, runner1 (load 0.75), bench profile, three interleaved rounds pinned to one core: - wall time 8.33-8.54 s -> 7.78-7.88 s (zstd 1.5.7: 7.45-7.58 s) - max RSS unchanged (166 MB) - dictionary bytes identical (md5 857c43bb70a0abeaf5e1bc28e8479edd) Part of #128 --- zstd/src/dictionary/suffix_array.rs | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/zstd/src/dictionary/suffix_array.rs b/zstd/src/dictionary/suffix_array.rs index d309736ca..688c26904 100644 --- a/zstd/src/dictionary/suffix_array.rs +++ b/zstd/src/dictionary/suffix_array.rs @@ -19,10 +19,16 @@ const NAIVE_THRESHOLD: usize = 10; /// A symbol of the text being sorted: the bytes at the top level, the ranks of /// the reduced string in the recursion. trait Symbol: Copy + Ord { + /// Whether the bucket table can outgrow the cache: the recursion's + /// alphabet is its number of names. + const WIDE: bool; + fn index(self) -> usize; } impl Symbol for u8 { + const WIDE: bool = false; + #[inline] fn index(self) -> usize { usize::from(self) @@ -30,12 +36,46 @@ impl Symbol for u8 { } impl Symbol for u32 { + const WIDE: bool = true; + #[inline] fn index(self) -> usize { self as usize } } +/// How many entries ahead of the sweep its loads are prefetched. The text a +/// swept entry points into is prefetched twice as far ahead, so that the +/// symbol is in cache when the bucket cursor it selects is prefetched. +const PREFETCH_DISTANCE: usize = 32; + +/// Warm the loads the sweep makes when it reaches entries `near` and `far`: +/// the text before the suffix at `far` and, where the bucket table is large, +/// the cursor of the bucket the suffix at `near` selects. An index past the +/// array (a sweep's wrapped subtraction included) warms nothing. A hint only: +/// entries not yet written, or not live, aim the text prefetch anywhere and +/// read a clamped symbol. +#[inline(always)] +fn prefetch_ahead(s: &[T], sa: &[u32], buf: &[u32], near: usize, far: usize) { + let n = sa.len(); + if far < n { + // SAFETY: bounded by the test above. + let pos = unsafe { *sa.get_unchecked(far) } as usize; + crate::decoding::prefetch::prefetch_l1_at( + s.as_ptr().wrapping_add(pos.wrapping_sub(1)).cast(), + ); + } + if T::WIDE && near < n { + // SAFETY: bounded by the test above. + let near = unsafe { *sa.get_unchecked(near) } as usize; + // A dead entry (top bit set) or an empty one wraps past `n - 1`. + let at = near.wrapping_sub(1).min(n - 1); + // SAFETY: `at < n == s.len()`. + let c = unsafe { s.get_unchecked(at) }.index(); + crate::decoding::prefetch::prefetch_l1_at(buf.as_ptr().wrapping_add(c).cast()); + } +} + /// The suffix array of `text`: `sa[i]` is the start of the `i`-th smallest /// suffix. /// @@ -315,6 +355,7 @@ fn induce(s: &[T], sa: &mut [u32], counts: &[u32], buf: &mut [u32]) { }; b += 1; for i in 0..n { + prefetch_ahead(s, sa, buf, i + PREFETCH_DISTANCE, i + 2 * PREFETCH_DISTANCE); // SAFETY: `i < n == sa.len()`. let v = unsafe { *sa.get_unchecked(i) }; unsafe { *sa.get_unchecked_mut(i) = !v }; @@ -345,6 +386,13 @@ fn induce(s: &[T], sa: &mut [u32], counts: &[u32], buf: &mut [u32]) { let mut c1 = 0usize; let mut b = buf[0] as usize; for i in (0..n).rev() { + prefetch_ahead( + s, + sa, + buf, + i.wrapping_sub(PREFETCH_DISTANCE), + i.wrapping_sub(2 * PREFETCH_DISTANCE), + ); // SAFETY: `i < n == sa.len()`. let v = unsafe { *sa.get_unchecked(i) }; if live(v) { From 31438d5feeb70ad0ed6361fa78e38a5e8eda5bed Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 20:50:44 +0300 Subject: [PATCH 28/36] perf(dictionary): prefetch ahead of the legacy analysis The legacy trainer analyses every text position in order, and each analysis starts from the position's rank, which is random: the suffix array around that rank and the text of the two neighbours there miss the cache every time. Those loads were most of the trainer's time, as they are of the reference's. The loop now warms them ahead of reaching a position: the array line around the rank of the position 16 on, and the neighbours' text for the position 8 on, whose array line that earlier hint has brought in. Positions the loop then skips cost a wasted hint. --train-legacy -B4096 over decodecorpus_files, runner1 (load 0.75), bench profile, three interleaved rounds pinned to one core: - wall time 7.77-7.92 s -> 7.29-7.38 s - zstd 1.5.7 in the same session: 7.35-7.39 s against our 7.22-7.35 s - dictionary bytes identical (md5 857c43bb70a0abeaf5e1bc28e8479edd) - distances of 32 and 16 measured the same as 16 and 8 Part of #128 --- zstd/src/dictionary/legacy.rs | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index f388525e3..7f498e621 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -197,6 +197,14 @@ impl<'a> Corpus<'a> { } } +/// How many text positions ahead the analysis warms the suffix-array line +/// around a position's rank. +const FAR_AHEAD: usize = 16; + +/// How many text positions ahead the analysis warms the text of a position's +/// two neighbours in suffix order. +const NEAR_AHEAD: usize = 8; + /// The suffix array with one extra slot on each side, both pointing into the /// noise band, as the reference lays it out (`suffix0[0]` and /// `suffix[bufferSize]`): a walk off either end compares against noise and @@ -227,6 +235,36 @@ impl Suffixes { None => self.noise, } } + + /// Warm what the analysis of text position `cursor` will read, ahead of + /// reaching it: its rank is random, so the array around that rank and the + /// text of the two neighbours there miss the cache every time. The array + /// line is warmed for the position [`FAR_AHEAD`] on, the neighbours' text + /// for the one [`NEAR_AHEAD`] on, whose array line that earlier hint has + /// brought in. A hint only; ranks at either end read a clamped slot. + #[inline(always)] + fn prefetch_for(&self, rank: &[u32], samples: &[u8], cursor: usize) { + use crate::decoding::prefetch::prefetch_l1_at; + let len = self.sa.len(); + if let Some(&far) = rank.get(cursor + FAR_AHEAD) { + prefetch_l1_at(self.sa.as_ptr().wrapping_add(far as usize).cast()); + } + if let Some(&near) = rank.get(cursor + NEAR_AHEAD) { + let near = near as usize; + // Rank 0 wraps past the end and clamps with the top rank. + let below = near.wrapping_sub(1).min(len - 1); + let above = (near + 1).min(len - 1); + // SAFETY: both are clamped below `len == self.sa.len()`. + let (below, above) = unsafe { + ( + *self.sa.get_unchecked(below) as usize, + *self.sa.get_unchecked(above) as usize, + ) + }; + prefetch_l1_at(samples.as_ptr().wrapping_add(below)); + prefetch_l1_at(samples.as_ptr().wrapping_add(above)); + } + } } /// Train dictionary content from `samples`, the concatenation of samples whose @@ -328,6 +366,7 @@ fn find_segments(list: &mut [DictItem], corpus: &Corpus<'_>, len: usize, min_rep let mut cursor = 0usize; while cursor < len { + suffixes.prefetch_for(&rank, corpus.samples, cursor); if done[cursor] { cursor += 1; continue; From 1e5ee304e6f97f5730c294176f33bd4648391fa6 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 21:06:50 +0300 Subject: [PATCH 29/36] perf(dictionary): keep the recursion's bucket tables in the array Each level of the suffix-array recursion allocated its two bucket tables, as long as its alphabet, which there is the number of names: tens of megabytes on a large corpus, allocated and freed before the analysis starts. glibc returns such blocks to the system; macOS keeps them resident, so they stayed in the trainer's peak there. The tables now go in the free slots between a level's reduced string and its sorted front, as sais.c places them: a level reduced to m symbols leaves n - 2m slots there. A level whose tables do not fit still allocates them; the top level, with 256 symbols and no free slots, always does. --train-legacy -B4096 over decodecorpus_files, dictionary bytes identical (md5 857c43bb70a0abeaf5e1bc28e8479edd) on both hosts: - M1: max RSS 203 MB -> 184 MB (zstd 1.5.7: 186 MB) - runner1: max RSS unchanged (166 MB), wall time 7.24-7.30 s -> 7.19-7.22 s, three interleaved rounds, below what a rebuild resolves Part of #128 --- zstd/src/dictionary/suffix_array.rs | 51 ++++++++++++++++++----------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/zstd/src/dictionary/suffix_array.rs b/zstd/src/dictionary/suffix_array.rs index 688c26904..69a2b2a4b 100644 --- a/zstd/src/dictionary/suffix_array.rs +++ b/zstd/src/dictionary/suffix_array.rs @@ -147,17 +147,20 @@ fn bucket_ends(counts: &[u32], buf: &mut [u32]) { } } -/// SA-IS over `s` into `sa` (as long as `s`), whose symbols all lie in -/// `0..=upper`. The text is taken to end in a virtual sentinel smaller than -/// every symbol. +/// SA-IS over `s` into the first `s.len()` slots of `space`, whose symbols all +/// lie in `0..=upper`. The text is taken to end in a virtual sentinel smaller +/// than every symbol. Slots of `space` past the array are free for the +/// construction to use. /// -/// Laid out as Yuta Mori's `sais.c` lays it out, with nothing beside `sa` but -/// the bucket counters and a bit per position: the reduced string is gathered -/// at the back of `sa` and sorted recursively into the front, so no level -/// holds a copy of its input or of the array. -fn sa_is(s: &[T], sa: &mut [u32], upper: usize) { +/// Laid out as Yuta Mori's `sais.c` lays it out, with nothing beside the array +/// but the bucket tables and a bit per position: the reduced string is +/// gathered at the back of the array and sorted recursively into the front, so +/// no level holds a copy of its input or of the array. The bucket tables go in +/// the free slots when they fit, which in the recursion they usually do: a +/// level reduced to `m` symbols leaves `n - 2m` slots between the two. +fn sa_is(s: &[T], space: &mut [u32], upper: usize) { let n = s.len(); - debug_assert_eq!(sa.len(), n); + let (sa, free) = space.split_at_mut(n); match n { 0 => return, 1 => { @@ -179,16 +182,25 @@ fn sa_is(s: &[T], sa: &mut [u32], upper: usize) { } let types = Types::new(s); - let mut counts = vec![0u32; upper + 1]; + let k = upper + 1; + let mut owned: Vec; + let (counts, buf) = if free.len() >= 2 * k { + let (counts, rest) = free.split_at_mut(k); + counts.fill(0); + (counts, &mut rest[..k]) + } else { + owned = vec![0u32; 2 * k]; + owned.split_at_mut(k) + }; for &c in s { counts[c.index()] += 1; } - let mut buf = vec![0u32; upper + 1]; + let counts = &*counts; // Stage 1: the LMS positions at their buckets' ends, in any order, induce // the order of the LMS substrings. sa.fill(0); - bucket_ends(&counts, &mut buf); + bucket_ends(counts, buf); let mut m = 0; for p in (1..n).rev() { if types.is_lms(p, n) { @@ -198,7 +210,7 @@ fn sa_is(s: &[T], sa: &mut [u32], upper: usize) { m += 1; } } - induce(s, sa, &counts, &mut buf); + induce(s, sa, counts, buf); if m == 0 { // No LMS suffix: the induction from the last suffix alone is the // whole order. @@ -264,16 +276,17 @@ fn sa_is(s: &[T], sa: &mut [u32], upper: usize) { // name distinct its order is the names themselves, and no recursion is // needed. { - // `m <= n / 2`, so the front `m` slots and the back `m` are disjoint. + // `m <= n / 2`, so the front `m` slots and the back `m` are disjoint, + // and the recursion has the `n - 2m` between them to spare. let (front, reduced) = sa.split_at_mut(n - m); - let order = &mut front[..m]; if name as usize + 1 == m { for (at, &rank) in reduced.iter().enumerate() { - order[rank as usize] = at as u32; + front[rank as usize] = at as u32; } } else { - sa_is(&*reduced, order, name as usize); + sa_is(&*reduced, front, name as usize); } + let order = &mut front[..m]; // The reduced string is done with: its slots take the LMS positions // in text order, which turn the sorted indices into positions. let mut at = 0; @@ -294,7 +307,7 @@ fn sa_is(s: &[T], sa: &mut [u32], upper: usize) { // (`sais.c`, `sais_main` stage 3). A bucket's end is at least the number // of LMS positions of its symbol and below, so the write cursor never // passes the read one. - bucket_ends(&counts, &mut buf); + bucket_ends(counts, buf); let mut i = m; let mut j = n; while i > 0 { @@ -320,7 +333,7 @@ fn sa_is(s: &[T], sa: &mut [u32], upper: usize) { } } sa[..j].fill(0); - induce(s, sa, &counts, &mut buf); + induce(s, sa, counts, buf); } /// The two induction sweeps from the LMS positions already in `sa` (every From 099259958e8e63c2cc6606d371fca90aba276416 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 21:07:07 +0300 Subject: [PATCH 30/36] perf(dictionary): compare legacy prefixes with the vector kernel The analysis measures how long the prefixes of suffix-order neighbours agree, and on a corpus with long repeats those comparisons run for many words: callgrind put the word loop at 48 of the trainer's 59 billion instructions on decodecorpus_files, against 40 billion for the whole of the reference's analysis. The loop tested its bound on every word, where the reference relies on the noise band instead. The comparison now goes through the crate's common-prefix kernel, resolved once per run (AVX2, SSE, NEON or scalar), for the part of both runs inside the samples; the band is still read a byte at a time past their end. --train-legacy -B4096 over decodecorpus_files, dictionary bytes identical (md5 857c43bb70a0abeaf5e1bc28e8479edd) on both hosts: - runner1 (AVX2, load 3.5 from another job), three interleaved rounds pinned to one core: 7.18-7.23 s -> 5.90-5.94 s (zstd 1.5.7: 7.35-7.39 s) - M1 (NEON), one round: 5.00-5.06 s -> 4.91 s, 60.4 -> 53.4 billion instructions, 12.17 -> 11.59 billion cycles (zstd: 4.35-4.67 s) Part of #128 --- zstd/src/dictionary/legacy.rs | 49 ++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index 7f498e621..3e37b44f5 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -13,6 +13,9 @@ use alloc::vec; use alloc::vec::Vec; use super::suffix_array::suffix_array; +use crate::encoding::fastpath::{ + FastpathKernel, dispatch_common_prefix_len_ptr_with_kernel, select_kernel, +}; /// Fewest repetitions that make a prefix a candidate (`MINRATIO`). const MIN_RATIO: u32 = 4; @@ -80,6 +83,8 @@ fn noise_band() -> [u8; NOISE_LENGTH] { struct Corpus<'a> { samples: &'a [u8], noise: [u8; NOISE_LENGTH], + /// The compare kernel for this CPU, resolved once for the whole run. + kernel: FastpathKernel, } impl<'a> Corpus<'a> { @@ -87,6 +92,7 @@ impl<'a> Corpus<'a> { Self { samples, noise: noise_band(), + kernel: select_kernel(), } } @@ -135,42 +141,37 @@ impl<'a> Corpus<'a> { self.read::<8>(at).map(u64::from_le_bytes) } - /// Bytes `a` and `b` have in common (`ZDICT_count`), compared a word at - /// a time as the reference compares them while both words lie in the - /// samples, then a byte at a time across into the band. + /// Bytes `a` and `b` have in common (`ZDICT_count`): through the CPU's + /// vector compare while both runs lie in the samples, where the corpus' + /// long repeats make the comparison most of the analysis, then a byte at + /// a time across into the band. #[inline] fn common(&self, a: usize, b: usize) -> usize { let samples = self.samples; - // A comparison that starts in the band has no word to read in the + // A comparison that starts in the band has nothing to compare in the // samples. let Some(in_samples) = samples.len().checked_sub(a.max(b)) else { return self.common_tail(a, b, 0); }; let base = samples.as_ptr(); - let mut n = 0; - while n + 8 <= in_samples { - // SAFETY: `a, b <= max(a, b)` and `n + 8 <= samples.len() - - // max(a, b)`, so both eight-byte reads end inside the samples. - // Read unchecked: through a slice index the bound is tested on - // every word, since the loop limit does not tell the optimiser - // that each index stays inside. - let (x, y) = unsafe { - ( - base.add(a + n).cast::().read_unaligned(), - base.add(b + n).cast::().read_unaligned(), - ) - }; - let diff = u64::from_le(x) ^ u64::from_le(y); - if diff != 0 { - return n + (diff.trailing_zeros() / 8) as usize; - } - n += 8; + // SAFETY: `a, b <= max(a, b)` and `in_samples == samples.len() - + // max(a, b)`, so both runs of `in_samples` bytes lie in the samples. + let n = unsafe { + dispatch_common_prefix_len_ptr_with_kernel( + self.kernel, + base.add(a), + base.add(b), + in_samples, + ) + }; + if n < in_samples { + return n; } self.common_tail(a, b, n) } - /// The last bytes of [`Self::common`] from `n` on, fewer than a word of - /// them in the samples, then on into the band. + /// The bytes of [`Self::common`] from `n` on, where the shorter run has + /// reached the end of the samples: on into the band. #[cold] #[inline(never)] fn common_tail(&self, a: usize, b: usize, mut n: usize) -> usize { From 6d09f0378747a866973322367ef39e05605aec78 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 21:14:28 +0300 Subject: [PATCH 31/36] perf(encoding): compare 32 bytes a step in the NEON prefix kernel The NEON common-prefix loop took 16 bytes a step and tested each eight-byte half of the equality mask with its own lane move, compare and branch, which made it little cheaper per byte than the scalar word loop. Once the legacy dictionary trainer compared its long repeats through it, M1 ran 53 billion instructions to the reference's 31. A step now compares 32 bytes and branches once: the two equality masks are joined and their minimum lane tested. The mismatch is located only on the step that has one, from a nibble-per-byte mask narrowed out of the equality result. A lone 16-byte step handles what the loop leaves before the scalar tail, so the result is the same byte count as before on every input. Carries neon_prefix_len_matches_scalar_at_every_mismatch_position, which checks every length up to 100 and every mismatch position in it against the scalar kernel. M1, --train-legacy -B4096 over decodecorpus_files, two interleaved rounds, dictionary bytes identical: - 53.3 -> 38.5 billion instructions, 11.62-11.75 -> 10.19-10.24 billion cycles (zstd 1.5.7: 10.6-10.8 billion), 4.96-5.98 s -> 4.19-4.41 s M1, encode_loop_z000033 level 19, 4 frames, three interleaved rounds, same output: 4.17 -> 4.15 billion instructions, 1.594-1.624 -> 1.548- 1.580 billion cycles. Part of #128 --- zstd/src/encoding/fastpath/neon.rs | 59 +++++++++++++++++------- zstd/src/encoding/fastpath/neon/tests.rs | 20 ++++++++ 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/zstd/src/encoding/fastpath/neon.rs b/zstd/src/encoding/fastpath/neon.rs index f93fa327b..a9be64b83 100644 --- a/zstd/src/encoding/fastpath/neon.rs +++ b/zstd/src/encoding/fastpath/neon.rs @@ -9,13 +9,33 @@ #![cfg(all(target_arch = "aarch64", target_endian = "little"))] -use core::arch::aarch64::{uint8x16_t, vceqq_u8, vgetq_lane_u64, vld1q_u8, vreinterpretq_u64_u8}; +use core::arch::aarch64::{ + uint8x16_t, vandq_u8, vceqq_u8, vget_lane_u64, vld1q_u8, vminvq_u8, vreinterpret_u64_u8, + vreinterpretq_u16_u8, vshrn_n_u16, +}; use super::scalar; -/// 16-byte NEON vector prefix-length probe. Returns the number of leading -/// equal bytes that fit in whole 16-byte chunks; the caller (or the wrapper -/// below) handles the scalar tail. +/// Index of the first unequal byte of a 16-byte `vceqq_u8` result that has +/// one. NEON has no byte mask move: narrowing each 16-bit lane by four keeps a +/// nibble per byte, in order, so the first zero nibble is the first mismatch. +#[target_feature(enable = "neon")] +#[inline] +fn first_unequal(eq: uint8x16_t) -> usize { + let nibbles = vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(eq), 4)), + 0, + ); + ((!nibbles).trailing_zeros() / 4) as usize +} + +/// NEON vector prefix-length probe. Returns the number of leading equal bytes +/// that fit in whole 16-byte chunks; the caller (or the wrapper below) handles +/// the scalar tail. +/// +/// Compares 32 bytes a step with one branch: the two equality masks are +/// joined and their minimum lane tested, and the mismatch is only located on +/// the step that has one. /// /// # Safety /// `lhs` / `rhs` must point to at least `max` initialized bytes. NEON must be @@ -25,20 +45,25 @@ use super::scalar; #[inline] pub(crate) unsafe fn prefix_len_simd(lhs: *const u8, rhs: *const u8, max: usize) -> usize { let mut off = 0usize; - while off + 16 <= max { - let a: uint8x16_t = unsafe { vld1q_u8(lhs.add(off)) }; - let b: uint8x16_t = unsafe { vld1q_u8(rhs.add(off)) }; - let eq = vceqq_u8(a, b); - let lanes = vreinterpretq_u64_u8(eq); - let low = vgetq_lane_u64(lanes, 0); - if low != u64::MAX { - let diff = low ^ u64::MAX; - return off + scalar::mismatch_byte_index(diff as usize); + while off + 32 <= max { + let (eq0, eq1) = unsafe { + ( + vceqq_u8(vld1q_u8(lhs.add(off)), vld1q_u8(rhs.add(off))), + vceqq_u8(vld1q_u8(lhs.add(off + 16)), vld1q_u8(rhs.add(off + 16))), + ) + }; + if vminvq_u8(vandq_u8(eq0, eq1)) != u8::MAX { + if vminvq_u8(eq0) != u8::MAX { + return off + first_unequal(eq0); + } + return off + 16 + first_unequal(eq1); } - let high = vgetq_lane_u64(lanes, 1); - if high != u64::MAX { - let diff = high ^ u64::MAX; - return off + 8 + scalar::mismatch_byte_index(diff as usize); + off += 32; + } + if off + 16 <= max { + let eq = unsafe { vceqq_u8(vld1q_u8(lhs.add(off)), vld1q_u8(rhs.add(off))) }; + if vminvq_u8(eq) != u8::MAX { + return off + first_unequal(eq); } off += 16; } diff --git a/zstd/src/encoding/fastpath/neon/tests.rs b/zstd/src/encoding/fastpath/neon/tests.rs index 30f99935e..6cb0e77be 100644 --- a/zstd/src/encoding/fastpath/neon/tests.rs +++ b/zstd/src/encoding/fastpath/neon/tests.rs @@ -14,6 +14,26 @@ fn neon_prefix_len_matches_scalar_on_long_run() { assert_eq!(neon, 25); } +/// Every length up to three 32-byte steps and every mismatch position in it, +/// or none: each half of a step, the lone 16-byte step after the loop and the +/// scalar tail all have to find the first unequal byte the scalar kernel finds. +#[test] +fn neon_prefix_len_matches_scalar_at_every_mismatch_position() { + let a: Vec = (0..100u8).map(|i| i.wrapping_mul(37)).collect(); + for max in 0..=a.len() { + for mismatch in (0..max).map(Some).chain([None]) { + let mut b = a.clone(); + if let Some(at) = mismatch { + b[at] ^= 0x5A; + } + let neon = unsafe { common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; + let scl = unsafe { scalar::common_prefix_len_ptr(a.as_ptr(), b.as_ptr(), max) }; + assert_eq!(neon, scl, "max {max}, mismatch at {mismatch:?}"); + assert_eq!(neon, mismatch.unwrap_or(max)); + } + } +} + #[test] fn neon_handles_short_input() { let a = b"abc"; From 5e5c24c4c5907c15dfff18f7efcb121d66202ddc Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 27 Sep 2026 00:06:33 +0300 Subject: [PATCH 32/36] fix: benchmark input checks, training sizes, probe table handles - `--max` tests assumed a 64-bit target; on 32-bit the command refuses it, as the reference does, so those tests are 64-bit only and a 32-bit test checks the refusal. - A benchmark input that shrank between being sized and being read was accepted short, and the frames, cut at the recorded sizes, panicked in `split_at`. Any change of size is now an error. Carries a_benchmark_input_that_shrank_is_refused. - Training file sizes under `-B` were summed unchecked, so sparse files reporting huge lengths wrapped the totals into a small training set. The sums stop at the top of the range, which the loader's cap then cuts. Carries training_sizes_past_the_integer_range_do_not_wrap. - Under `-M` the benchmark listed every frame just to weigh them before the limit could refuse the run; the room and the widest frame are now worked out per file from its whole blocks and its tail. Carries the_frame_extent_matches_the_frames_it_describes. - The legacy trainer's measuring walk gets its own noise-slot test, which hangs with that walk's bound removed. - Split probes no longer take three FSE table handles each: the tables a probe may repeat are borrowed from the state it starts from, and a handle is taken only for a repeated table the next state keeps. The arena for the probes' Huffman tables is kept across blocks instead of allocated per block. The optimal parser's segment loop bounds itself once per block. Frames are byte-identical at L1-L22 on z000033[..200000], z000033, z000033 with dict_tests/dictionary, and --long at L16-L22 (73 checks). runner1, 50 frames of z000033[..200000], interleaved: L16 1785-1808 -> 1768-1792 ms, L19 2640-2693 -> 2647-2680 ms, control L3 219-224 -> 219-221 ms, within what a rebuild resolves; the segment loop bound alone L16 1762-1792 -> 1759-1761 ms with L3 moving the same. Fewer operations, no measurable time change. Part of #128 --- zstd/src/bin/structured-zstd/main.rs | 111 +++++++++----- zstd/src/bin/structured-zstd/tests.rs | 64 +++++++- zstd/src/dictionary/legacy/tests.rs | 24 +++ zstd/src/encoding/blocks/compressed.rs | 199 ++++++++++++++++--------- zstd/src/encoding/hc/optimal.rs | 5 +- 5 files changed, 290 insertions(+), 113 deletions(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index fb0a08474..cee7b6c17 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -2802,17 +2802,17 @@ fn run_benchmark(opts: &Options, dict: Option>) -> Result<()> { &sizes }; let inputs = if opts.bench_separately { largest } else { sum }; - let chunks = bench_chunk_lengths(subject, opts.block_size); // Three buffers exist at once: the input, the frames it compresses to, // and the decoded copy. Each is allocated at the size named here and // never grows past it, so this is what the run actually holds rather // than a lower bound on it. The frames' is `compress_bound` of each, // which is the input plus the framing an incompressible input still - // pays — the case a ceiling has to survive. - let frame = bench_frames_bound(&chunks); - // The encoder is sized by the frame it builds, and the frames are the - // chunks, so the widest chunk is the source it is weighed against. - let widest_chunk = chunks.iter().copied().max().unwrap_or(0); + // pays — the case a ceiling has to survive. The encoder is sized by the + // frame it builds, so the widest frame is the source it is weighed + // against. + let extent = bench_frames_extent(subject, opts.block_size); + let frame = extent.map(|(room, _)| room); + let widest_chunk = extent.map_or(0, |(_, widest)| widest); // Beside them stands the match finder every compression pass builds, // whose tables are the largest thing at the higher levels — hundreds of // MiB where the buffers are tens. It is sized by the level, by the @@ -2952,9 +2952,12 @@ fn read_inputs_bounded(inputs: &[PathBuf], sizes: &[u64]) -> Result> { file.take(size + 1) .read_to_end(&mut data) .wrap_err_with(|| format!("failed to read {}", input.display()))?; - if (data.len() - before) as u64 > *size { + // Short is refused as well as long: the frames are cut at the sizes + // recorded before the read, so a file that shrank would be cut past + // the end of what was read. + if (data.len() - before) as u64 != *size { bail!( - "{} grew while it was being read; run again", + "{} changed size while it was being read; run again", input.display() ); } @@ -2982,13 +2985,35 @@ fn bench_chunk_lengths(file_sizes: &[u64], block_size: Option) -> Vec chunks } -/// The room the frames of `chunks` can take at most: `compress_bound` of each. -/// `None` when that is more than this machine can address. -fn bench_frames_bound(chunks: &[u64]) -> Option { - chunks.iter().try_fold(0u64, |total, &chunk| { - let bound = structured_zstd::encoding::compress_bound(usize::try_from(chunk).ok()?); - total.checked_add(u64::try_from(bound).ok()?) - }) +/// The frames [`bench_chunk_lengths`] cuts `file_sizes` into, measured without +/// listing them: the room they take at most (`compress_bound` of each) and the +/// widest of them. Worked out per file from its whole blocks and its tail, so a +/// small `-B` over a large input costs nothing before `-M` has weighed it. +/// `None` when the room is more than this machine can address. +fn bench_frames_extent(file_sizes: &[u64], block_size: Option) -> Option<(u64, u64)> { + let block = block_size.filter(|&size| size >= MIN_BENCH_BLOCK_SIZE); + let bound = |len: u64| -> Option { + u64::try_from(structured_zstd::encoding::compress_bound( + usize::try_from(len).ok()?, + )) + .ok() + }; + let mut room = 0u64; + let mut widest = 0u64; + for &size in file_sizes { + if size == 0 { + continue; + } + let piece = block.unwrap_or(size); + let whole = size / piece; + let tail = size % piece; + room = room.checked_add(whole.checked_mul(bound(piece)?)?)?; + if tail > 0 { + room = room.checked_add(bound(tail)?)?; + } + widest = widest.max(size.min(piece)); + } + Some((room, widest)) } /// Measure one benchmark subject: every input together, or a single file under @@ -3041,8 +3066,8 @@ fn benchmark_one( // growing `Vec` ends up with — and it keeps the growth out of the timed // sections, which would otherwise be reported as compression and // decompression speed. - let frames_bound = bench_frames_bound(&chunks) - .and_then(|bound| usize::try_from(bound).ok()) + let frames_bound = bench_frames_extent(file_sizes, opts.block_size) + .and_then(|(room, _)| usize::try_from(room).ok()) .ok_or_else(|| eyre!("-b: {label} is more than this machine can hold compressed"))?; let mut compressed = Vec::with_capacity(frames_bound); let mut decoded = Vec::with_capacity(data.len()); @@ -3468,6 +3493,32 @@ fn shuffle_training_files(files: &mut [T]) { } } +/// How many samples `file_sizes` make and how many bytes they hold, as the +/// loader would take them without a limit: each file one sample of at most +/// [`TRAINING_SAMPLE_MAX`] bytes, or cut into `block_size` samples; empty files +/// left out. +fn training_extent(file_sizes: &[u64], block_size: Option) -> (u64, u64) { + // Both only ever meet a bound: `wanted` is cut to the trainer's cap and + // `samples` is compared with the minimum and the count loaded. A sum past + // the integer range (sparse files report any length) is therefore "more + // than any bound", which is what stopping at the top says; wrapping would + // turn it into a small number and a silently smaller training set. + let mut wanted = 0u64; + let mut samples = 0u64; + for &size in file_sizes { + if size == 0 { + continue; + } + let (count, bytes) = match block_size { + Some(block) => (size.div_ceil(block), size), + None => (1, size.min(TRAINING_SAMPLE_MAX)), + }; + samples = samples.saturating_add(count); + wanted = wanted.saturating_add(bytes); + } + (samples, wanted) +} + /// Load the training samples as the reference's command does: files in its /// shuffled order, each one sample of at most [`TRAINING_SAMPLE_MAX`] bytes, or /// cut whole into `block_size` samples when `-B` gives one; empty files left @@ -3480,27 +3531,15 @@ fn load_training_samples( let mut order: Vec<&PathBuf> = inputs.iter().collect(); shuffle_training_files(&mut order); - // What would be loaded without a limit, and as how many samples. - let mut wanted = 0u64; - let mut samples = 0u64; + let mut file_sizes = Vec::with_capacity(order.len()); for input in &order { - let size = fs::metadata(input) - .wrap_err_with(|| format!("failed to inspect {}", input.display()))? - .len(); - if size == 0 { - continue; - } - match block_size { - Some(block) => { - samples += size.div_ceil(block); - wanted += size; - } - None => { - samples += 1; - wanted += size.min(TRAINING_SAMPLE_MAX); - } - } + file_sizes.push( + fs::metadata(input) + .wrap_err_with(|| format!("failed to inspect {}", input.display()))? + .len(), + ); } + let (samples, wanted) = training_extent(&file_sizes, block_size); if samples < TRAINING_SAMPLES_MIN as u64 { bail!( "{samples} training sample(s) is too few; provide one file per sample, or \ diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 693ebd088..e81a396e5 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -943,6 +943,55 @@ fn the_benchmark_input_is_the_size_it_was_counted_at() { assert_eq!(combined[8000], 2, "the second file's follow"); } +/// The `-M` check measures the benchmark's frames without listing them; what it +/// works out has to be what the list itself adds up to, for whole blocks, +/// tails, empty inputs and block sizes below the minimum alike. +#[test] +fn the_frame_extent_matches_the_frames_it_describes() { + let files: [&[u64]; 4] = [&[], &[0, 1, 5000], &[1 << 20, 3], &[70_000, 0, 131_071]]; + for sizes in files { + for block in [None, Some(1024), Some(MIN_BENCH_BLOCK_SIZE), Some(40_000)] { + let chunks = bench_chunk_lengths(sizes, block); + let room: u64 = chunks + .iter() + .map(|&len| structured_zstd::encoding::compress_bound(len as usize) as u64) + .sum(); + let widest = chunks.iter().copied().max().unwrap_or(0); + assert_eq!( + bench_frames_extent(sizes, block), + Some((room, widest)), + "{sizes:?} in blocks of {block:?}" + ); + } + } +} + +/// File sizes that add up past `u64::MAX` (sparse files report any length) do +/// not wrap: the total stays at the most there is, which the loader's cap then +/// cuts to what it trains on, and the sample count is exact. +#[test] +fn training_sizes_past_the_integer_range_do_not_wrap() { + let huge = u64::MAX / 2 + 1; + let (samples, wanted) = training_extent(&[huge, huge, 0], Some(4096)); + assert_eq!(wanted, u64::MAX); + assert_eq!(samples, 2 * huge.div_ceil(4096)); +} + +/// A benchmark input that shrank between being sized and being read is +/// refused. The frames are cut at the sizes recorded first, so a short read +/// would cut past the end of the buffer instead of measuring anything. +#[test] +fn a_benchmark_input_that_shrank_is_refused() { + let input = std::env::temp_dir().join(format!("szstd-benchshrink-{}", std::process::id())); + fs::write(&input, vec![7u8; 1000]).unwrap(); + + let read = read_inputs_bounded(std::slice::from_ref(&input), &[4000]); + + let _ = fs::remove_file(&input); + let err = read.expect_err("a file shorter than its recorded size is an error"); + assert!(err.to_string().contains("changed"), "{err}"); +} + /// Permission bits alone do not say who they let in. Two samples at `0640` may /// belong to different groups, and the dictionary belongs to whichever group the /// directory it was created in gave it — so keeping the group bits would open @@ -2022,8 +2071,8 @@ fn benchmark_frames_follow_the_inputs_and_the_block_size() { ); assert_eq!(bench_chunk_lengths(&[100], Some(32)), vec![32, 32, 32, 4]); assert_eq!( - bench_frames_bound(&[40, 40]), - Some(2 * structured_zstd::encoding::compress_bound(40) as u64), + bench_frames_extent(&[80], Some(40)), + Some((2 * structured_zstd::encoding::compress_bound(40) as u64, 40)), "each frame pays its own framing" ); } @@ -4348,6 +4397,7 @@ fn advanced_parameters_reach_the_frame() { /// decodes. It unlocks the ultra levels and long-distance matching, replaces a /// `--zstd=` list given before it, and is adjusted by one given after it. #[test] +#[cfg(target_pointer_width = "64")] fn max_sets_every_knob_to_its_hardest_end() { let opts = parse(&["--max", "f"]).unwrap(); assert!(opts.long, "--max enables long-distance matching"); @@ -4399,6 +4449,7 @@ fn max_sets_every_knob_to_its_hardest_end() { /// A `--max` frame over a known-size input is down-sized to the input, so it /// compresses without the widest tables and decodes back to the input. #[test] +#[cfg(target_pointer_width = "64")] fn a_max_frame_round_trips() { let opts = parse(&["--max", "f"]).unwrap(); // Small: at a search depth of 2^30 a debug build walks every candidate. @@ -4420,6 +4471,15 @@ fn a_max_frame_round_trips() { assert_eq!(decoded(&frame).unwrap(), payload); } +/// On a 32-bit target `--max` is refused, as the reference refuses it: its +/// tables at their widest do not fit the address space. +#[test] +#[cfg(not(target_pointer_width = "64"))] +fn max_is_refused_on_a_32_bit_target() { + let err = parse(&["--max", "f"]).unwrap_err(); + assert!(err.to_string().contains("32-bit"), "{err}"); +} + /// `--long` below level 16 is refused because the matcher does not run there, /// unless `--zstd=strat=` moves the level onto a parser where it does. #[test] diff --git a/zstd/src/dictionary/legacy/tests.rs b/zstd/src/dictionary/legacy/tests.rs index 31c4984fa..8bf76c46e 100644 --- a/zstd/src/dictionary/legacy/tests.rs +++ b/zstd/src/dictionary/legacy/tests.rs @@ -20,6 +20,30 @@ fn neighbour_walks_stop_at_the_noise_slots() { assert_eq!(solution.length, 0); } +/// The same hang in the second walk, the one that measures the kept segment's +/// neighbourhood: four copies of the band repeat `MIN_RATIO` times, so the +/// analysis gets past the repetition check and walks the ranks again, reaching +/// the upper noise slot with the band still matching. +#[test] +fn the_measuring_walk_stops_at_the_noise_slot() { + let band = noise_band(); + let corpus_bytes: Vec = band + .iter() + .copied() + .cycle() + .take(4 * NOISE_LENGTH) + .collect(); + let corpus = Corpus::new(&corpus_bytes); + // In suffix order: each copy's suffix is a prefix of the one before it. + let suffixes = Suffixes::new(vec![96, 64, 32, 0], corpus_bytes.len()); + let mut done = vec![false; corpus_bytes.len() + 16]; + let solution = analyze_position(&mut done, &suffixes, 0, &corpus, MIN_RATIO); + assert!( + solution.length as usize >= MIN_MATCH_LENGTH, + "four copies repeat often enough to be kept" + ); +} + /// `count` log lines of a few shapes, each line a sample. fn log_samples(count: u32) -> (Vec, Vec) { const SHAPES: [&str; 4] = [ diff --git a/zstd/src/encoding/blocks/compressed.rs b/zstd/src/encoding/blocks/compressed.rs index 2cdc71344..62750da4c 100644 --- a/zstd/src/encoding/blocks/compressed.rs +++ b/zstd/src/encoding/blocks/compressed.rs @@ -416,6 +416,7 @@ pub(crate) fn compress_block_with_post_split( scratch.partitions.clear(); scratch.prefix_sums.rebuild(&scratch.parts.sequences); let mut workspace = scratch.estimator_workspace.take().unwrap_or_default(); + let built_huff = core::mem::take(&mut workspace.built_huff); // Reuse the estimator's inner scratch across frames instead of // allocating a fresh `CompressedBlockScratch` (count tables + Vecs) // every block-split. Lazily created on the first split. @@ -439,7 +440,7 @@ pub(crate) fn compress_block_with_post_split( offset_hist: state.offset_hist, }, entry_huff: state.last_huff_table.as_ref(), - built_huff: Vec::new(), + built_huff, scratch_state: CompressState { matcher: EntropyOnlyMatcher, // The splitter's scratch state never reaches the raw-skip, which @@ -458,7 +459,7 @@ pub(crate) fn compress_block_with_post_split( // on every post-split block and drop them at the end of it. Handed // back below, so the emitter that follows keeps using the same one. huff_weights: core::mem::take(&mut state.huff_weights), - fse_tables: clone_fse_tables(&state.fse_tables), + fse_tables: probe_fse_tables(&state.fse_tables), block_scratch: inner_scratch, offset_hist: state.offset_hist, strategy_tag: state.strategy_tag, @@ -494,11 +495,14 @@ pub(crate) fn compress_block_with_post_split( { workspace.recycle_fse(handle); } - scratch.estimator_workspace = Some(workspace); - // The tables the probes built go back to the builder for its next tables. - for table in estimator.built_huff { + // The tables the probes built go back to the builder for its next tables, + // and their emptied arena to the workspace for the next block. + let mut built_huff = estimator.built_huff; + for table in built_huff.drain(..) { huff_weights.recycle(table); } + workspace.built_huff = built_huff; + scratch.estimator_workspace = Some(workspace); state.huff_weights = huff_weights; scratch.estimator_inner = Some(Box::new(inner_block_scratch)); @@ -904,13 +908,19 @@ struct EstimatorWorkspace { /// FSE table handles no probe state holds any more, for the next probe to /// build into. Every one is unique, so a build writes it in place. spare_fse: Vec, + /// The arena a block's probes keep the Huffman tables they build in + /// ([`SplitEstimator::built_huff`]), empty between blocks: the tables go back + /// to the weight builder, the room stays for the next block's probes. + built_huff: Vec, } impl EstimatorWorkspace { /// The four boxed count tables, always present once the workspace exists, - /// and the pooled tables with their reference counts. + /// the pooled tables with their reference counts, and the room kept for + /// the probes' Huffman tables. fn heap_size(&self) -> usize { 4 * core::mem::size_of::<[usize; 256]>() + + self.built_huff.capacity() * core::mem::size_of::() + self.spare_fse.capacity() * core::mem::size_of::() + self.spare_fse.len() * (core::mem::size_of::() @@ -941,6 +951,7 @@ impl Default for EstimatorWorkspace { ml_counts: Box::new([0; 256]), of_counts: Box::new([0; 256]), spare_fse: Vec::new(), + built_huff: Vec::new(), } } } @@ -963,9 +974,20 @@ fn estimate_block_parts_size( let mut sums = SequencePrefixSums::default(); sums.rebuild(raw_sequences); let previous = state.last_huff_table.take(); - let (bytes, outcome) = estimate_block_parts_size_with( + let tables = &mut state.fse_tables; + let previous_fse = [ + tables.ll_previous.take(), + tables.ml_previous.take(), + tables.of_previous.take(), + ]; + let (bytes, outcome, decisions) = estimate_block_parts_size_with( state, previous.as_ref(), + [ + previous_fse[0].as_ref(), + previous_fse[1].as_ref(), + previous_fse[2].as_ref(), + ], literals_vec, raw_sequences, sums.codes(0, raw_sequences.len()), @@ -976,6 +998,21 @@ fn estimate_block_parts_size( HuffOutcome::Clear => None, HuffOutcome::New(table) => Some(table), }; + let [ll, ml, of] = previous_fse; + let tables = &mut state.fse_tables; + tables.ll_previous = ll; + tables.ml_previous = ml; + tables.of_previous = of; + remember_last_used_tables(tables, decisions); + for next in [ + &mut tables.ll_next, + &mut tables.ml_next, + &mut tables.of_next, + ] { + if let Some(handle) = next.take() { + workspace.recycle_fse(handle); + } + } bytes } @@ -991,17 +1028,19 @@ enum HuffOutcome { /// `estimate_block_parts_size` for sequences whose length codes are already /// derived: the splitter's probes, which price many ranges of one block. The -/// Huffman table the section may repeat is `previous`, and what the section -/// does with it is returned beside the size; the FSE repeat tables and the -/// offset history are advanced in `state`. +/// tables the section may repeat are borrowed, the Huffman table as `previous` +/// and the FSE tables as `previous_fse`, and what the section does with each is +/// returned beside the size, a built FSE table in its `*_next` slot of +/// `state.fse_tables`; the offset history is advanced in `state`. fn estimate_block_parts_size_with( state: &mut CompressState, previous: Option<&huff0_encoder::HuffmanTable>, + previous_fse: [Option<&PreviousFseTable>; 3], literals_vec: &[u8], raw_sequences: &[RawSequence], codes: LengthCodes<'_>, workspace: &mut EstimatorWorkspace, -) -> (usize, HuffOutcome) { +) -> (usize, HuffOutcome, [LastUsedTable; 3]) { let (lit_bytes, outcome) = estimate_literals_section_bytes( literals_vec, previous, @@ -1013,20 +1052,22 @@ fn estimate_block_parts_size_with( literals_suspected_incompressible(literals_vec.len(), raw_sequences.len()), ); - let seq_bytes = if raw_sequences.is_empty() { - 1 + // A section without sequences writes no tables, so every axis keeps its own. + let (seq_bytes, decisions) = if raw_sequences.is_empty() { + (1, [LastUsedTable::Keep; 3]) } else { estimate_sequences_section_bytes( raw_sequences, codes, &mut state.offset_hist, + previous_fse, &mut state.fse_tables, workspace, state.strategy_tag, ) }; - (lit_bytes + seq_bytes, outcome) + (lit_bytes + seq_bytes, outcome, decisions) } // One argument over the lint's threshold. Every one of them is a distinct @@ -1200,14 +1241,20 @@ fn estimate_literals_section_bytes( /// that advances `offset_hist` as the emitter would; the length codes and /// their extra bits come precomputed in `codes`. Each histogram is built once /// and handed to the table selection, which the emitter's path counts again. +/// +/// The tables the section may repeat are `previous` (LL, ML, OF), borrowed: a +/// probe reads them and does not take a handle. What the section decided for +/// each axis is returned beside the size, a built table left in that axis's +/// `*_next` slot of `fse_tables`, for the caller to commit. fn estimate_sequences_section_bytes( sequences: &[RawSequence], codes: LengthCodes<'_>, offset_hist: &mut [u32; 3], + previous: [Option<&PreviousFseTable>; 3], fse_tables: &mut FseTables, workspace: &mut EstimatorWorkspace, strategy: crate::encoding::strategy::StrategyTag, -) -> usize { +) -> (usize, [LastUsedTable; 3]) { let EstimatorWorkspace { ll_counts, ml_counts, @@ -1258,23 +1305,22 @@ fn estimate_sequences_section_bytes( // Destructured for the same reason as the emitter: the default accessors // borrow the whole struct, which would collide with the `*_next` slots. let FseTables { - ll_previous, - ml_previous, - of_previous, ll_next, ml_next, of_next, ll_default, ml_default, of_default, + .. } = fse_tables; let ll_default: &FSETable = ll_default; let ml_default: &FSETable = ml_default; let of_default: &FSETable = of_default; + let [ll_previous, ml_previous, of_previous] = previous; // The table selection the real encoder makes, from the same histograms. let ll_mode = choose_table_from_counts( - ll_previous.as_ref(), + ll_previous, ll_default, ll_counts, total, @@ -1285,7 +1331,7 @@ fn estimate_sequences_section_bytes( ll_next, ); let ml_mode = choose_table_from_counts( - ml_previous.as_ref(), + ml_previous, ml_default, ml_counts, total, @@ -1296,7 +1342,7 @@ fn estimate_sequences_section_bytes( ml_next, ); let of_mode = choose_table_from_counts( - of_previous.as_ref(), + of_previous, of_default, of_counts, total, @@ -1333,34 +1379,22 @@ fn estimate_sequences_section_bytes( }; let stream_bytes = (bit_content + padding_bits) / 8; - // Mirror state mutation done by `encode_block_parts`. + // What `encode_block_parts` would commit, left to the caller to commit. let decisions = [ into_last_used_table(ll_mode), into_last_used_table(ml_mode), into_last_used_table(of_mode), ]; - remember_last_used_tables(fse_tables, decisions); - // The emitter keeps the handle a commit displaces, to build the next - // block's table into. A probe must not: the splitter holds many of these - // states at once, and a spare per axis per probe doubles the tables alive - // at any moment. It goes to the workspace pool instead, which every probe - // draws its build slots from. - for next in [ - &mut fse_tables.ll_next, - &mut fse_tables.ml_next, - &mut fse_tables.of_next, - ] { - if let Some(handle) = next.take() { - workspace.recycle_fse(handle); - } - } - nb_seq_header - + mode_byte - + ll_table_desc_bytes - + of_table_desc_bytes - + ml_table_desc_bytes - + stream_bytes + ( + nb_seq_header + + mode_byte + + ll_table_desc_bytes + + of_table_desc_bytes + + ml_table_desc_bytes + + stream_bytes, + decisions, + ) } /// Bit cost of a sequence section under `mode`, matching what @@ -1842,7 +1876,10 @@ fn highest_used_code(counts: &[usize; 256]) -> usize { .unwrap_or(0) } -fn clone_fse_tables(fse_tables: &FseTables) -> FseTables { +/// The FSE tables a split probe's scratch state works with: the defaults, and +/// no previous table, since every probe reads the one it may repeat from the +/// state it starts from. +fn probe_fse_tables(fse_tables: &FseTables) -> FseTables { // The `*_default` fields are cfg-typed via the // [`crate::fse::fse_encoder::FseDefaultTable`] alias — // `&'static FSETable` on atomic / `critical-section` targets @@ -1864,17 +1901,17 @@ fn clone_fse_tables(fse_tables: &FseTables) -> FseTables { ll_default: fse_tables.ll_default, #[cfg(not(any(target_has_atomic = "ptr", feature = "critical-section")))] ll_default: fse_tables.ll_default.clone(), - ll_previous: fse_tables.ll_previous.clone(), + ll_previous: None, #[cfg(any(target_has_atomic = "ptr", feature = "critical-section"))] ml_default: fse_tables.ml_default, #[cfg(not(any(target_has_atomic = "ptr", feature = "critical-section")))] ml_default: fse_tables.ml_default.clone(), - ml_previous: fse_tables.ml_previous.clone(), + ml_previous: None, #[cfg(any(target_has_atomic = "ptr", feature = "critical-section"))] of_default: fse_tables.of_default, #[cfg(not(any(target_has_atomic = "ptr", feature = "critical-section")))] of_default: fse_tables.of_default.clone(), - of_previous: fse_tables.of_previous.clone(), + of_previous: None, // Empty, not blank tables: a probe gets its own slots so it cannot // overwrite what the emitter is describing, but most probes never // build a custom table and must not pay for one. @@ -1942,30 +1979,16 @@ impl SplitEstimator<'_> { } else { lit_start + lit_len }; - // The FSE repeat tables are shared handles, so seeding them is a - // reference-count bump; the Huffman table is only borrowed. What the - // last probe left behind goes to the pool, and a build slot comes from - // it, so a probe that builds a table writes into one it already has. + // Every table the probe may repeat is read where the entry state keeps + // it; a handle is taken only for what the post state keeps. A build slot + // comes from the pool, so a probe that builds a table writes into one it + // already has. let tables = &mut self.scratch_state.fse_tables; - for (previous, next, seed) in [ - ( - &mut tables.ll_previous, - &mut tables.ll_next, - &entry.ll_previous, - ), - ( - &mut tables.ml_previous, - &mut tables.ml_next, - &entry.ml_previous, - ), - ( - &mut tables.of_previous, - &mut tables.of_next, - &entry.of_previous, - ), + for next in [ + &mut tables.ll_next, + &mut tables.ml_next, + &mut tables.of_next, ] { - self.workspace - .recycle_previous(core::mem::replace(previous, seed.clone())); if next.is_none() { *next = self.workspace.spare_fse.pop(); } @@ -1976,9 +1999,14 @@ impl SplitEstimator<'_> { HuffRef::BlockEntry => self.entry_huff, HuffRef::Built(at) => Some(&self.built_huff[at]), }; - let (emitted_payload, outcome) = estimate_block_parts_size_with( + let (emitted_payload, outcome, decisions) = estimate_block_parts_size_with( &mut self.scratch_state, previous, + [ + entry.ll_previous.as_ref(), + entry.ml_previous.as_ref(), + entry.of_previous.as_ref(), + ], &self.parts.literals[lit_start..lit_end], &self.parts.sequences[start_idx..end_idx], self.prefix_sums.codes(start_idx, end_idx), @@ -1993,7 +2021,8 @@ impl SplitEstimator<'_> { emitted_payload } + 3; // Real emit on raw fallback restores the entry state — see - // `emit_single_sequence_block`'s saved-state restore branch. + // `emit_single_sequence_block`'s saved-state restore branch. A table the + // probe built stays in its slot for the next probe to build into. let post = if raw_fallback { if let HuffOutcome::New(table) = outcome { self.scratch_state.huff_weights.recycle(table); @@ -2008,11 +2037,13 @@ impl SplitEstimator<'_> { HuffRef::Built(self.built_huff.len() - 1) } }; + let tables = &mut self.scratch_state.fse_tables; + let [ll, ml, of] = decisions; ProbeEntryState { huff, - ll_previous: self.scratch_state.fse_tables.ll_previous.take(), - ml_previous: self.scratch_state.fse_tables.ml_previous.take(), - of_previous: self.scratch_state.fse_tables.of_previous.take(), + ll_previous: decided_previous(ll, &entry.ll_previous, &mut tables.ll_next), + ml_previous: decided_previous(ml, &entry.ml_previous, &mut tables.ml_next), + of_previous: decided_previous(of, &entry.of_previous, &mut tables.of_next), offset_hist: self.scratch_state.offset_hist, } }; @@ -2532,6 +2563,26 @@ enum LastUsedTable { Encoded, } +/// The table an axis repeats after a probe decided `decision` from `entry`: +/// what [`commit_last_used_table`] would leave in the previous slot, taken for +/// a probe state. Only a repeat takes a handle to the entry's table; a built +/// table moves out of `next`, which the next probe refills from the pool. +fn decided_previous( + decision: LastUsedTable, + entry: &Option, + next: &mut Option, +) -> Option { + match decision { + LastUsedTable::Keep => entry.clone(), + LastUsedTable::Default => Some(PreviousFseTable::Default), + LastUsedTable::Rle(symbol) => Some(PreviousFseTable::Rle(symbol)), + LastUsedTable::Encoded => Some(PreviousFseTable::Custom( + next.take() + .expect("an encoded axis built its table into the slot"), + )), + } +} + fn into_last_used_table(mode: FseTableMode<'_>) -> LastUsedTable { match mode { FseTableMode::Encoded(_) => LastUsedTable::Encoded, diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index ffe87097f..1cc4411c7 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -1106,7 +1106,10 @@ macro_rules! optimal_block_body { reps: $reps, literals_cursor: 0, }; - while $buffers.pass.cursor < $current.len().saturating_sub(8) { + // Bound once per block. A block shorter than the 8-byte tail the parser + // leaves as literals has no segment, which the floor at zero says. + let match_loop_limit = $current.len().saturating_sub(8); + while $buffers.pass.cursor < match_loop_limit { let cursor = $buffers.pass.cursor; let segment = &$current[cursor..]; let segment_abs_start = $current_abs_start + cursor; From b5d9c9d1f9c4bad3f39f78b39da18695170d515d Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 27 Sep 2026 00:40:13 +0300 Subject: [PATCH 33/36] fix: bound benchmark frames and training sizes by their budget - The benchmark listed one length per frame after the `-M` check, a word a frame that the ceiling never counted: 512 MiB for `-B32` over 2 GiB. The frame lengths are now yielded from the same arithmetic layout instead of collected. - Training samples were loaded against a budget for their bytes alone, while each one also records its length in a word: under `-B1` the list was eight times the bytes. The budget now holds both, and the list is sized from it. Carries training_sample_sizes_count_against_the_budget, which fails without the change (64 bytes and 64 sizes against a 200-byte budget). - The legacy trainer's prefix comparison tested a subtraction that cannot underflow on every call: every position it is given is a suffix or the noise slot. The invariant is a debug assertion now. - The legacy trainer finalizes with every sample on purpose, as the reference does; the call says so. - The NEON prefix kernel records the measurements behind it. Part of #128 --- zstd/src/bin/structured-zstd/main.rs | 45 ++++++++++++++------------- zstd/src/bin/structured-zstd/tests.rs | 26 +++++++++++++++- zstd/src/dictionary/legacy.rs | 10 +++--- zstd/src/dictionary/mod.rs | 4 +++ zstd/src/encoding/fastpath/neon.rs | 7 +++++ 5 files changed, 65 insertions(+), 27 deletions(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index cee7b6c17..873aa1293 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -2970,19 +2970,18 @@ fn read_inputs_bounded(inputs: &[PathBuf], sizes: &[u64]) -> Result> { /// one of at least [`MIN_BENCH_BLOCK_SIZE`] is given. An empty input yields no /// frame. This is the reference's block table (`benchzstd.c`, /// `BMK_benchMemAdvancedNoAlloc`), so ratio and speed describe the same frames. -fn bench_chunk_lengths(file_sizes: &[u64], block_size: Option) -> Vec { +/// +/// Yielded, not listed: with a small `-B` over a large input a list would hold +/// a word per frame that the `-M` ceiling never counted. +fn bench_chunk_lengths( + file_sizes: &[u64], + block_size: Option, +) -> impl Iterator + '_ { let block = block_size.filter(|&size| size >= MIN_BENCH_BLOCK_SIZE); - let mut chunks = Vec::new(); - for &size in file_sizes { + file_sizes.iter().flat_map(move |&size| { let piece = block.unwrap_or(size.max(1)); - let mut left = size; - while left > 0 { - let take = left.min(piece); - chunks.push(take); - left -= take; - } - } - chunks + (0..size.div_ceil(piece)).map(move |at| piece.min(size - at * piece)) + }) } /// The frames [`bench_chunk_lengths`] cuts `file_sizes` into, measured without @@ -3053,9 +3052,8 @@ fn benchmark_one( ); } - let chunks = bench_chunk_lengths(file_sizes, opts.block_size); debug_assert_eq!( - chunks.iter().sum::(), + bench_chunk_lengths(file_sizes, opts.block_size).sum::(), data.len() as u64, "the frames cover the input exactly" ); @@ -3088,7 +3086,7 @@ fn benchmark_one( compressed.clear(); let t = Instant::now(); let mut rest = data; - for &chunk in &chunks { + for chunk in bench_chunk_lengths(file_sizes, opts.block_size) { // Each piece is a frame of its own, as the reference's // benchmark compresses every block independently. The length is // exact, so it is pledged rather than estimated. @@ -3546,15 +3544,19 @@ fn load_training_samples( split files into fixed-size samples with -B#" ); } - let budget = wanted - .min(TRAINING_DATA_MAX) - .min(memory_limit.unwrap_or(u64::MAX)); - let budget = usize::try_from(budget) + // The budget holds each sample's recorded length as well as its bytes: under + // a small `-B` the lengths outweigh the bytes they describe, a word per byte + // at `-B1`, and a limit on the bytes alone would be exceeded by the list. + const SIZE_ENTRY: u64 = core::mem::size_of::() as u64; + let budget = TRAINING_DATA_MAX.min(memory_limit.unwrap_or(u64::MAX)); + let corpus_room = usize::try_from(wanted.min(budget)) .map_err(|_| eyre!("{budget} bytes of samples is more than this machine can hold"))?; + // At most as many entries as the smallest samples fit in the budget. + let entries = samples.min(budget / (SIZE_ENTRY + block_size.unwrap_or(1))); let mut set = TrainingSet { - corpus: Vec::with_capacity(budget), - sizes: Vec::new(), + corpus: Vec::with_capacity(corpus_room), + sizes: Vec::with_capacity(entries as usize), sources: Vec::new(), }; 'files: for input in order { @@ -3577,7 +3579,8 @@ fn load_training_samples( Some(block) => (size - taken).min(block), None => size.min(TRAINING_SAMPLE_MAX), }; - if set.corpus.len() as u64 + piece > budget as u64 { + let held = set.corpus.len() as u64 + (set.sizes.len() as u64 + 1) * SIZE_ENTRY; + if held + piece > budget { if taken == 0 { break 'files; } diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index e81a396e5..b1710f9bb 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -951,7 +951,7 @@ fn the_frame_extent_matches_the_frames_it_describes() { let files: [&[u64]; 4] = [&[], &[0, 1, 5000], &[1 << 20, 3], &[70_000, 0, 131_071]]; for sizes in files { for block in [None, Some(1024), Some(MIN_BENCH_BLOCK_SIZE), Some(40_000)] { - let chunks = bench_chunk_lengths(sizes, block); + let chunks: Vec = bench_chunk_lengths(sizes, block).collect(); let room: u64 = chunks .iter() .map(|&len| structured_zstd::encoding::compress_bound(len as usize) as u64) @@ -966,6 +966,28 @@ fn the_frame_extent_matches_the_frames_it_describes() { } } +/// The training budget holds the sample sizes as well as the samples. Under +/// `-B1` every byte is a sample whose recorded length is a word, eight times +/// the byte it describes, so a budget spent on the bytes alone was exceeded +/// several times over by the list beside them. +#[test] +fn training_sample_sizes_count_against_the_budget() { + let input = std::env::temp_dir().join(format!("szstd-train-b1-{}", std::process::id())); + fs::write(&input, [3u8; 64]).unwrap(); + + let set = load_training_samples(std::slice::from_ref(&input), Some(1), Some(200)); + + let _ = fs::remove_file(&input); + let set = set.expect("the samples load within the budget"); + assert!(!set.sizes.is_empty(), "some samples fit"); + assert!( + set.corpus.len() + set.sizes.len() * core::mem::size_of::() <= 200, + "{} bytes of samples and {} sizes exceed the 200-byte budget", + set.corpus.len(), + set.sizes.len() + ); +} + /// File sizes that add up past `u64::MAX` (sparse files report any length) do /// not wrap: the total stays at the most there is, which the loader's cap then /// cuts to what it trains on, and the sample count is exact. @@ -2055,6 +2077,8 @@ fn block_size_is_read_like_the_reference_reads_it() { /// input yields no frame, and a smaller `-B` cuts nothing. #[test] fn benchmark_frames_follow_the_inputs_and_the_block_size() { + let bench_chunk_lengths = + |sizes: &[u64], block| super::bench_chunk_lengths(sizes, block).collect::>(); assert_eq!(bench_chunk_lengths(&[100, 50], None), vec![100, 50]); assert_eq!( bench_chunk_lengths(&[100, 50], Some(40)), diff --git a/zstd/src/dictionary/legacy.rs b/zstd/src/dictionary/legacy.rs index 3e37b44f5..27f68ce2e 100644 --- a/zstd/src/dictionary/legacy.rs +++ b/zstd/src/dictionary/legacy.rs @@ -148,11 +148,11 @@ impl<'a> Corpus<'a> { #[inline] fn common(&self, a: usize, b: usize) -> usize { let samples = self.samples; - // A comparison that starts in the band has nothing to compare in the - // samples. - let Some(in_samples) = samples.len().checked_sub(a.max(b)) else { - return self.common_tail(a, b, 0); - }; + // Every position compared is a suffix, below the samples' length, or + // the noise slot, at it; one at the slot compares nothing in the + // samples and goes straight to the band. + debug_assert!(a.max(b) <= samples.len()); + let in_samples = samples.len() - a.max(b); let base = samples.as_ptr(); // SAFETY: `a, b <= max(a, b)` and `in_samples == samples.len() - // max(a, b)`, so both runs of `in_samples` bytes lie in the samples. diff --git a/zstd/src/dictionary/mod.rs b/zstd/src/dictionary/mod.rs index 3266ab601..01f316165 100644 --- a/zstd/src/dictionary/mod.rs +++ b/zstd/src/dictionary/mod.rs @@ -774,6 +774,10 @@ pub fn create_legacy_dict_from_slice( }; io::Error::new(io::ErrorKind::InvalidInput, reason) })?; + // Every sample, not only those the content search kept: the reference cuts + // the corpus to its size limit inside the search alone, for its suffix + // sort (zdict.c, `ZDICT_trainBuffer_legacy`), and builds the entropy tables + // from all of them (`ZDICT_trainFromBuffer_unsafe_legacy`). let finalized = finalize_raw_dict(content.as_slice(), samples, dict_size, finalize)?; output.write_all(finalized.as_slice()) } diff --git a/zstd/src/encoding/fastpath/neon.rs b/zstd/src/encoding/fastpath/neon.rs index a9be64b83..7283ffdd4 100644 --- a/zstd/src/encoding/fastpath/neon.rs +++ b/zstd/src/encoding/fastpath/neon.rs @@ -37,6 +37,13 @@ fn first_unequal(eq: uint8x16_t) -> usize { /// joined and their minimum lane tested, and the mismatch is only located on /// the step that has one. /// +/// Measured on M1 against the 16-byte step with two lane tests it replaced, +/// interleaved: the legacy dictionary trainer, whose comparisons run long, +/// 4.96-5.98 s -> 4.19-4.41 s wall (zstd 1.5.7: 4.35-4.67 s); the level-19 +/// encoder on z000033 1.664-1.691 -> 1.613-1.648 G cycles (libzstd: +/// 1.546-1.617 G) while a decode-only control moved 3.458-3.475 -> +/// 3.421-3.457 G. +/// /// # Safety /// `lhs` / `rhs` must point to at least `max` initialized bytes. NEON must be /// available — guaranteed on AArch64 baseline but enforced by the From 1fda06a402f28dcaf62b02c7af2ad4d9cea11d4c Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 27 Sep 2026 00:52:37 +0300 Subject: [PATCH 34/36] fix(cli): build the 32-bit --max test without a Debug bound on Options unwrap_err needs the Ok type to be Debug, which Options is not, so the i686 test target did not compile. The refusal is matched with let-else instead. Part of #128 --- zstd/src/bin/structured-zstd/tests.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index b1710f9bb..48cb68eba 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -4500,7 +4500,9 @@ fn a_max_frame_round_trips() { #[test] #[cfg(not(target_pointer_width = "64"))] fn max_is_refused_on_a_32_bit_target() { - let err = parse(&["--max", "f"]).unwrap_err(); + let Err(err) = parse(&["--max", "f"]) else { + panic!("--max must be refused on a 32-bit target"); + }; assert!(err.to_string().contains("32-bit"), "{err}"); } From 8255e81eb67cc56324dba811fab44691b788c305 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 27 Sep 2026 02:07:46 +0300 Subject: [PATCH 35/36] fix(cli): split the training budget, sum sample sizes exactly - The samples buffer and the list of their lengths each reserved room from the whole budget, so under -B1 the two together reserved close to twice it. They are now split from it: a sample holds at most one piece and costs one entry, which bounds the bytes at piece / (piece + entry) of the budget and the entries at one per whole piece plus one per file. - A block size near u64::MAX overflowed the budget arithmetic (a sample's cost, and the check of a piece against what is left). The cost is computed in u128 and the check compares without adding the piece. Carries a_block_size_at_the_top_of_the_range_loads_whole_files, which panicked on the overflow. - The training totals were summed with saturation, which turned the sample count into "many" instead of a count once it overflowed. They are summed exactly in u128. - The optimal parser's segment loop ran to `len.saturating_sub(8)`; it now tests `cursor + 8 < len`, which needs no floor because the cursor never passes the block's end. Frames are identical by construction; runner1, 50 frames of z000033[..200000], interleaved: L16 1766-1780 -> 1768-1792 ms, L19 2617-2660 -> 2610-2638 ms, control L3 flat. training_sample_sizes_count_against_the_budget now also checks the reserved room, and failed on the previous reservations (64 bytes of room and 22 entries against a 200-byte budget). Part of #128 --- zstd/src/bin/structured-zstd/main.rs | 47 +++++++++++++++++---------- zstd/src/bin/structured-zstd/tests.rs | 44 ++++++++++++++++++++++--- zstd/src/encoding/hc/optimal.rs | 9 ++--- 3 files changed, 73 insertions(+), 27 deletions(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 873aa1293..1508fa600 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -3495,14 +3495,11 @@ fn shuffle_training_files(files: &mut [T]) { /// loader would take them without a limit: each file one sample of at most /// [`TRAINING_SAMPLE_MAX`] bytes, or cut into `block_size` samples; empty files /// left out. -fn training_extent(file_sizes: &[u64], block_size: Option) -> (u64, u64) { - // Both only ever meet a bound: `wanted` is cut to the trainer's cap and - // `samples` is compared with the minimum and the count loaded. A sum past - // the integer range (sparse files report any length) is therefore "more - // than any bound", which is what stopping at the top says; wrapping would - // turn it into a small number and a silently smaller training set. - let mut wanted = 0u64; - let mut samples = 0u64; +fn training_extent(file_sizes: &[u64], block_size: Option) -> (u128, u128) { + // Summed exactly: sparse files report any length, and the sum of a slice of + // `u64`s cannot outgrow `u128` (a slice holds fewer than 2^64 of them). + let mut wanted = 0u128; + let mut samples = 0u128; for &size in file_sizes { if size == 0 { continue; @@ -3511,8 +3508,8 @@ fn training_extent(file_sizes: &[u64], block_size: Option) -> (u64, u64) { Some(block) => (size.div_ceil(block), size), None => (1, size.min(TRAINING_SAMPLE_MAX)), }; - samples = samples.saturating_add(count); - wanted = wanted.saturating_add(bytes); + samples += u128::from(count); + wanted += u128::from(bytes); } (samples, wanted) } @@ -3538,7 +3535,7 @@ fn load_training_samples( ); } let (samples, wanted) = training_extent(&file_sizes, block_size); - if samples < TRAINING_SAMPLES_MIN as u64 { + if samples < TRAINING_SAMPLES_MIN as u128 { bail!( "{samples} training sample(s) is too few; provide one file per sample, or \ split files into fixed-size samples with -B#" @@ -3549,18 +3546,29 @@ fn load_training_samples( // at `-B1`, and a limit on the bytes alone would be exceeded by the list. const SIZE_ENTRY: u64 = core::mem::size_of::() as u64; let budget = TRAINING_DATA_MAX.min(memory_limit.unwrap_or(u64::MAX)); - let corpus_room = usize::try_from(wanted.min(budget)) + // The two buffers share the budget, so their room is split from it rather + // than each given all of it. A sample holds at most `piece` bytes and costs + // an entry, so the bytes are at most `piece / (piece + entry)` of the + // budget; the entries are one per whole piece that fits, plus one per file + // for the shorter piece at its end. + let piece = block_size.unwrap_or(TRAINING_SAMPLE_MAX); + let files = file_sizes.iter().filter(|&&size| size > 0).count() as u64; + // In `u128`: `piece` is whatever `-B` asked for, up to `u64::MAX`. + let per_sample = u128::from(piece) + u128::from(SIZE_ENTRY); + let bytes_room = u128::from(budget) * u128::from(piece) / per_sample; + let corpus_room = usize::try_from(wanted.min(bytes_room)) .map_err(|_| eyre!("{budget} bytes of samples is more than this machine can hold"))?; - // At most as many entries as the smallest samples fit in the budget. - let entries = samples.min(budget / (SIZE_ENTRY + block_size.unwrap_or(1))); + let entries = samples.min(u128::from(budget) / per_sample + u128::from(files)); + let entries = usize::try_from(entries) + .map_err(|_| eyre!("{entries} samples is more than this machine can hold"))?; let mut set = TrainingSet { corpus: Vec::with_capacity(corpus_room), - sizes: Vec::with_capacity(entries as usize), + sizes: Vec::with_capacity(entries), sources: Vec::new(), }; 'files: for input in order { - if set.sizes.len() as u64 >= samples { + if set.sizes.len() as u128 >= samples { break; } let file = File::open(input) @@ -3579,8 +3587,11 @@ fn load_training_samples( Some(block) => (size - taken).min(block), None => size.min(TRAINING_SAMPLE_MAX), }; + // What is held with this sample's entry, never more than a word past + // the budget; compared without adding `piece`, which a large `-B` + // makes as large as the file. let held = set.corpus.len() as u64 + (set.sizes.len() as u64 + 1) * SIZE_ENTRY; - if held + piece > budget { + if held > budget || piece > budget - held { if taken == 0 { break 'files; } @@ -3599,7 +3610,7 @@ fn load_training_samples( } set.sizes.push(piece as usize); taken += piece; - if block_size.is_none() || taken >= size || set.sizes.len() as u64 >= samples { + if block_size.is_none() || taken >= size || set.sizes.len() as u128 >= samples { break; } } diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 48cb68eba..959337860 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -980,6 +980,15 @@ fn training_sample_sizes_count_against_the_budget() { let _ = fs::remove_file(&input); let set = set.expect("the samples load within the budget"); assert!(!set.sizes.is_empty(), "some samples fit"); + // Reserved as well as used: both buffers are sized from the one budget, + // which allows a word per file for the partial sample at its end. + let entry = core::mem::size_of::(); + assert!( + set.corpus.capacity() + set.sizes.capacity() * entry <= 200 + entry, + "{} bytes of room for samples and {} for sizes exceed the 200-byte budget", + set.corpus.capacity(), + set.sizes.capacity() + ); assert!( set.corpus.len() + set.sizes.len() * core::mem::size_of::() <= 200, "{} bytes of samples and {} sizes exceed the 200-byte budget", @@ -988,15 +997,40 @@ fn training_sample_sizes_count_against_the_budget() { ); } -/// File sizes that add up past `u64::MAX` (sparse files report any length) do -/// not wrap: the total stays at the most there is, which the loader's cap then -/// cuts to what it trains on, and the sample count is exact. +/// A block size at the top of the range makes each file one sample, and the +/// budget arithmetic that divides by a sample's cost, or compares a piece with +/// what is left, must not overflow on it. +#[test] +fn a_block_size_at_the_top_of_the_range_loads_whole_files() { + let input = std::env::temp_dir().join(format!("szstd-train-bmax-{}", std::process::id())); + fs::write(&input, [5u8; 700]).unwrap(); + + let set = load_training_samples( + &[ + input.clone(), + input.clone(), + input.clone(), + input.clone(), + input.clone(), + ], + Some(u64::MAX), + None, + ); + + let _ = fs::remove_file(&input); + let set = set.expect("five files are five samples"); + assert_eq!(set.sizes, vec![700; 5]); +} + +/// File sizes that add up past `u64::MAX` (sparse files report any length) are +/// summed exactly rather than wrapped or stopped at a bound: both the byte +/// total and the sample count are the true ones. #[test] fn training_sizes_past_the_integer_range_do_not_wrap() { let huge = u64::MAX / 2 + 1; let (samples, wanted) = training_extent(&[huge, huge, 0], Some(4096)); - assert_eq!(wanted, u64::MAX); - assert_eq!(samples, 2 * huge.div_ceil(4096)); + assert_eq!(wanted, 2 * u128::from(huge)); + assert_eq!(samples, 2 * u128::from(huge.div_ceil(4096))); } /// A benchmark input that shrank between being sized and being read is diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index 1cc4411c7..289db7918 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -1106,10 +1106,11 @@ macro_rules! optimal_block_body { reps: $reps, literals_cursor: 0, }; - // Bound once per block. A block shorter than the 8-byte tail the parser - // leaves as literals has no segment, which the floor at zero says. - let match_loop_limit = $current.len().saturating_sub(8); - while $buffers.pass.cursor < match_loop_limit { + // The last 8 bytes are left as literals. Written as an addition on the + // cursor, which never passes the block's end, so a block shorter than + // the tail needs no floor: it simply has no segment. + let block_len = $current.len(); + while $buffers.pass.cursor + 8 < block_len { let cursor = $buffers.pass.cursor; let segment = &$current[cursor..]; let segment_abs_start = $current_abs_start + cursor; From 50ccfc4c8ac00ffaf535c01c4ce9dfc1f67b2d07 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sun, 27 Sep 2026 02:33:01 +0300 Subject: [PATCH 36/36] fix(cli): read --block-size into the setting -B sets --block-size=# was parsed and dropped, while the reference reads it into the same setting as -B# (zstdcli.c, `--block-size` with NEXT_TSIZE and `-B` into `blockSize`). So a benchmark or a training run given the long spelling cut nothing and fell back to whole files or 128 KiB samples. It now sets the block size as -B does, zero meaning none. Carries the_long_block_size_sets_what_b_sets, which failed before the change. The file shuffle used for training records that it is the reference's DiB_shuffle step for step. Part of #128 --- zstd/src/bin/structured-zstd/main.rs | 13 ++++++++++--- zstd/src/bin/structured-zstd/tests.rs | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 1508fa600..fd221d493 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -1191,9 +1191,12 @@ fn parse_args_into( } else if let Some(v) = option_text(long, "threads", arg_os, &mut iter)? { let _ = v.parse::().wrap_err("invalid --threads")?; } else if let Some(v) = option_text(long, "block-size", arg_os, &mut iter)? { - // The job size of a multi-threaded run: nothing here, - // but a malformed size is still a broken command line. - parse_size(&v).wrap_err("invalid --block-size")?; + // The long spelling of `-B#`: the reference reads both + // into one setting (zstdcli.c, `--block-size` and `-B`), + // which cuts the benchmark's frames and the training + // samples, and is a multi-threaded job size otherwise. + let size = parse_size(&v).wrap_err("invalid --block-size")?; + opts.block_size = (size != 0).then_some(size); } else if let Some(list) = option_value(long, "filelist", arg_os, &mut iter)? { opts.filelists.push(list); } else if let Some(dir) = @@ -3479,6 +3482,10 @@ struct TrainingSet { /// Reorder the sample files the way the reference does before loading /// (`DiB_shuffle`), so a sample set too large to load keeps a spread of files /// rather than the first ones, and the corpus is laid out as there. +/// +/// The same loop step for step (dibio.c, `DiB_shuffle` and `DiB_rand`, alike in +/// 1.5.7 and later): from the last position down to 1, swap with +/// `rand % (i + 1)`, the generator seeded with `0xFD2FB528`. fn shuffle_training_files(files: &mut [T]) { let mut seed: u32 = 0xFD2F_B528; let mut next = || { diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 959337860..f0b93ba85 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -2106,6 +2106,28 @@ fn block_size_is_read_like_the_reference_reads_it() { assert!(parse(&["-b", "-Bx", "f"]).is_err()); } +/// `--block-size=#` is the long spelling of `-B#`, as in the reference, which +/// reads both into the one setting the benchmark and the trainers cut by. +#[test] +fn the_long_block_size_sets_what_b_sets() { + assert_eq!( + parse(&["-b", "--block-size=64K", "f"]).unwrap().block_size, + Some(64 << 10) + ); + assert_eq!( + parse(&["-b", "--block-size", "4096", "f"]) + .unwrap() + .block_size, + Some(4096) + ); + assert_eq!( + parse(&["-b", "--block-size=0", "f"]).unwrap().block_size, + None, + "zero is no block size, as for -B0" + ); + assert!(parse(&["-b", "--block-size=x", "f"]).is_err()); +} + /// The benchmark compresses every input as frames of its own, cut into `-B` /// pieces from 32 bytes up, as the reference's block table does; an empty /// input yields no frame, and a smaller `-B` cuts nothing.