From 40108ce6001bcf1647e578780f0fd894372fa195 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 21:49:42 -0400 Subject: [PATCH 01/53] perf(j2k-native): decode bypass and terminated classic blocks on the packed passes Classic code-blocks coded with selective arithmetic coding bypass or per-pass termination, which split into several codeword segments, now take the packed-column Tier-1 passes, as single-segment blocks already did (context resets and segmentation symbols included, in either case). Raw (bypass) segments get packed cleanup, significance propagation and magnitude refinement passes that read the same bits as the per-coefficient raw passes, and the arithmetic segments of such blocks run the existing packed arithmetic passes. Only vertically causal blocks keep the per-coefficient passes. A test-only switch forces the per-coefficient passes. The parity test decodes blocks of all sixteen non-causal style combinations, sizes from 1x1 to 64x64 and corrupted copies of them both ways, and requires the same samples or the same error, in strict and lenient modes. Based on the prototype in #311. Co-authored-by: Justin Coyne --- crates/j2k-native/src/j2c/bitplane.rs | 2 + crates/j2k-native/src/j2c/bitplane/bypass.rs | 85 +++++++ crates/j2k-native/src/j2c/bitplane/facade.rs | 1 - .../src/j2c/bitplane/flag_passes.rs | 174 +++++++++++++- .../j2k-native/src/j2c/bitplane/schedule.rs | 79 ++++++- crates/j2k-native/src/j2c/bitplane/state.rs | 25 +- crates/j2k-native/src/j2c/bitplane/tests.rs | 56 ++--- crates/j2k-native/src/tests.rs | 1 + crates/j2k-native/src/tests/classic_passes.rs | 219 ++++++++++++++++++ 9 files changed, 595 insertions(+), 47 deletions(-) create mode 100644 crates/j2k-native/src/tests/classic_passes.rs diff --git a/crates/j2k-native/src/j2c/bitplane.rs b/crates/j2k-native/src/j2c/bitplane.rs index 7e326a71e..d46a85e49 100644 --- a/crates/j2k-native/src/j2c/bitplane.rs +++ b/crates/j2k-native/src/j2c/bitplane.rs @@ -16,6 +16,8 @@ pub(crate) use facade::{ decode, decode_code_block_segments_validated, decode_code_block_segments_validated_profiled, }; pub(crate) use observer::J2kBlockDecodeStats; +#[cfg(test)] +pub(crate) use state::FORCE_PER_COEFFICIENT; pub(crate) use state::{ classic_decode_workspace_bytes, BitPlaneDecodeBuffers, BitPlaneDecodeContext, Coefficient, CoefficientState, BITPLANE_BIT_SIZE, diff --git a/crates/j2k-native/src/j2c/bitplane/bypass.rs b/crates/j2k-native/src/j2c/bitplane/bypass.rs index 0695e3e88..e810d7075 100644 --- a/crates/j2k-native/src/j2c/bitplane/bypass.rs +++ b/crates/j2k-native/src/j2c/bitplane/bypass.rs @@ -46,6 +46,91 @@ impl BitDecoder for BypassDecoder<'_> { } } +/// Raw (bypass) bits for the packed-column passes: the bits of a raw +/// segment as [`BypassDecoder`] reads them, buffered up to 64 at a time. +/// +/// After a 0xFF byte, the next byte's top bit is a stuffed zero and is +/// skipped. Once the data is exhausted every bit reads as one. Strict +/// decoding fails when a stuffed bit is one or the data ends right after a +/// 0xFF byte; buffering stops before such a 0xFF byte, and the rest of the +/// segment is read by a [`BypassDecoder`], so those cases behave exactly as +/// it does. +pub(super) struct RawBits<'a> { + data: &'a [u8], + /// Next byte to buffer. + position: usize, + /// Buffered bits, most significant first. + buffer: u64, + available: u32, + /// Whether the byte at `position` follows a 0xFF byte. + stuffed: bool, + /// Bits from `position` on, once buffering has stopped before a 0xFF + /// byte whose stuffing [`BypassDecoder`] must check. + tail: Option>, + strict: bool, +} + +impl<'a> RawBits<'a> { + pub(super) fn new(data: &'a [u8], strict: bool) -> Self { + Self { + data, + position: 0, + buffer: 0, + available: 0, + stuffed: false, + tail: None, + strict, + } + } + + fn refill(&mut self) { + while self.available <= 56 && self.tail.is_none() { + let Some(&byte) = self.data.get(self.position) else { + return; + }; + let (bits, width) = if self.stuffed { + // The top bit was checked when the 0xFF byte was buffered. + (u64::from(byte & 0x7F), 7) + } else { + (u64::from(byte), 8) + }; + if byte == 0xFF { + let next = self.data.get(self.position + 1); + if next.is_none_or(|next| next & 0x80 != 0) { + let mut tail = BypassDecoder::new(&self.data[self.position..], self.strict); + if self.stuffed { + tail.0.read_bit(); + } + self.tail = Some(tail); + return; + } + } + self.buffer |= bits << (64 - width - self.available); + self.available += width; + self.stuffed = byte == 0xFF; + self.position += 1; + } + } + + #[expect(clippy::inline_always, reason = "per-bit raw decode in Tier-1 passes")] + #[inline(always)] + pub(super) fn read(&mut self) -> Option { + if self.available == 0 { + self.refill(); + if self.available == 0 { + // Past the end, bits read as one, as in `BypassDecoder`. + return self.tail.as_mut().map_or(Some(1), |tail| { + tail.read_bit(&mut ArithmeticDecoderContext::default()) + }); + } + } + let bit = (self.buffer >> 63) as u32; + self.buffer <<= 1; + self.available -= 1; + Some(bit) + } +} + pub(super) struct SafeScalarTier1; impl SafeScalarTier1 { diff --git a/crates/j2k-native/src/j2c/bitplane/facade.rs b/crates/j2k-native/src/j2c/bitplane/facade.rs index 63200b20c..16595b850 100644 --- a/crates/j2k-native/src/j2c/bitplane/facade.rs +++ b/crates/j2k-native/src/j2c/bitplane/facade.rs @@ -149,7 +149,6 @@ pub(super) fn decode_code_block_segments_validated_with_observer { + bits: &'r mut RawBits<'d>, + flags: &'c mut [u32], + coefficients: &'c mut [Coefficient], + width: usize, + height: usize, + stride: usize, + bit_position: u8, +} + +impl<'c, 'r, 'd> RawScan<'c, 'r, 'd> { + fn new(ctx: &'c mut BitPlaneDecodeContext, bits: &'r mut RawBits<'d>) -> Self { + Self { + bits, + width: ctx.width as usize, + height: ctx.height as usize, + stride: ctx.padded_width as usize, + bit_position: ctx.current_bit_position, + flags: &mut ctx.flags, + coefficients: &mut ctx.coefficients, + } + } + + fn stripe_start(&self, stripe: usize) -> (usize, usize, u32) { + let rows = (self.height - 4 * stripe).min(4); + ( + (stripe + 1) * self.stride + 1, + (4 * stripe + 1) * self.stride + 1, + ROW_BITS & ((1 << (3 * rows)) - 1), + ) + } + + #[expect(clippy::inline_always, reason = "Tier-1 coefficient-loop hot path")] + #[inline(always)] + fn become_significant( + &mut self, + own: &mut u32, + flag_index: usize, + top: usize, + shift: u32, + ) -> Option<()> { + let index = top + (shift / 3) as usize * self.stride; + self.coefficients[index].push_bit_at(1, self.bit_position); + let sign = self.bits.read()?; + self.coefficients[index].set_sign(u8::from(sign != 0)); + set_significant(self.flags, own, flag_index, self.stride, shift, sign); + Some(()) + } + + #[expect(clippy::inline_always, reason = "Tier-1 coefficient-loop hot path")] + #[inline(always)] + fn zero_code( + &mut self, + own: &mut u32, + flag_index: usize, + top: usize, + shift: u32, + ) -> Option { + let significant = self.bits.read()? != 0; + if significant { + self.become_significant(own, flag_index, top, shift)?; + } + Some(significant) + } +} + +/// The significance propagation pass of a raw (bypass) segment, with the +/// decisions and order of [`significance_propagation_pass_flags`]. +pub(super) fn significance_propagation_pass_raw( + ctx: &mut BitPlaneDecodeContext, + bits: &mut RawBits<'_>, +) -> Option<()> { + let mut scan = RawScan::new(ctx, bits); + for stripe in 0..scan.height.div_ceil(4) { + let (flag_row, top_row, rows) = scan.stripe_start(stripe); + for x in 0..scan.width { + let flag_index = flag_row + x; + let mut own = scan.flags[flag_index]; + let mut candidates = propagation_rows(own) & rows; + if candidates == 0 { + continue; + } + while candidates != 0 { + let shift = candidates.trailing_zeros(); + own |= PI_THIS << shift; + if scan.zero_code(&mut own, flag_index, top_row + x, shift)? { + candidates = propagation_rows(own) & rows & (u32::MAX << (shift + 1)); + } else { + candidates &= candidates - 1; + } + } + scan.flags[flag_index] = own; + } + } + Some(()) +} + +/// The magnitude refinement pass of a raw (bypass) segment, with the +/// decisions and order of [`magnitude_refinement_pass_flags`]. +pub(super) fn magnitude_refinement_pass_raw( + ctx: &mut BitPlaneDecodeContext, + bits: &mut RawBits<'_>, +) -> Option<()> { + let scan = RawScan::new(ctx, bits); + for stripe in 0..scan.height.div_ceil(4) { + let (flag_row, top_row, rows) = scan.stripe_start(stripe); + for x in 0..scan.width { + let flag_index = flag_row + x; + let mut own = scan.flags[flag_index]; + let mut candidates = + (own >> SIGMA_THIS.trailing_zeros()) & !(own >> PI_THIS.trailing_zeros()) & rows; + if candidates == 0 { + continue; + } + while candidates != 0 { + let shift = candidates.trailing_zeros(); + let bit = scan.bits.read()?; + scan.coefficients[top_row + x + (shift / 3) as usize * scan.stride] + .push_bit_at(bit, scan.bit_position); + own |= MU_THIS << shift; + candidates &= candidates - 1; + } + scan.flags[flag_index] = own; + } + } + Some(()) +} + +/// The cleanup pass of a raw segment, with the decisions and order of +/// [`cleanup_pass_flags`]. Selective bypass codes cleanup passes with the MQ +/// coder; a raw segment only reaches one when its pass counts say so. +pub(super) fn cleanup_pass_raw( + ctx: &mut BitPlaneDecodeContext, + bits: &mut RawBits<'_>, +) -> Option<()> { + let mut scan = RawScan::new(ctx, bits); + for stripe in 0..scan.height.div_ceil(4) { + let (flag_row, top_row, rows) = scan.stripe_start(stripe); + for x in 0..scan.width { + let flag_index = flag_row + x; + let top = top_row + x; + let mut own = scan.flags[flag_index]; + if rows == ROW_BITS && own == 0 { + if scan.bits.read()? == 0 { + continue; + } + let first = (scan.bits.read()? << 1) | scan.bits.read()?; + scan.become_significant(&mut own, flag_index, top, 3 * first)?; + for row in first + 1..4 { + scan.zero_code(&mut own, flag_index, top, 3 * row)?; + } + } else { + let mut candidates = uncoded_rows(own) & rows; + while candidates != 0 { + scan.zero_code(&mut own, flag_index, top, candidates.trailing_zeros())?; + candidates &= candidates - 1; + } + } + scan.flags[flag_index] = own & !PI_ALL; + } + } + Some(()) +} diff --git a/crates/j2k-native/src/j2c/bitplane/schedule.rs b/crates/j2k-native/src/j2c/bitplane/schedule.rs index 61d13ae95..68da6ea88 100644 --- a/crates/j2k-native/src/j2c/bitplane/schedule.rs +++ b/crates/j2k-native/src/j2c/bitplane/schedule.rs @@ -7,9 +7,11 @@ use super::arithmetic::{ cleanup_pass_arithmetic, magnitude_refinement_pass_arithmetic, significance_propagation_pass_arithmetic, }; -use super::bypass::{BitDecoder, BypassDecoder, SafeScalarTier1}; +use super::bypass::{BitDecoder, BypassDecoder, RawBits, SafeScalarTier1}; use super::flag_passes::{ - cleanup_pass_flags, magnitude_refinement_pass_flags, significance_propagation_pass_flags, + cleanup_pass_flags, cleanup_pass_raw, magnitude_refinement_pass_flags, + magnitude_refinement_pass_raw, significance_propagation_pass_flags, + significance_propagation_pass_raw, }; use super::observer::{J2kDecodeObserver, NoJ2kDecodeStats}; use super::state::{ @@ -115,11 +117,30 @@ fn decode_inner_with_observer( if use_arithmetic { let mut decoder = ArithmeticDecoder::new(data); - handle_arithmetic_coding_passes( + if ctx.uses_packed_columns() { + handle_packed_arithmetic_coding_passes( + start_coding_pass, + end_coding_pass, + ctx, + &mut decoder, + observer, + )?; + } else { + handle_arithmetic_coding_passes( + start_coding_pass, + end_coding_pass, + ctx, + &mut decoder, + observer, + )?; + } + } else if ctx.uses_packed_columns() { + let mut bits = RawBits::new(data, ctx.strict); + handle_packed_raw_coding_passes( start_coding_pass, end_coding_pass, ctx, - &mut decoder, + &mut bits, observer, )?; } else { @@ -272,6 +293,47 @@ fn handle_bypass_coding_passes( Some(()) } +/// The passes of a raw (bypass) segment on the packed columns, with the +/// decisions of [`handle_bypass_coding_passes`]. +fn handle_packed_raw_coding_passes( + start: u8, + end: u8, + ctx: &mut BitPlaneDecodeContext, + bits: &mut RawBits<'_>, + observer: &mut impl J2kDecodeObserver, +) -> Option<()> { + for coding_pass in start..end { + let phase_start = observer.phase_start(); + let current_bitplane = coding_pass.div_ceil(3); + ctx.current_bit_position = ctx.bitplanes - 1 - current_bitplane; + + match coding_pass % 3 { + 0 => { + cleanup_pass_raw(ctx, bits)?; + + if ctx.style.segmentation_symbols { + let symbol = [bits.read()?, bits.read()?, bits.read()?, bits.read()?]; + if symbol != [1, 0, 1, 0] && ctx.strict { + return None; + } + } + + ctx.reset_for_next_bitplane(); + } + 1 => significance_propagation_pass_raw(ctx, bits)?, + 2 => magnitude_refinement_pass_raw(ctx, bits)?, + _ => unreachable!(), + } + + if ctx.style.reset_context_probabilities { + ctx.reset_contexts(); + } + observer.add_bypass_us(phase_start); + } + + Some(()) +} + pub(super) fn decode_code_block_segments_inner( data: &[u8], segments: &[J2kCodeBlockSegment], @@ -315,6 +377,15 @@ pub(super) fn decode_code_block_segments_inner( observer, )?; } + } else if ctx.uses_packed_columns() { + let mut bits = RawBits::new(segment_data, ctx.strict); + handle_packed_raw_coding_passes( + start_coding_pass, + end_coding_pass, + ctx, + &mut bits, + observer, + )?; } else { let mut decoder = BypassDecoder::new(segment_data, ctx.strict); handle_bypass_coding_passes( diff --git a/crates/j2k-native/src/j2c/bitplane/state.rs b/crates/j2k-native/src/j2c/bitplane/state.rs index e0bf791c3..802285f81 100644 --- a/crates/j2k-native/src/j2c/bitplane/state.rs +++ b/crates/j2k-native/src/j2c/bitplane/state.rs @@ -92,6 +92,14 @@ fn include_capacity(bytes: &mut usize, capacity: usize) -> Result<()> { Ok(()) } +#[cfg(test)] +std::thread_local! { + /// Test-only: decode every block with the per-coefficient passes, to + /// compare the packed passes against them. + pub(crate) static FORCE_PER_COEFFICIENT: core::cell::Cell = + const { core::cell::Cell::new(false) }; +} + pub(crate) struct BitPlaneDecodeContext { /// A vector of bit-packed fields for each coefficient in the code-block. pub(super) coefficient_states: Vec, @@ -207,14 +215,12 @@ impl BitPlaneDecodeContext { code_block_style: &CodeBlockStyle, total_bitplanes: u8, strict: bool, - all_segments_arithmetic: bool, ) -> Result<()> { - // Packed and per-coefficient bookkeeping cannot be mixed in a block, - // so a raw (bypass) segment sends the whole block down the - // per-coefficient passes. - let packed_columns = all_segments_arithmetic - && code_block_style.single_codeword_segment() - && !code_block_style.vertically_causal_context; + // Raw (bypass) and terminated segments have packed passes too; only + // vertically causal contexts need the per-coefficient passes. + let packed_columns = !code_block_style.vertically_causal_context; + #[cfg(test)] + let packed_columns = packed_columns && !FORCE_PER_COEFFICIENT.get(); let layout = if packed_columns { workspace::StateLayout::PackedColumns } else { @@ -290,9 +296,6 @@ impl BitPlaneDecodeContext { code_block_style, total_bitplanes, strict, - // Codestream blocks take raw segments only under selective - // bypass, which already selects the per-coefficient passes. - true, ) } @@ -342,7 +345,7 @@ impl BitPlaneDecodeContext { // `zero_coding_scan_masks`, which is reset below. Raw bypass cleanup // also reads the per-coefficient marker and therefore still needs it // cleared explicitly. - if self.style.selective_arithmetic_coding_bypass { + if self.style.selective_arithmetic_coding_bypass && !self.packed_columns { let padded_width = self.padded_width as usize; let width = self.width as usize; let row_start = COEFFICIENTS_PADDING as usize; diff --git a/crates/j2k-native/src/j2c/bitplane/tests.rs b/crates/j2k-native/src/j2c/bitplane/tests.rs index 892b5d4be..d08527e04 100644 --- a/crates/j2k-native/src/j2c/bitplane/tests.rs +++ b/crates/j2k-native/src/j2c/bitplane/tests.rs @@ -537,22 +537,13 @@ fn packed_column_contexts_match_per_coefficient_contexts() { seed ^= seed << 5; seed }; + let _per_coefficient = PerCoefficient::force(); for (width, height) in [(1, 1), (3, 5), (7, 10), (9, 12), (16, 3)] { for density in [3_u32, 8, 14] { let mut ctx = BitPlaneDecodeContext::default(); let style = CodeBlockStyle::default(); - ctx.reset_for_job( - width, - height, - 0, - 1, - SubBandType::LowLow, - &style, - 8, - true, - false, - ) - .expect("per-coefficient context"); + ctx.reset_for_job(width, height, 0, 1, SubBandType::LowLow, &style, 8, true) + .expect("per-coefficient context"); let stride = ctx.padded_width as usize; let mut flags = vec![0_u32; (height.div_ceil(4) as usize + 2) * stride]; let flag_index = |x: usize, y: usize| (y / 4 + 1) * stride + x + 1; @@ -668,6 +659,7 @@ fn decode_through_both_pass_families( .expect("packed decode"); assert!(packed.uses_packed_columns()); + let _per_coefficient = PerCoefficient::force(); let mut per_coefficient = BitPlaneDecodeContext::default(); per_coefficient .reset_for_job( @@ -679,7 +671,6 @@ fn decode_through_both_pass_families( &style, 10, true, - false, ) .expect("per-coefficient context"); decode_code_block_segments_inner( @@ -772,9 +763,10 @@ fn first_magnitude_context_uses_neighbor_presence() { #[test] fn scan_unit_masks_track_significance_and_current_bitplane_zero_coding() { + let _per_coefficient = PerCoefficient::force(); let mut ctx = BitPlaneDecodeContext::default(); let style = CodeBlockStyle::default(); - ctx.reset_for_job(5, 6, 0, 4, SubBandType::LowLow, &style, 8, true, false) + ctx.reset_for_job(5, 6, 0, 4, SubBandType::LowLow, &style, 8, true) .expect("reset context"); let padded_width = ctx.padded_width as usize; @@ -805,18 +797,8 @@ fn scan_unit_masks_track_significance_and_current_bitplane_zero_coding() { selective_arithmetic_coding_bypass: true, ..CodeBlockStyle::default() }; - ctx.reset_for_job( - 5, - 6, - 0, - 4, - SubBandType::LowLow, - &bypass_style, - 8, - true, - false, - ) - .expect("reset bypass context"); + ctx.reset_for_job(5, 6, 0, 4, SubBandType::LowLow, &bypass_style, 8, true) + .expect("reset bypass context"); ctx.set_zero_coding_index(idx, y, padded_width); ctx.reset_for_next_bitplane(); assert_eq!(ctx.coefficient_states[idx].0 & HAS_ZERO_CODING_MASK, 0); @@ -883,9 +865,10 @@ fn classic_selective_bypass_round_trips_padded_rgb8_cb_block() { #[test] fn cleanup_candidate_mask_excludes_significant_and_zero_coded_coefficients() { + let _per_coefficient = PerCoefficient::force(); let mut ctx = BitPlaneDecodeContext::default(); let style = CodeBlockStyle::default(); - ctx.reset_for_job(5, 6, 0, 4, SubBandType::LowLow, &style, 8, true, false) + ctx.reset_for_job(5, 6, 0, 4, SubBandType::LowLow, &style, 8, true) .expect("reset context"); let padded_width = ctx.padded_width as usize; @@ -904,9 +887,10 @@ fn cleanup_candidate_mask_excludes_significant_and_zero_coded_coefficients() { #[test] fn cleanup_run_length_rejects_a_zero_coded_coefficient_without_significant_neighbors() { + let _per_coefficient = PerCoefficient::force(); let mut ctx = BitPlaneDecodeContext::default(); let style = CodeBlockStyle::default(); - ctx.reset_for_job(1, 4, 0, 4, SubBandType::LowLow, &style, 8, true, false) + ctx.reset_for_job(1, 4, 0, 4, SubBandType::LowLow, &style, 8, true) .expect("reset context"); let padded_width = ctx.padded_width as usize; @@ -964,3 +948,19 @@ fn classic_bitplane_round_trips_subband_and_style_matrix() { } } } + +/// Decodes on this thread use the per-coefficient passes while it lives. +struct PerCoefficient; + +impl PerCoefficient { + fn force() -> Self { + super::state::FORCE_PER_COEFFICIENT.set(true); + Self + } +} + +impl Drop for PerCoefficient { + fn drop(&mut self) { + super::state::FORCE_PER_COEFFICIENT.set(false); + } +} diff --git a/crates/j2k-native/src/tests.rs b/crates/j2k-native/src/tests.rs index b1e7c0ee6..3c5e10a36 100644 --- a/crates/j2k-native/src/tests.rs +++ b/crates/j2k-native/src/tests.rs @@ -4,6 +4,7 @@ use super::*; use crate::j2c::ComponentData; mod banded_tier1; +mod classic_passes; mod component_layout; mod fused_last_level; mod ht_refinement; diff --git a/crates/j2k-native/src/tests/classic_passes.rs b/crates/j2k-native/src/tests/classic_passes.rs new file mode 100644 index 000000000..18544c12e --- /dev/null +++ b/crates/j2k-native/src/tests/classic_passes.rs @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Code-blocks with selective bypass, per-pass termination, context resets +//! or segmentation symbols decode on the packed-column passes, raw segments +//! included. They must give the per-coefficient passes' samples bit for bit, +//! and the same outcome on corrupt data, strict or not. + +use alloc::vec::Vec; + +use crate::j2c::bitplane::FORCE_PER_COEFFICIENT; +use crate::{ + decode_j2k_code_block_scalar, encode_j2k_code_block_scalar_with_style, J2kCodeBlockDecodeJob, + J2kCodeBlockSegment, J2kCodeBlockStyle, J2kSubBandType, +}; + +/// Xorshift, for reproducible blocks and mutations. +struct Random(u64); + +impl Random { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + /// A value below `bound`. + fn below(&mut self, bound: usize) -> usize { + usize::try_from(self.next() % u64::try_from(bound).expect("bound")).expect("below bound") + } +} + +struct Block { + data: Vec, + segments: Vec, + width: u32, + height: u32, + missing_bit_planes: u8, + passes: u8, + total_bitplanes: u8, + sub_band_type: J2kSubBandType, + style: J2kCodeBlockStyle, +} + +/// Mostly small magnitudes, as in wavelet sub-bands, `density` in 16 of +/// them nonzero, below `2^total_bitplanes`. +fn coefficients( + random: &mut Random, + samples: u32, + density: usize, + total_bitplanes: u8, +) -> Vec { + let limit = 1_usize << total_bitplanes; + (0..samples) + .map(|_| { + if random.below(16) >= density { + return 0; + } + let magnitude = random.below(limit) >> random.below(usize::from(total_bitplanes)); + let magnitude = i32::try_from(magnitude.min(limit - 1)).expect("magnitude"); + if random.below(2) == 0 { + magnitude + } else { + -magnitude + } + }) + .collect() +} + +/// The samples' bits, or the error, decoding `data` for `block` on the +/// per-coefficient passes or not. +fn decode( + block: &Block, + data: &[u8], + strict: bool, + per_coefficient: bool, +) -> Result, String> { + let mut output = alloc::vec![0.0_f32; (block.width * block.height) as usize]; + FORCE_PER_COEFFICIENT.set(per_coefficient); + let result = decode_j2k_code_block_scalar( + J2kCodeBlockDecodeJob { + data, + segments: &block.segments, + width: block.width, + height: block.height, + output_stride: block.width as usize, + missing_bit_planes: block.missing_bit_planes, + number_of_coding_passes: block.passes, + total_bitplanes: block.total_bitplanes, + roi_shift: 0, + sub_band_type: block.sub_band_type, + style: block.style, + strict, + dequantization_step: 1.0, + }, + &mut output, + ); + FORCE_PER_COEFFICIENT.set(false); + result + .map(|()| output.iter().map(|sample| sample.to_bits()).collect()) + .map_err(|error| alloc::format!("{error:?}")) +} + +/// A copy of `data` with a few bits flipped or bytes set to 0xFF, sometimes +/// truncated. +fn mutate(random: &mut Random, data: &[u8]) -> Vec { + let mut data = data.to_vec(); + for _ in 0..=random.below(3) { + let at = random.below(data.len()); + if random.below(4) == 0 { + data[at] = 0xFF; + } else { + data[at] ^= 1 << random.below(8); + } + } + if random.below(5) == 0 { + data.truncate(random.below(data.len())); + } + data +} + +/// Decode `block` and corrupt copies of it both ways; returns how many +/// corrupt decodes failed. +fn check(random: &mut Random, block: &Block, label: &str) -> usize { + for strict in [false, true] { + let packed = decode(block, &block.data, strict, false); + assert!(packed.is_ok(), "{label}: {packed:?}"); + assert_eq!(packed, decode(block, &block.data, strict, true), "{label}"); + } + let mut failures = 0; + if block.data.is_empty() { + return failures; + } + for mutant in 0..24 { + let data = mutate(random, &block.data); + for strict in [false, true] { + let per_coefficient = decode(block, &data, strict, true); + failures += usize::from(per_coefficient.is_err()); + assert_eq!( + decode(block, &data, strict, false), + per_coefficient, + "{label} mutant={mutant} strict={strict}" + ); + } + } + failures +} + +#[test] +fn packed_passes_match_per_coefficient_passes_for_every_non_causal_style() { + let mut random = Random(0x9e37_79b9_7f4a_7c15); + let sub_bands = [ + J2kSubBandType::LowLow, + J2kSubBandType::HighLow, + J2kSubBandType::LowHigh, + J2kSubBandType::HighHigh, + ]; + let sizes = [ + (1, 1), + (3, 5), + (4, 4), + (7, 13), + (16, 16), + (33, 17), + (64, 8), + (64, 64), + ]; + let (mut blocks, mut raw_blocks, mut failures) = (0, 0, 0); + for styles in 0..16_u32 { + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: styles & 1 != 0, + reset_context_probabilities: styles & 2 != 0, + termination_on_each_pass: styles & 4 != 0, + vertically_causal_context: false, + segmentation_symbols: styles & 8 != 0, + }; + for (width, height) in sizes { + for density in [1, 4, 12] { + // Raw coding starts after the fourth bitplane. + let total_bitplanes = 1 + u8::try_from(random.below(16)).expect("bitplanes"); + let coefficients = + coefficients(&mut random, width * height, density, total_bitplanes); + let sub_band_type = sub_bands[random.below(4)]; + let encoded = encode_j2k_code_block_scalar_with_style( + &coefficients, + width, + height, + sub_band_type, + total_bitplanes, + style, + ) + .expect("encode code-block"); + let block = Block { + data: encoded.data, + segments: encoded.segments, + width, + height, + missing_bit_planes: encoded.missing_bit_planes, + passes: encoded.number_of_coding_passes, + total_bitplanes, + sub_band_type, + style, + }; + blocks += 1; + raw_blocks += + usize::from(block.segments.iter().any(|segment| !segment.use_arithmetic)); + let label = alloc::format!( + "style={styles:04b} {width}x{height} density={density} bitplanes={total_bitplanes}" + ); + failures += check(&mut random, &block, &label); + } + } + } + assert!( + raw_blocks > blocks / 4, + "{raw_blocks} of {blocks} blocks had raw segments" + ); + assert!(failures > 100, "{failures} corrupt decodes failed"); +} From f9580da6a495fc6e2a698fef84b1f230bcfcaec0 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 21:49:42 -0400 Subject: [PATCH 02/53] docs(changelog): note BYPASS and terminated classic blocks on the packed passes --- CHANGELOG.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbf717479..eea32aafc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ and stale roadmap entries have been removed from the public documentation set. ## [Unreleased] +- Classic code-blocks with BYPASS, per-pass termination, context resets or + segmentation symbols use the packed-column passes, including raw segments. + Vertically causal blocks retain the per-coefficient passes. Clean and corrupt + code-blocks preserve the same samples and errors in strict and lenient modes. +- Decodes into a caller buffer whose stride is wider than a row and which + ends right after the last row's pixels, the length `strided_output_len` + reports and buffer validation accepts, now write that last row. Full-image + and scaled 8-bit decodes at such a stride and every 16-bit decode at one, + regions included, left it unwritten and still returned `Ok`. 8-bit region + decodes, and buffers with any bytes after the last row, were not affected. + The other rows are unchanged. - Parallel CPU region decodes on the row-streamed IDWT decode Tier-1 band by band on the worker pool while the IDWT reads the bands, instead of decoding every code-block of a component first. Each sub-band holds three bands at @@ -15,13 +26,6 @@ and stale roadmap entries have been removed from the public documentation set. 18000 x 12000 RGB image decodes 3 to 11% faster at 12 threads and 9 to 18% faster at 4, with about 3 MiB less peak heap. Output is unchanged, and a failure reruns whole-window Tier-1 serially for the same error. -- Decodes into a caller buffer whose stride is wider than a row and which - ends right after the last row's pixels, the length `strided_output_len` - reports and buffer validation accepts, now write that last row. Full-image - and scaled 8-bit decodes at such a stride and every 16-bit decode at one, - regions included, left it unwritten and still returned `Ok`. 8-bit region - decodes, and buffers with any bytes after the last row, were not affected. - The other rows are unchanged. ## [0.13.1] - 2026-10-29 From 73b993daeac992c42f58de4e50eccde0b1109abe Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:36:22 -0400 Subject: [PATCH 03/53] perf(j2k-native): decode runs of run-length zeros in one step In the classic cleanup pass, a column of four insignificant, unvisited coefficients decodes one run-length decision. While that context's MPS is zero and the MQ decoder needs no renormalization, each such decision returns zero and changes nothing but A, which drops by Qe. From the current registers, ArithmeticDecoder::mps_run gives how many decisions stay on that path (A - k * Qe at least 0x8000 and above the upper half of C), and the pass skips that many consecutive all-zero columns with one subtraction. Decisions, registers and context states are exactly those of decoding each column; a unit test compares the registers after every skip with per-decision decoding over adapting and fixed contexts. Classic full-image decode, interleaved A/B against the parent: 2 bpp -6.7% at 1 thread, -5.8% at 6, -6.4% at 18; lossless -2.3% at 1 thread and -3.5% at 18. Output is byte-identical. --- .../j2k-native/src/j2c/arithmetic_decoder.rs | 116 +++++++++++++++++- .../src/j2c/bitplane/flag_passes.rs | 24 +++- 2 files changed, 138 insertions(+), 2 deletions(-) diff --git a/crates/j2k-native/src/j2c/arithmetic_decoder.rs b/crates/j2k-native/src/j2c/arithmetic_decoder.rs index f698f937a..40eb9e3b7 100644 --- a/crates/j2k-native/src/j2c/arithmetic_decoder.rs +++ b/crates/j2k-native/src/j2c/arithmetic_decoder.rs @@ -194,6 +194,36 @@ impl<'a> ArithmeticDecoder<'a> { d } + /// How many decisions in `context`, from the current registers, take the + /// MPS path of C.3.2 without renormalizing: each leaves C and the + /// context's state as they are and lowers A by Qe, which holds while + /// A - k * Qe stays at least 0x8000 and above the upper half of C. + #[expect( + clippy::cast_possible_truncation, + clippy::inline_always, + reason = "the upper half of C is 16 bits; checked on a measured per-column hot path" + )] + #[inline(always)] + pub(crate) fn mps_run(&self, context: ArithmeticDecoderContext) -> u32 { + let qe = PACKED_DECODER_STATES[context.0 as usize] as u32 & 0xffff; + let floor = ((self.c >> 48) as u32 + 1).max(0x8000); + self.a.saturating_sub(floor) / qe + } + + /// Decode `count` decisions in `context` that [`Self::mps_run`] covers: + /// all of them yield the MPS and only A changes. + #[expect( + clippy::cast_possible_truncation, + clippy::inline_always, + reason = "Qe is the low 16 bits of the packed state; per-column hot path" + )] + #[inline(always)] + pub(crate) fn skip_mps_run(&mut self, context: ArithmeticDecoderContext, count: u32) { + debug_assert!(count <= self.mps_run(context)); + let qe = PACKED_DECODER_STATES[context.0 as usize] as u32 & 0xffff; + self.a -= count * qe; + } + #[expect( clippy::inline_always, reason = "MQ state transitions are measured per-symbol hot paths" @@ -251,7 +281,6 @@ impl ArithmeticDecoderContext { clippy::inline_always, reason = "MQ state transitions are measured per-symbol hot paths" )] - #[cfg(test)] #[inline(always)] pub(crate) fn mps(self) -> u32 { u32::from(self.0 >> 7) @@ -275,3 +304,88 @@ impl ArithmeticDecoderContext { self.0 = index; } } + +#[cfg(test)] +mod tests { + use super::{ArithmeticDecoder, ArithmeticDecoderContext}; + use crate::j2c::arithmetic_encoder::{ArithmeticEncoder, ArithmeticEncoderContext}; + use alloc::vec::Vec; + + /// Symbols of one context that are 1 about once in `one_in`. + fn sparse_symbols(count: usize, one_in: u32, seed: u32) -> Vec { + let mut state = seed; + (0..count) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + u32::from((state >> 8).is_multiple_of(one_in)) + }) + .collect() + } + + fn registers(decoder: &ArithmeticDecoder<'_>) -> (u64, u32, u32, u32) { + ( + decoder.c, + decoder.a, + decoder.base_pointer, + decoder.shift_count, + ) + } + + #[test] + fn skipping_mps_runs_matches_decoding_each_decision() { + for (one_in, start_index, seed) in [ + (40, 3, 1), + (300, 3, 2), + (7, 0, 3), + (2000, 0, 4), + (3, 3, 5), + (5, 46, 6), + ] { + let symbols = sparse_symbols(20_000, one_in, seed); + let mut encoder = ArithmeticEncoder::new(); + let mut encoder_context = ArithmeticEncoderContext::default(); + encoder_context.reset_with_index(start_index); + for &symbol in &symbols { + encoder.encode(symbol, &mut encoder_context); + } + let encoded = encoder.finish(); + + let mut each = ArithmeticDecoder::new(&encoded); + let mut each_context = ArithmeticDecoderContext::empty(); + each_context.reset_with_index(start_index); + let mut runs = ArithmeticDecoder::new(&encoded); + let mut runs_context = each_context; + let mut skipped = 0usize; + let mut position = 0; + while position < symbols.len() { + let run = if runs_context.mps() == 0 { + runs.mps_run(runs_context) + } else { + 0 + }; + if run == 0 { + assert_eq!(runs.read_bit(&mut runs_context), symbols[position]); + assert_eq!(each.read_bit(&mut each_context), symbols[position]); + position += 1; + } else { + let count = run.min(u32::try_from(symbols.len() - position).expect("fits")); + for _ in 0..count { + let before = each_context.0; + assert_eq!(each.read_bit(&mut each_context), 0); + assert_eq!(each_context.0, before, "a skipped decision changed state"); + assert_eq!(symbols[position], 0); + position += 1; + } + runs.skip_mps_run(runs_context, count); + skipped += count as usize; + } + assert_eq!(registers(&runs), registers(&each)); + assert_eq!(runs_context.0, each_context.0); + } + // State 46 (the uniform context) never adapts, so its runs are short. + if one_in >= 40 { + assert!(skipped > 0, "no run skipped for one_in {one_in}"); + } + } + } +} diff --git a/crates/j2k-native/src/j2c/bitplane/flag_passes.rs b/crates/j2k-native/src/j2c/bitplane/flag_passes.rs index fb694b6aa..e7657ad28 100644 --- a/crates/j2k-native/src/j2c/bitplane/flag_passes.rs +++ b/crates/j2k-native/src/j2c/bitplane/flag_passes.rs @@ -203,13 +203,34 @@ pub(super) fn cleanup_pass_flags( let mut uniform = scan.contexts[18]; for stripe in 0..scan.height.div_ceil(4) { let (flag_row, top_row, rows) = scan.stripe_start(stripe); - for x in 0..scan.width { + let mut x = 0; + while x < scan.width { let flag_index = flag_row + x; let top = top_row + x; let mut own = scan.flags[flag_index]; // A zero word is a full column of insignificant, unvisited // coefficients with zero contexts: Annex D's run-length case. if rows == ROW_BITS && own == 0 { + // While the run-length context's MPS is a zero and it decodes + // without renormalizing, each zero word from here on decodes + // that zero and changes nothing but A, so the run of such + // words is decoded at once. + let run = if run_length.mps() == 0 { + scan.decoder.mps_run(run_length) + } else { + 0 + }; + if run != 0 { + let limit = run.min(u32::try_from(scan.width - x).unwrap_or(u32::MAX)); + let mut columns = 1; + while columns < limit && scan.flags[flag_index + columns as usize] == 0 { + columns += 1; + } + scan.decoder.skip_mps_run(run_length, columns); + x += columns as usize; + continue; + } + x += 1; if scan.decoder.read_bit(&mut run_length) == 0 { continue; } @@ -220,6 +241,7 @@ pub(super) fn cleanup_pass_flags( scan.zero_code(&mut own, flag_index, top, 3 * row); } } else { + x += 1; let mut candidates = uncoded_rows(own) & rows; while candidates != 0 { scan.zero_code(&mut own, flag_index, top, candidates.trailing_zeros()); From f37ed3719d831b50374110d8d49981ce139db47d Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 01:05:24 -0400 Subject: [PATCH 04/53] perf(j2k-native): convert exact reversible code-block rows with a select When a reversible code-block's passes decode down to bit-plane 0, so that Equation E-8 adds no midpoint, with no ROI shift and at most 31 coded bit-planes, every coefficient converts as decoded: a magnitude below 2^31 converts exactly through u32, and negation commutes with the rounding of the conversion and of the product, so the sign is a bit flip. Such rows now convert in a branch-free u32-to-f32 loop that vectorizes, on the tile route and on the serial per-sub-band path, instead of reconstructing each coefficient through i64. A unit test checks the row against the per-coefficient reconstruction for exact and inexact blocks. Classic lossless full-image decode, interleaved A/B against the parent: -5.6% at 1 thread, -4.6% at 6, -6.7% at 18, -5.0% at the default count. 2 bpp (irreversible) is unchanged. Output is byte-identical. --- crates/j2k-native/src/j2c/bitplane.rs | 1 + .../src/j2c/bitplane/reconstruction.rs | 35 +++++++++++++ crates/j2k-native/src/j2c/bitplane/tests.rs | 51 +++++++++++++++++++ .../src/j2c/decode/subband/classic.rs | 9 ++++ .../j2k-native/src/scalar/classic_decode.rs | 9 ++++ 5 files changed, 105 insertions(+) diff --git a/crates/j2k-native/src/j2c/bitplane.rs b/crates/j2k-native/src/j2c/bitplane.rs index d46a85e49..6bdd2d7c1 100644 --- a/crates/j2k-native/src/j2c/bitplane.rs +++ b/crates/j2k-native/src/j2c/bitplane.rs @@ -16,6 +16,7 @@ pub(crate) use facade::{ decode, decode_code_block_segments_validated, decode_code_block_segments_validated_profiled, }; pub(crate) use observer::J2kBlockDecodeStats; +pub(crate) use reconstruction::MidpointReconstructor; #[cfg(test)] pub(crate) use state::FORCE_PER_COEFFICIENT; pub(crate) use state::{ diff --git a/crates/j2k-native/src/j2c/bitplane/reconstruction.rs b/crates/j2k-native/src/j2c/bitplane/reconstruction.rs index e12a40e34..0cb0f0258 100644 --- a/crates/j2k-native/src/j2c/bitplane/reconstruction.rs +++ b/crates/j2k-native/src/j2c/bitplane/reconstruction.rs @@ -122,6 +122,41 @@ impl MidpointReconstructor { } } + /// Whether [`Self::reconstruct_reversible`] returns every coefficient as + /// decoded, without a midpoint offset, and at most 31 bit-planes are + /// coded, so magnitudes are below 2^31. + pub(crate) fn reversible_is_exact(self) -> bool { + self.decoded_bitplanes <= 31 + && self + .fast + .is_none_or(|(lowest, final_sigprop)| lowest == 0 && !final_sigprop) + } + + /// `output[i] = reconstruct_reversible(c[i]) as f32 * step` for as many + /// elements as both slices hold, for a block whose reconstruction + /// [`Self::reversible_is_exact`]. A magnitude below 2^31 converts exactly + /// through `u32`, and negation commutes with the rounding of both the + /// conversion and the product, so the sign is a bit flip and the row + /// vectorizes. + #[expect( + clippy::cast_possible_truncation, + clippy::cast_precision_loss, + reason = "magnitudes are below 2^31 when the reconstruction is exact" + )] + pub(crate) fn scale_exact_reversible_row( + coefficients: &[Coefficient], + output: &mut [f32], + step: f32, + ) { + for (sample, &coefficient) in output.iter_mut().zip(coefficients) { + let bits = coefficient.to_bits(); + let magnitude = bits as u32; + let negative = (bits >> 63) as u32; + let value = f32::from_bits((magnitude as f32 * step).to_bits() ^ (negative << 31)); + *sample = if magnitude == 0 { 0.0 } else { value }; + } + } + /// Reversible (5/3) reconstruction: the per-block form of /// [`reversible_midpoint_bit`](j2k_codec_math::classic::reversible_midpoint_bit), /// which the CUDA and Metal classic decoders apply too. A fully decoded diff --git a/crates/j2k-native/src/j2c/bitplane/tests.rs b/crates/j2k-native/src/j2c/bitplane/tests.rs index d08527e04..fbc264325 100644 --- a/crates/j2k-native/src/j2c/bitplane/tests.rs +++ b/crates/j2k-native/src/j2c/bitplane/tests.rs @@ -194,6 +194,57 @@ fn irreversible_midpoint_reconstruction_tracks_the_last_decoded_pass() { ); } +#[test] +#[expect( + clippy::cast_precision_loss, + reason = "the reference converts the reconstructed integer as the scalar output does" +)] +fn exact_reversible_rows_match_per_coefficient_reconstruction() { + for decoded_bitplanes in [1_u8, 2, 8, 17, 31, 32] { + let top = (1_u64 << decoded_bitplanes) - 1; + let mut magnitudes = vec![0, 1, 2, 3, top, top >> 1, top & 0x5555_5555]; + magnitudes.extend((0..decoded_bitplanes).map(|bit| 1_u64 << bit)); + let coefficients = magnitudes + .iter() + .flat_map(|&magnitude| { + [0_u8, 1].map(|sign| { + let mut coefficient = Coefficient::default(); + for bit in 0..63 { + coefficient.push_bit_at(u32::from(magnitude >> bit & 1 == 1), bit); + } + coefficient.set_sign(u8::from(magnitude != 0) & sign); + coefficient + }) + }) + .collect::>(); + let max_passes = 1 + 3 * (decoded_bitplanes - 1); + let mut exact_blocks = 0; + for number_of_coding_passes in 0..=max_passes { + let block = MidpointReconstructor::new(decoded_bitplanes, number_of_coding_passes, 0); + if !block.reversible_is_exact() { + continue; + } + exact_blocks += 1; + for step in [1.0_f32, 0.5, 3.0, 0.1] { + let mut row = vec![f32::NAN; coefficients.len()]; + MidpointReconstructor::scale_exact_reversible_row(&coefficients, &mut row, step); + for (value, &coefficient) in row.iter().zip(&coefficients) { + assert_eq!( + value.to_bits(), + (block.reconstruct_reversible(coefficient) as f32 * step).to_bits(), + "bitplanes {decoded_bitplanes} passes {number_of_coding_passes} \ + step {step} coefficient {:#x}", + coefficient.to_bits() + ); + } + } + } + // Blocks through the last plane's refinement, and blocks without + // passes, add no midpoint; 32 coded planes take the general path. + assert_eq!(exact_blocks == 0, decoded_bitplanes > 31); + } +} + #[test] fn reversible_reconstruction_is_exact_when_complete_and_centered_when_stopped_early() { let coefficient = |magnitude: u64, sign: u8| { diff --git a/crates/j2k-native/src/j2c/decode/subband/classic.rs b/crates/j2k-native/src/j2c/decode/subband/classic.rs index af12da2b5..0cdf67373 100644 --- a/crates/j2k-native/src/j2c/decode/subband/classic.rs +++ b/crates/j2k-native/src/j2c/decode/subband/classic.rs @@ -332,6 +332,9 @@ pub(super) fn decode_sub_band_classic_blocks( code_block.number_of_coding_passes, component_info.roi_shift, ); + let exact = !irreversible_midpoint + && component_info.roi_shift == 0 + && midpoint.reversible_is_exact(); placement.store_rows( tile_ctx.bit_plane_decode_context.coefficient_rows(), base_store, @@ -339,6 +342,12 @@ pub(super) fn decode_sub_band_classic_blocks( |coefficients, out_row| { if irreversible_midpoint { midpoint.dequantize_row(coefficients, out_row, dequantization_step); + } else if exact { + bitplane::MidpointReconstructor::scale_exact_reversible_row( + coefficients, + out_row, + dequantization_step, + ); } else { for (output, coefficient) in out_row.iter_mut().zip(coefficients) { *output = apply_roi_maxshift_inverse_i64( diff --git a/crates/j2k-native/src/scalar/classic_decode.rs b/crates/j2k-native/src/scalar/classic_decode.rs index 6817f176c..4e5d7ed1c 100644 --- a/crates/j2k-native/src/scalar/classic_decode.rs +++ b/crates/j2k-native/src/scalar/classic_decode.rs @@ -148,6 +148,7 @@ fn write_j2k_code_block_output( ) { let midpoint = decode_context.midpoint_reconstructor(job.number_of_coding_passes, job.roi_shift); + let exact = !irreversible_midpoint && job.roi_shift == 0 && midpoint.reversible_is_exact(); for (row_idx, coeff_row) in decode_context .coefficient_rows() .enumerate() @@ -159,6 +160,14 @@ fn write_j2k_code_block_output( midpoint.dequantize_row(coeff_row, output_row, job.dequantization_step); continue; } + if exact { + j2c::bitplane::MidpointReconstructor::scale_exact_reversible_row( + coeff_row, + output_row, + job.dequantization_step, + ); + continue; + } for (coefficient, sample) in coeff_row.iter().zip(output_row.iter_mut()) { *sample = apply_roi_maxshift_inverse_i64( midpoint.reconstruct_reversible(*coefficient), From cd57f84748bef4209c7e0781a7bd9272afc26649 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 01:05:24 -0400 Subject: [PATCH 05/53] perf(j2k-native): form sign contexts before the significance decision In the classic significance propagation and cleanup passes, a coefficient's sign is decoded right after the decision that makes it significant, a branch the data makes hard to predict. Its sign label (from the neighbors' flags) and context are now formed and loaded before the significance decision, so after a mispredicted significance the sign decode no longer waits for them. Sign labels (9-13) never equal zero-coding labels (0-8), so the significance decision cannot change the context loaded early. Classic full-image decode, interleaved A/B against the parent, for the significance propagation pass: lossless -7.3% at 1 thread, -7.1% at 6, -4.6% at 18; 2 bpp -3.0% at 1 thread, -3.1% at 6, -2.6% at 18. The cleanup pass adds another 1-2% at 1 and 6 threads (2 bpp -1.3% to -2.1%, lossless -0.8% to -1.2%) and is within noise at 18. Output is byte-identical. --- .../j2k-native/src/j2c/arithmetic_decoder.rs | 6 +++--- .../src/j2c/bitplane/flag_passes.rs | 20 +++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/crates/j2k-native/src/j2c/arithmetic_decoder.rs b/crates/j2k-native/src/j2c/arithmetic_decoder.rs index 40eb9e3b7..c9f5f344a 100644 --- a/crates/j2k-native/src/j2c/arithmetic_decoder.rs +++ b/crates/j2k-native/src/j2c/arithmetic_decoder.rs @@ -348,12 +348,12 @@ mod tests { for &symbol in &symbols { encoder.encode(symbol, &mut encoder_context); } - let encoded = encoder.finish(); + let bytes = encoder.finish(); - let mut each = ArithmeticDecoder::new(&encoded); + let mut each = ArithmeticDecoder::new(&bytes); let mut each_context = ArithmeticDecoderContext::empty(); each_context.reset_with_index(start_index); - let mut runs = ArithmeticDecoder::new(&encoded); + let mut runs = ArithmeticDecoder::new(&bytes); let mut runs_context = each_context; let mut skipped = 0usize; let mut position = 0; diff --git a/crates/j2k-native/src/j2c/bitplane/flag_passes.rs b/crates/j2k-native/src/j2c/bitplane/flag_passes.rs index e7657ad28..bfd52683c 100644 --- a/crates/j2k-native/src/j2c/bitplane/flag_passes.rs +++ b/crates/j2k-native/src/j2c/bitplane/flag_passes.rs @@ -111,13 +111,29 @@ impl<'c, 'd> Scan<'c, 'd> { } /// Zero-code the insignificant coefficient in the row at window shift - /// `shift`; returns whether it became significant. + /// `shift`; returns whether it became significant. Its sign context is + /// formed and loaded before the significance decision, so after a + /// mispredicted significance the sign decode does not wait for them. Sign + /// labels (9-13) never equal zero-coding labels (0-8), so the decision + /// does not change the context loaded early. #[expect(clippy::inline_always, reason = "Tier-1 coefficient-loop hot path")] #[inline(always)] fn zero_code(&mut self, own: &mut u32, flag_index: usize, top: usize, shift: u32) -> bool { + let (label, xor) = sign_coding_label( + *own, + self.flags[flag_index - 1], + self.flags[flag_index + 1], + shift, + ); + let mut sign_context = self.contexts[usize::from(label)]; let significant = self.read(zero_coding_label(self.zero_coding, *own, shift)) != 0; if significant { - self.become_significant(own, flag_index, top, shift); + let index = top + (shift / 3) as usize * self.stride; + self.coefficients[index].push_bit_at(1, self.bit_position); + let sign = self.decoder.read_bit(&mut sign_context) ^ u32::from(xor); + self.contexts[usize::from(label)] = sign_context; + self.coefficients[index].set_sign(u8::from(sign != 0)); + set_significant(self.flags, own, flag_index, self.stride, shift, sign); } significant } From d72aaa146690ab8bb8687ea28e6a26daa62ebb9c Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 01:05:24 -0400 Subject: [PATCH 06/53] docs(changelog): note the classic Tier-1 decode speedups --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eea32aafc..f531b7d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,17 @@ and stale roadmap entries have been removed from the public documentation set. 18000 x 12000 RGB image decodes 3 to 11% faster at 12 threads and 9 to 18% faster at 4, with about 3 MiB less peak heap. Output is unchanged, and a failure reruns whole-window Tier-1 serially for the same error. +- Classic (EBCOT) code-block decoding is faster on the CPU. The cleanup pass + decodes a run of all-zero run-length columns in one step while the MQ + decoder needs no renormalization, the significance propagation and cleanup + passes form a coefficient's sign context before its significance decision, + and reversible code-blocks decoded down to bit-plane 0 convert to samples + in a vectorized loop. On the tile-wide route, long classic code-block rows + split into parts of about equal work, so a decode no longer ends on one + long row. A 4096 x 2048 RGB image decodes 10% faster at 2 bpp and 14% + faster lossless at 1 and 6 threads (5% and 13% at 18), and 1024 x 1024 + regions and reduced-resolution decodes of an 18000 x 12000 image 2 to 10% + faster. Output and peak heap are unchanged. ## [0.13.1] - 2026-10-29 From 6e7bd41e8f6b409411843cebde920ccd0222f7e3 Mon Sep 17 00:00:00 2001 From: GF Date: Thu, 29 Oct 2026 23:58:18 -0400 Subject: [PATCH 07/53] perf(j2k-native): merge position-major progressions from their ordered runs instead of sorting RPCL and CPRL list each component's and resolution's precincts in raster order, which is already their progression order whenever the reference-grid positions increase. Interleave those runs per leading-key group (resolution for RPCL, component for CPRL) instead of sorting every precinct of the tile. A tile whose runs are not strictly ordered, and PCRL, sort as before; the merge of sorted runs under the same total order yields the same packets. Measured with the next commit (PLT run skip) against base, on an 18-core ARM64 machine, 5 interleaved rounds, median of per-round ratios: on the 18000 x 12000 test images: the HTJ2K one reduced four levels -7.7% on 18 threads; regions of the PLT one -3.5% (bottom-right), -1.9% (top-left), -1.4% (middle) on 18 threads and -2.1% (middle) on 6; the no-PLT one's top-left region -2.4% on 18; one-thread rows within 1%. --- crates/j2k-native/src/j2c/progression.rs | 268 ++++++++++++++++++++--- 1 file changed, 235 insertions(+), 33 deletions(-) diff --git a/crates/j2k-native/src/j2c/progression.rs b/crates/j2k-native/src/j2c/progression.rs index ec23d5213..040e4cf5a 100644 --- a/crates/j2k-native/src/j2c/progression.rs +++ b/crates/j2k-native/src/j2c/progression.rs @@ -12,6 +12,7 @@ use crate::error::{DecodingError, Result}; use crate::j2c::decode::DecodeAllocationBudget; use crate::ValidationError; use core::cmp::Ordering; +use core::ops::Range; const PACKETS_PER_INCLUSION_WORD: usize = u64::BITS as usize; @@ -572,6 +573,7 @@ fn first_resolution_tile<'a>(input: &IteratorInput<'a>) -> Option Ordering; + +/// How a position-major progression's precincts are grouped by its leading +/// key, when that key is the resolution or the component: every precinct of +/// one group precedes every precinct of the next, so the progression is each +/// group's runs merged in turn. +#[derive(Clone, Copy, PartialEq, Eq)] +enum LeadingKey { + Resolution, + Component, + Position, +} + +/// A position-major progression, read as a merge of runs: the precincts of +/// one component and resolution, which come in raster order, are already in +/// progression order among themselves whenever their reference-grid +/// positions increase, so the progression only has to interleave them. A +/// tile whose runs are not in order is sorted whole and read as one run. struct PositionProgression { - elements: alloc::vec::IntoIter, + elements: Vec, + /// Unread part of each run, ordered by group. + runs: Vec>, + /// Each group's runs in `runs`, in progression order. + groups: Vec>, + group: usize, + order: PrecinctOrder, current: Option, min_layer: u8, max_layer: u8, next_layer: u8, } +impl PositionProgression { + /// The next precinct in progression order: the least unread head among + /// the current group's runs. + fn next_precinct(&mut self) -> Option { + while let Some(group) = self.groups.get(self.group) { + let runs = self.runs.get_mut(group.clone())?; + let mut least: Option<(usize, &PrecinctStore)> = None; + for (slot, run) in runs.iter().enumerate() { + let Some(head) = self.elements.get(run.start).filter(|_| !run.is_empty()) else { + continue; + }; + if least.is_none_or(|(_, best)| (self.order)(head, best) == Ordering::Less) { + least = Some((slot, head)); + } + } + if let Some((slot, &head)) = least { + runs[slot].start += 1; + return Some(head); + } + self.group += 1; + } + None + } +} + impl Iterator for PositionProgression { type Item = ProgressionData; @@ -605,7 +657,7 @@ impl Iterator for PositionProgression { return Some(packet); } } - self.current = self.elements.next(); + self.current = self.next_precinct(); self.next_layer = self.min_layer; self.current.as_ref()?; } @@ -615,36 +667,44 @@ impl Iterator for PositionProgression { fn position_progression_common( input: &IteratorInput<'_>, budget: &mut DecodeAllocationBudget, - sort: impl FnMut(&PrecinctStore, &PrecinctStore) -> Ordering, + leading: LeadingKey, + order: PrecinctOrder, ) -> Result { + let components = input.max_comp() as usize - input.min_comp() as usize; + let resolutions = input.total_max_resolution() as usize - input.min_resolution() as usize; let mut element_count = 0usize; + let mut run_count = 0usize; for component in input .tile .component_tiles() .skip(input.min_comp() as usize) - .take(input.max_comp() as usize - input.min_comp() as usize) + .take(components) { for resolution_tile in component .resolution_tiles() .skip(input.min_resolution() as usize) - .take(input.total_max_resolution() as usize - input.min_resolution() as usize) + .take(resolutions) { let precincts = usize::try_from(resolution_tile.num_precincts()) .map_err(|_| ValidationError::ImageTooLarge)?; element_count = element_count .checked_add(precincts) .ok_or(ValidationError::ImageTooLarge)?; + run_count += 1; } } let mut elements = Vec::new(); budget.reserve_new(&mut elements, element_count)?; + // Runs in push order: each component's resolutions in turn. + let mut pushed_runs = Vec::new(); + budget.reserve_new(&mut pushed_runs, run_count)?; for (component_idx, component) in input .tile .component_tiles() .enumerate() .skip(input.min_comp() as usize) - .take(input.max_comp() as usize - input.min_comp() as usize) + .take(components) { let component_idx = u16::try_from(component_idx).map_err(|_| ValidationError::ImageTooLarge)?; @@ -652,10 +712,11 @@ fn position_progression_common( .resolution_tiles() .enumerate() .skip(input.min_resolution() as usize) - .take(input.total_max_resolution() as usize - input.min_resolution() as usize) + .take(resolutions) { let resolution = u8::try_from(resolution).map_err(|_| ValidationError::ImageTooLarge)?; + let run_start = elements.len(); for precinct in resolution_tile .precincts() .ok_or(DecodingError::InvalidProgressionIterator)? @@ -671,6 +732,10 @@ fn position_progression_common( precinct_idx: precinct.idx, }); } + if pushed_runs.len() == pushed_runs.capacity() { + return Err(DecodingError::HostAllocationFailed.into()); + } + pushed_runs.push(run_start..elements.len()); } } @@ -678,10 +743,14 @@ fn position_progression_common( return Err(DecodingError::InvalidProgressionIterator.into()); } - elements.sort_unstable_by(sort); + let (runs, groups) = group_runs(&mut elements, pushed_runs, leading, order, budget)?; Ok(PositionProgression { - elements: elements.into_iter(), + elements, + runs, + groups, + group: 0, + order, current: None, min_layer: input.min_layer(), max_layer: input.max_layer(), @@ -689,19 +758,94 @@ fn position_progression_common( }) } +type RunGroups = (Vec>, Vec>); + +/// Group the runs by the progression's leading key, in progression order, +/// when every run is strictly in progression order. The order is total, so +/// merging each group's runs yields exactly the sorted sequence. Otherwise, +/// and for position-led orders, sort `elements` and read them as one run. +fn group_runs( + elements: &mut [PrecinctStore], + mut runs: Vec>, + leading: LeadingKey, + order: PrecinctOrder, + budget: &mut DecodeAllocationBudget, +) -> Result { + let key = |elements: &[PrecinctStore], run: &Range| { + let first = elements.get(run.start)?; + match leading { + LeadingKey::Resolution => Some(usize::from(first.resolution)), + LeadingKey::Component => Some(usize::from(first.component_idx)), + LeadingKey::Position => None, + } + }; + let merge = leading != LeadingKey::Position + && runs.iter().all(|run| { + elements.get(run.clone()).is_some_and(|run| { + run.windows(2) + .all(|pair| order(&pair[0], &pair[1]) == Ordering::Less) + }) + }); + if merge { + runs.retain(|run| !run.is_empty()); + // A group's runs may come in any order: each step takes the least head. + runs.sort_unstable_by_key(|run| key(elements, run)); + } else { + elements.sort_unstable_by(order); + runs.clear(); + if !elements.is_empty() { + if runs.capacity() == 0 { + return Err(DecodingError::InvalidProgressionIterator.into()); + } + runs.push(0..elements.len()); + } + } + let mut groups: Vec> = Vec::new(); + budget.reserve_new(&mut groups, runs.len())?; + for (index, run) in runs.iter().enumerate() { + match groups.last_mut() { + Some(group) if merge && key(elements, &runs[group.start]) == key(elements, run) => { + group.end = index + 1; + } + _ => groups.push(index..index + 1), + } + } + Ok((runs, groups)) +} + +fn rpcl_order(p: &PrecinctStore, s: &PrecinctStore) -> Ordering { + p.resolution + .cmp(&s.resolution) + .then_with(|| p.precinct_y.cmp(&s.precinct_y)) + .then_with(|| p.precinct_x.cmp(&s.precinct_x)) + .then_with(|| p.component_idx.cmp(&s.component_idx)) + .then_with(|| p.precinct_idx.cmp(&s.precinct_idx)) +} + +fn pcrl_order(p: &PrecinctStore, s: &PrecinctStore) -> Ordering { + p.precinct_y + .cmp(&s.precinct_y) + .then_with(|| p.precinct_x.cmp(&s.precinct_x)) + .then_with(|| p.component_idx.cmp(&s.component_idx)) + .then_with(|| p.resolution.cmp(&s.resolution)) + .then_with(|| p.precinct_idx.cmp(&s.precinct_idx)) +} + +fn cprl_order(p: &PrecinctStore, s: &PrecinctStore) -> Ordering { + p.component_idx + .cmp(&s.component_idx) + .then_with(|| p.precinct_y.cmp(&s.precinct_y)) + .then_with(|| p.precinct_x.cmp(&s.precinct_x)) + .then_with(|| p.resolution.cmp(&s.resolution)) + .then_with(|| p.precinct_idx.cmp(&s.precinct_idx)) +} + /// B.12.1.3 Resolution level-position-component-layer progression. fn resolution_position_component_layer_progression( input: &IteratorInput<'_>, budget: &mut DecodeAllocationBudget, ) -> Result { - position_progression_common(input, budget, |p, s| { - p.resolution - .cmp(&s.resolution) - .then_with(|| p.precinct_y.cmp(&s.precinct_y)) - .then_with(|| p.precinct_x.cmp(&s.precinct_x)) - .then_with(|| p.component_idx.cmp(&s.component_idx)) - .then_with(|| p.precinct_idx.cmp(&s.precinct_idx)) - }) + position_progression_common(input, budget, LeadingKey::Resolution, rpcl_order) } /// B.12.1.4 Position-component-resolution level-layer progression. @@ -709,14 +853,7 @@ fn position_component_resolution_layer_progression( input: &IteratorInput<'_>, budget: &mut DecodeAllocationBudget, ) -> Result { - position_progression_common(input, budget, |p, s| { - p.precinct_y - .cmp(&s.precinct_y) - .then_with(|| p.precinct_x.cmp(&s.precinct_x)) - .then_with(|| p.component_idx.cmp(&s.component_idx)) - .then_with(|| p.resolution.cmp(&s.resolution)) - .then_with(|| p.precinct_idx.cmp(&s.precinct_idx)) - }) + position_progression_common(input, budget, LeadingKey::Position, pcrl_order) } /// B.12.1.5 Component-position-resolution level-layer progression. @@ -724,14 +861,7 @@ fn component_position_resolution_layer_progression( input: &IteratorInput<'_>, budget: &mut DecodeAllocationBudget, ) -> Result { - position_progression_common(input, budget, |p, s| { - p.component_idx - .cmp(&s.component_idx) - .then_with(|| p.precinct_y.cmp(&s.precinct_y)) - .then_with(|| p.precinct_x.cmp(&s.precinct_x)) - .then_with(|| p.resolution.cmp(&s.resolution)) - .then_with(|| p.precinct_idx.cmp(&s.precinct_idx)) - }) + position_progression_common(input, budget, LeadingKey::Component, cprl_order) } #[cfg(test)] @@ -959,6 +1089,78 @@ mod tests { } } + fn component_with_precincts(levels: u8, exponent: u8, sampling: (u8, u8)) -> ComponentInfo { + let mut component = component_with_levels(levels); + component.size_info.horizontal_resolution = sampling.0; + component.size_info.vertical_resolution = sampling.1; + component.coding_style.parameters.code_block_width = 2; + component.coding_style.parameters.code_block_height = 2; + component.coding_style.parameters.precinct_exponents = (0..=levels) + .map(|resolution| { + let exponent = exponent + u8::from(resolution > 0); + (exponent, exponent + u8::from(resolution % 2 == 1)) + }) + .collect(); + component + } + + /// Merging each component's and resolution's precincts gives the order + /// sorting all of them gives, packet for packet. + #[test] + fn merged_position_progressions_match_the_sorted_order() { + let orders: [(LeadingKey, PrecinctOrder); 2] = [ + (LeadingKey::Resolution, rpcl_order), + (LeadingKey::Component, cprl_order), + ]; + for (components, rect) in [ + ( + vec![ + component_with_precincts(3, 3, (1, 1)), + component_with_precincts(3, 3, (1, 1)), + component_with_precincts(3, 3, (1, 1)), + ], + IntRect::from_ltrb(0, 0, 203, 157), + ), + ( + vec![ + component_with_precincts(4, 2, (1, 1)), + component_with_precincts(2, 3, (2, 1)), + component_with_precincts(3, 2, (2, 2)), + component_with_precincts(1, 4, (1, 3)), + ], + IntRect::from_ltrb(37, 21, 290, 199), + ), + ] { + let tile = Tile { + #[cfg(feature = "std")] + packet_directory: None, + idx: 0, + tile_parts: vec![], + component_infos: components, + rect, + progression_order: ProgressionOrder::ResolutionPositionComponentLayer, + progression_changes: vec![], + num_layers: 2, + mct: false, + }; + for (leading, order) in orders { + let input = IteratorInput::new(&tile).expect("progression input"); + let mut budget = DecodeAllocationBudget::from_live_bytes(0).expect("budget"); + let merged = position_progression_common(&input, &mut budget, leading, order) + .expect("merged progression"); + assert!(merged.groups.len() > 1, "the precincts merge in groups"); + let sorted = + position_progression_common(&input, &mut budget, LeadingKey::Position, order) + .expect("sorted progression"); + assert_eq!(sorted.groups.len(), 1); + let merged: Vec<_> = merged.collect(); + let sorted: Vec<_> = sorted.collect(); + assert!(merged.len() > 100); + assert_eq!(merged, sorted); + } + } + } + /// The (resolution, component) of each packet, for a one-layer tile. fn packet_order( levels: &[u8], From 37f3507d402bc0c5dd91edb0f5a406a6f59c72e4 Mon Sep 17 00:00:00 2001 From: GF Date: Thu, 29 Oct 2026 23:58:22 -0400 Subject: [PATCH 08/53] perf(j2k-native): skip an unneeded precinct's remaining PLT packets at once When PLT lengths cover the tile, a packet of an unused precinct is skipped by its length. In position-major progressions the precinct's remaining layers come right after it, so skip them in the same step (up to the end of the tile-part) and advance the progression past them, instead of walking each packet through the progression and the relevance checks. TilePartCursor::skip_packet, unused once skip_packets exists, is removed. Measured with the previous commit (progression merge); see there. --- crates/j2k-native/src/j2c/progression.rs | 36 ++++++++++++++++++++++ crates/j2k-native/src/j2c/segment.rs | 37 +++++++++++++++++++--- crates/j2k-native/src/j2c/tile/cursor.rs | 39 ++++++++++++++---------- 3 files changed, 91 insertions(+), 21 deletions(-) diff --git a/crates/j2k-native/src/j2c/progression.rs b/crates/j2k-native/src/j2c/progression.rs index 040e4cf5a..326e4a99d 100644 --- a/crates/j2k-native/src/j2c/progression.rs +++ b/crates/j2k-native/src/j2c/progression.rs @@ -209,6 +209,42 @@ impl ProgressionIterator<'_> { pub(crate) const fn owned_bytes(&self) -> usize { self.owned_bytes } + + /// Packets of the precinct the last packet belongs to that come right + /// after it: its remaining layers, in a position-major progression + /// without progression changes, whose layer is the innermost index. + /// Zero for every other progression. + pub(crate) fn remaining_precinct_layers(&self) -> usize { + match &self.inner { + ProgressionInner::Single(ProgressionOrderIterator::Position(iterator)) + if iterator.current.is_some() => + { + usize::from(iterator.max_layer.saturating_sub(iterator.next_layer)) + } + _ => 0, + } + } + + /// Pass over `count` of [`Self::remaining_precinct_layers`], as `count` + /// calls to `next` would. + pub(crate) fn skip_precinct_layers(&mut self, count: usize) { + if count == 0 { + return; + } + if let ProgressionInner::Single(ProgressionOrderIterator::Position(iterator)) = + &mut self.inner + { + let remaining = usize::from(iterator.max_layer.saturating_sub(iterator.next_layer)); + if count <= remaining { + // `count` fits in the u8 layer range below `max_layer`. + iterator.next_layer += u8::try_from(count).unwrap_or(0); + return; + } + } + for _ in 0..count { + self.next(); + } + } } impl Iterator for ProgressionIterator<'_> { diff --git a/crates/j2k-native/src/j2c/segment.rs b/crates/j2k-native/src/j2c/segment.rs index 279198cbd..31df900db 100644 --- a/crates/j2k-native/src/j2c/segment.rs +++ b/crates/j2k-native/src/j2c/segment.rs @@ -7,7 +7,7 @@ use super::build::{CodeBlock, ParseScratch, Precinct, PrecinctSelection, Precinc use super::codestream::markers::{EPH, SOP}; use super::codestream::{ComponentInfo, Header}; use super::decode::{DecodeAllocationBudget, DecompositionStorage}; -use super::progression::{progression_iterator, ProgressionData}; +use super::progression::{progression_iterator, ProgressionData, ProgressionIterator}; use super::tag_tree::TagNode; use super::tile::{Tile, TilePartCursor}; use crate::error::{bail, DecodeError, DecodingError, Result, ValidationError}; @@ -217,7 +217,7 @@ fn parse_with_walk<'a>( fn parse_selected_inner<'a>( tile: &Tile<'a>, - progression_iterator: &mut dyn Iterator, + progression_iterator: &mut ProgressionIterator<'_>, skips: &mut PacketSkips, header: &Header<'_>, storage: &mut DecompositionStorage<'a>, @@ -242,7 +242,7 @@ fn parse_selected_inner<'a>( fn parse_tile_parts<'a>( tile: &Tile<'a>, - progression_iterator: &mut dyn Iterator, + progression_iterator: &mut ProgressionIterator<'_>, skips: &mut PacketSkips, header: &Header<'_>, storage: &mut DecompositionStorage<'a>, @@ -383,7 +383,7 @@ impl<'s> PacketRelevance<'s> { )] fn parse_inner<'a>( mut tile_part: TilePartCursor<'_, 'a>, - progression_iterator: &mut dyn Iterator, + progression_iterator: &mut ProgressionIterator<'_>, component_infos: &[ComponentInfo], num_layers: u8, storage: &mut DecompositionStorage<'a>, @@ -451,7 +451,34 @@ fn parse_inner<'a>( .as_ref() .is_some_and(|relevance| relevance.selection.skip_by_length()) { - if !tile_part.skip_packet() { + // Need is decided per precinct, so the precinct's remaining + // layers, which a position-major progression reads next, are + // skipped with this one, up to the end of the tile-part. + #[cfg(feature = "std")] + let layers = if walk.recording.is_none() { + progression_iterator.remaining_precinct_layers() + } else { + 0 + }; + #[cfg(not(feature = "std"))] + let layers = progression_iterator.remaining_precinct_layers(); + let (skipped, complete) = tile_part.skip_packets(1 + layers); + // The packets a one-by-one walk would have taken from the + // progression after this one: each skipped, plus a failing one. + let further = if complete { + skipped.saturating_sub(1) + } else { + skipped + }; + progression_iterator.skip_precinct_layers(further); + #[cfg(feature = "std")] + { + walk.next_packet = walk + .next_packet + .checked_add(further) + .ok_or("packet ordinal overflows")?; + } + if !complete { return Err("PLT packet length runs past the tile-part"); } continue; diff --git a/crates/j2k-native/src/j2c/tile/cursor.rs b/crates/j2k-native/src/j2c/tile/cursor.rs index 2bda2afe0..9b86f6186 100644 --- a/crates/j2k-native/src/j2c/tile/cursor.rs +++ b/crates/j2k-native/src/j2c/tile/cursor.rs @@ -152,31 +152,38 @@ impl<'data> TilePartCursor<'_, 'data> { (actual == expected as usize).then_some(()) } - /// Skip one whole packet, header and body, by its PLT length. - /// - /// Only in-band packet headers can be skipped this way. Returns `false` - /// without consuming anything when the length is not signalled, the - /// headers live in PPM/PPT, or the length runs past the tile-part, so the - /// caller parses the packet instead. - pub(crate) fn skip_packet(&mut self) -> bool { + /// Skip up to `count` whole packets, header and body, by their PLT + /// lengths, stopping after a packet that ends the tile-part's data. Only + /// in-band packet headers can be skipped this way. Returns how many were + /// skipped and whether the next skip would have succeeded: `false` when + /// the length is not signalled, the headers live in PPM/PPT, or the length + /// runs past the tile-part, which consumes nothing of that packet. + pub(crate) fn skip_packets(&mut self, count: usize) -> (usize, bool) { let Self::Merged { data, packet_lengths, } = self else { - return false; + return (0, false); }; if !packet_lengths.present { - return false; + return (0, false); } - let Some(&length) = packet_lengths.lengths.get(packet_lengths.next) else { - return false; - }; - if data.skip_bytes(length as usize).is_none() { - return false; + let mut skipped = 0; + while skipped < count { + if skipped > 0 && data.at_end() { + break; + } + let Some(&length) = packet_lengths.lengths.get(packet_lengths.next) else { + return (skipped, false); + }; + if data.skip_bytes(length as usize).is_none() { + return (skipped, false); + } + packet_lengths.next += 1; + skipped += 1; } - packet_lengths.next += 1; - true + (skipped, true) } pub(crate) fn validate_all_packet_lengths_consumed(&self) -> Option<()> { From 0ff21ce7b7203e15a11eab538f0ee298bb06d23f Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:06:45 -0400 Subject: [PATCH 09/53] perf(j2k-native): read stuffed packet-header bits a byte run at a time read_bits_with_stuffing takes each byte's remaining bits at once instead of one bit per iteration; reading the last bit of a 0xFF byte still consumes the stuff bit that starts the next byte (B.10.1), so values and positions match bit-by-bit reads (randomized test). The coding-pass count probes the header once instead of up to seven separate peeks. Measured against the previous commit, on an 18-core ARM64 machine, 5 interleaved rounds of J2K_PROFILE_STAGES (packet-header parse of a region of the no-PLT test images): the middle region of the 18000 x 12000 no-PLT image -22% on 18 threads and -20% on one; that of the HTJ2K image -30% on 18. --- crates/j2k-native/src/j2c/segment.rs | 107 +++++++++++++++++++++++++-- crates/j2k-native/src/reader.rs | 78 ++++++++++++++++--- 2 files changed, 168 insertions(+), 17 deletions(-) diff --git a/crates/j2k-native/src/j2c/segment.rs b/crates/j2k-native/src/j2c/segment.rs index 31df900db..be3f77a33 100644 --- a/crates/j2k-native/src/j2c/segment.rs +++ b/crates/j2k-native/src/j2c/segment.rs @@ -879,25 +879,29 @@ fn resolve_code_block_inclusion( } fn decode_num_coding_passes(reader: &mut BitReader<'_>) -> Option { - if reader.peak_bits_with_stuffing(9) == Some(0x1ff) { + // The longest codeword prefix has nine bits. Peek once at as many of them + // as the header holds: each shorter peek sees the first bits of it. + let (prefix, available) = peek_up_to_nine_bits(reader); + let peek = |bits: u8| (available >= bits).then(|| prefix >> (available - bits)); + if peek(9) == Some(0x1ff) { reader.read_bits_with_stuffing(9)?; u8::try_from(reader.read_bits_with_stuffing(7)? + 37).ok() - } else if reader.peak_bits_with_stuffing(4) == Some(0x0f) { + } else if peek(4) == Some(0x0f) { reader.read_bits_with_stuffing(4)?; u8::try_from(reader.read_bits_with_stuffing(5)? + 6).ok() - } else if reader.peak_bits_with_stuffing(4) == Some(0b1110) { + } else if peek(4) == Some(0b1110) { reader.read_bits_with_stuffing(4)?; Some(5) - } else if reader.peak_bits_with_stuffing(4) == Some(0b1101) { + } else if peek(4) == Some(0b1101) { reader.read_bits_with_stuffing(4)?; Some(4) - } else if reader.peak_bits_with_stuffing(4) == Some(0b1100) { + } else if peek(4) == Some(0b1100) { reader.read_bits_with_stuffing(4)?; Some(3) - } else if reader.peak_bits_with_stuffing(2) == Some(0b10) { + } else if peek(2) == Some(0b10) { reader.read_bits_with_stuffing(2)?; Some(2) - } else if reader.peak_bits_with_stuffing(1) == Some(0) { + } else if peek(1) == Some(0) { reader.read_bits_with_stuffing(1)?; Some(1) } else { @@ -905,6 +909,25 @@ fn decode_num_coding_passes(reader: &mut BitReader<'_>) -> Option { } } +/// The next bits, up to nine, and how many the header holds: a peek of `n` +/// bits succeeds exactly when `n` single-bit reads would. +fn peek_up_to_nine_bits(reader: &BitReader<'_>) -> (u32, u8) { + if let Some(bits) = reader.clone().read_bits_with_stuffing(9) { + return (bits, 9); + } + let mut probe = reader.clone(); + let mut bits = 0_u32; + let mut available = 0_u8; + while available < 9 { + let Some(bit) = probe.read_bits_with_stuffing(1) else { + break; + }; + bits = (bits << 1) | bit; + available += 1; + } + (bits, available) +} + fn read_lblock_increment(reader: &mut BitReader<'_>) -> Option { let mut increment = 0; @@ -934,6 +957,76 @@ mod tests { use super::*; use alloc::vec::Vec; + /// The original chain of separate peeks, as the reference for + /// `decode_num_coding_passes`. + fn coding_passes_by_peeks(reader: &mut BitReader<'_>) -> Option { + if reader.peak_bits_with_stuffing(9) == Some(0x1ff) { + reader.read_bits_with_stuffing(9)?; + u8::try_from(reader.read_bits_with_stuffing(7)? + 37).ok() + } else if reader.peak_bits_with_stuffing(4) == Some(0x0f) { + reader.read_bits_with_stuffing(4)?; + u8::try_from(reader.read_bits_with_stuffing(5)? + 6).ok() + } else if reader.peak_bits_with_stuffing(4) == Some(0b1110) { + reader.read_bits_with_stuffing(4)?; + Some(5) + } else if reader.peak_bits_with_stuffing(4) == Some(0b1101) { + reader.read_bits_with_stuffing(4)?; + Some(4) + } else if reader.peak_bits_with_stuffing(4) == Some(0b1100) { + reader.read_bits_with_stuffing(4)?; + Some(3) + } else if reader.peak_bits_with_stuffing(2) == Some(0b10) { + reader.read_bits_with_stuffing(2)?; + Some(2) + } else if reader.peak_bits_with_stuffing(1) == Some(0) { + reader.read_bits_with_stuffing(1)?; + Some(1) + } else { + None + } + } + + #[test] + fn coding_pass_counts_match_separate_prefix_peeks() { + let mut state = 0x9e37_79b9_7f4a_7c15_u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for _ in 0..3000 { + let len = usize::try_from(next() % 6).expect("small length"); + let data: Vec = (0..len) + .map(|_| match next() % 3 { + 0 => 0xff, + _ => u8::try_from(next() >> 56).expect("top byte"), + }) + .collect(); + let skip = u8::try_from(next() % 8).expect("bit offset"); + let mut fast = BitReader::new(&data); + if fast.read_bits_with_stuffing(skip).is_none() { + continue; + } + let mut reference = fast.clone(); + loop { + let (a, b) = ( + decode_num_coding_passes(&mut fast), + coding_passes_by_peeks(&mut reference), + ); + assert_eq!(a, b, "{data:02x?} skip {skip}"); + if a.is_none() { + break; + } + assert_eq!(fast.offset(), reference.offset()); + assert_eq!( + fast.clone().read_bits_with_stuffing(1), + reference.clone().read_bits_with_stuffing(1) + ); + } + } + } + #[test] fn packet_segment_growth_respects_remaining_workspace_budget() { let segment_bytes = size_of::>(); diff --git a/crates/j2k-native/src/reader.rs b/crates/j2k-native/src/reader.rs index 233b69b87..5344a3730 100644 --- a/crates/j2k-native/src/reader.rs +++ b/crates/j2k-native/src/reader.rs @@ -127,21 +127,31 @@ impl<'a> BitReader<'a> { /// in addition. #[inline] pub(crate) fn read_bits_with_stuffing(&mut self, bit_size: u8) -> Option { - let mut bit = 0; - - for _ in 0..bit_size { - let needs_stuff_bit = self.needs_to_read_stuff_bit(); - - bit = (bit << 1) | self.read_bit()?; - - if needs_stuff_bit { + // Whole runs of a byte's remaining bits at once. Reading the last bit + // of a 0xFF byte also reads the stuff bit that starts the next byte, + // which must be zero, as bit by bit reads do (B.10.1). Values keep + // their low 32 bits, as shifting one bit at a time keeps them. + let mut value = 0_u32; + let mut remaining = usize::from(bit_size); + while remaining > 0 { + let byte = *self.data.get(self.byte_pos())?; + let available = 8 - self.bit_pos(); + let take = available.min(remaining); + let bits = (u32::from(byte) >> (available - take)) & ((1_u32 << take) - 1); + value = (value << take) | bits; + self.cur_pos += take; + remaining -= take; + if take == available && byte == 0xff { self.read_stuff_bit()?; } } - Some(bit) + Some(value) } + /// Whether the next bit read ends a 0xFF byte, so a stuff bit follows: + /// the one-bit-at-a-time rule `read_bits_with_stuffing` applies to runs. + #[cfg(test)] pub(crate) fn needs_to_read_stuff_bit(&mut self) -> bool { // B.10.1: "If the value of the byte is 0xFF, the next byte includes an extra zero bit // stuffed into the MSB." @@ -159,7 +169,7 @@ impl<'a> BitReader<'a> { Some(()) } - #[inline] + #[cfg(test)] pub(crate) fn peak_bits_with_stuffing(&mut self, bit_size: u8) -> Option { self.clone().read_bits_with_stuffing(bit_size) } @@ -195,6 +205,54 @@ mod tests { assert_eq!(reader.read_bits_with_stuffing(1), None); } + /// The one-bit-at-a-time reference for `read_bits_with_stuffing`. + fn read_bits_one_at_a_time(reader: &mut BitReader<'_>, bit_size: u8) -> Option { + let mut bit = 0_u32; + for _ in 0..bit_size { + let needs_stuff_bit = reader.needs_to_read_stuff_bit(); + bit = (bit << 1) | reader.read_bit()?; + if needs_stuff_bit { + reader.read_stuff_bit()?; + } + } + Some(bit) + } + + #[test] + fn stuffed_reads_match_one_bit_at_a_time_reads() { + let mut state = 0x2545_f491_4f6c_dd1d_u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for _ in 0..2000 { + let len = usize::try_from(next() % 24).expect("small length"); + let data: alloc::vec::Vec = (0..len) + .map(|_| match next() % 4 { + 0 => 0xff, + 1 => 0x7f, + _ => u8::try_from(next() >> 56).expect("top byte"), + }) + .collect(); + let mut fast = BitReader::new(&data); + let mut reference = BitReader::new(&data); + loop { + let bits = u8::try_from(next() % 40).expect("small width"); + let (a, b) = ( + fast.read_bits_with_stuffing(bits), + read_bits_one_at_a_time(&mut reference, bits), + ); + assert_eq!(a, b, "{data:02x?} width {bits}"); + if a.is_none() { + break; + } + assert_eq!(fast.cur_pos, reference.cur_pos); + } + } + } + #[test] fn truncated_marker_returns_error_instead_of_panicking() { let mut reader = BitReader::new(&[0xff]); From 5c1d84e1ab7abb8d5b51630790ad11a81eedee53 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:13:12 -0400 Subject: [PATCH 10/53] perf(j2k-native): find region precincts by index and skip unused ones' geometry ResolutionTile::precincts() becomes a validated PrecinctGrid with closed-form positions (the old iterated next_multiple_of) that can also answer one precinct's rectangle by index. Allocation planning, build_precincts and the region precinct selection loop precinct indices and compute a rectangle only for precincts they use; unused ones get no geometry at all. Selection hoists the per-resolution sub-band windows out of its precinct loop. Measured against the previous commit, on an 18-core ARM64 machine, 5 interleaved rounds: on the 18000 x 12000 test images: the HTJ2K one reduced four levels -7% on 18 threads; regions of the PLT one -3.7% (top-left), -3% (middle) and -2.8% (bottom-right) on 18 threads, -2% (middle) on 6. --- crates/j2k-native/src/j2c/build.rs | 28 ++- .../src/j2c/build/allocation/plan.rs | 14 +- crates/j2k-native/src/j2c/build/selection.rs | 27 ++- crates/j2k-native/src/j2c/tile.rs | 227 ++++++++++++++---- 4 files changed, 219 insertions(+), 77 deletions(-) diff --git a/crates/j2k-native/src/j2c/build.rs b/crates/j2k-native/src/j2c/build.rs index b778d9aec..5a49770f4 100644 --- a/crates/j2k-native/src/j2c/build.rs +++ b/crates/j2k-native/src/j2c/build.rs @@ -303,18 +303,22 @@ fn build_precincts( let required = |rect: IntRect| roi_plan.is_none_or(|plan| plan.code_block_required(sub_band_idx, rect)); - for (precinct_idx, precinct_data) in resolution_tile - .precincts() - .ok_or(DecodingError::InvalidPrecinct)? - .enumerate() - { + let precincts = resolution_tile + .precinct_grid() + .ok_or(DecodingError::InvalidPrecinct)?; + let precinct_count = usize::try_from(resolution_tile.num_precincts()) + .map_err(|_| ValidationError::ImageTooLarge)?; + for precinct_idx in 0..precinct_count { let precinct_use = keep.and_then(|keep| keep.get(precinct_idx).copied()); if precinct_use == Some(PrecinctUse::Unused) { // Region decode never parses this precinct's packets, so it gets // no record. The parser finds the others through the selection. continue; } - let grid = code_block_grid(resolution_tile, sub_band_rect, precinct_data.rect)?; + let precinct_rect = precincts + .rect_at(precinct_idx) + .ok_or(DecodingError::InvalidPrecinct)?; + let grid = code_block_grid(resolution_tile, sub_band_rect, precinct_rect)?; if precinct_use == Some(PrecinctUse::Parsed) && parsed_in_scratch { // Its packets are parsed in a scratch precinct, which only needs // its code-block grid. @@ -332,10 +336,10 @@ fn build_precincts( ltrace!( "Precinct rect: [{},{} {}x{}], num_code_blocks_wide: {}, num_code_blocks_high: {}", - precinct_data.rect.x0, - precinct_data.rect.y0, - precinct_data.rect.width(), - precinct_data.rect.height(), + precinct_rect.x0, + precinct_rect.y0, + precinct_rect.width(), + precinct_rect.height(), grid.columns, grid.rows ); @@ -521,7 +525,9 @@ pub(crate) struct PrecinctData { /// The y coordinate mapped back to the reference grid. pub(crate) r_y: u32, /// The actual rectangle of the precinct (in the sub-band coordinate - /// system). + /// system). Decodes ask `PrecinctGrid::rect_at` for the rectangles they + /// need; tests check the two agree. + #[cfg(test)] pub(crate) rect: IntRect, /// The index of the precinct in the sub-band. pub(crate) idx: u64, diff --git a/crates/j2k-native/src/j2c/build/allocation/plan.rs b/crates/j2k-native/src/j2c/build/allocation/plan.rs index f9e625194..9a6839099 100644 --- a/crates/j2k-native/src/j2c/build/allocation/plan.rs +++ b/crates/j2k-native/src/j2c/build/allocation/plan.rs @@ -189,11 +189,10 @@ fn include_sub_band_allocations( let mut observed_precincts = 0usize; let mut observed_code_blocks = 0usize; - for (precinct_idx, precinct_data) in resolution_tile - .precincts() - .ok_or(DecodingError::InvalidPrecinct)? - .enumerate() - { + let precincts = resolution_tile + .precinct_grid() + .ok_or(DecodingError::InvalidPrecinct)?; + for precinct_idx in 0..precinct_count { observed_precincts = observed_precincts .checked_add(1) .ok_or(ValidationError::ImageTooLarge)?; @@ -212,7 +211,10 @@ fn include_sub_band_allocations( DecompositionAllocationPlan::checked_add_count(&mut plan.unbuilt_precincts, 1)?; continue; } - let grid = code_block_grid(resolution_tile, sub_band_rect, precinct_data.rect)?; + let precinct_rect = precincts + .rect_at(precinct_idx) + .ok_or(DecodingError::InvalidPrecinct)?; + let grid = code_block_grid(resolution_tile, sub_band_rect, precinct_rect)?; let block_count = (grid.columns as usize) .checked_mul(grid.rows as usize) .ok_or(ValidationError::ImageTooLarge)?; diff --git a/crates/j2k-native/src/j2c/build/selection.rs b/crates/j2k-native/src/j2c/build/selection.rs index 270bdd31f..464af68b3 100644 --- a/crates/j2k-native/src/j2c/build/selection.rs +++ b/crates/j2k-native/src/j2c/build/selection.rs @@ -470,17 +470,24 @@ fn select( decomposition.sub_bands, ) }; - for precinct in resolution_tile - .precincts() - .ok_or(DecodingError::InvalidPrecinct)? - { + // Each sub-band the ROI plan reads, in order, with its window. + let windows: [Option<(IntRect, IntRect)>; 3] = core::array::from_fn(|slot| { + let sub_band_type = sub_band_types.get(slot)?; + let window = plan.sub_band_window(sub_band_indices[slot])?; + Some((window, resolution_tile.sub_band_rect(*sub_band_type))) + }); + let precincts = resolution_tile + .precinct_grid() + .ok_or(DecodingError::InvalidPrecinct)?; + let precinct_count = usize::try_from(resolution_tile.num_precincts()) + .map_err(|_| ValidationError::ImageTooLarge)?; + for precinct_idx in 0..precinct_count { + let precinct_rect = precincts + .rect_at(precinct_idx) + .ok_or(DecodingError::InvalidPrecinct)?; let mut required = false; - for (sub_band_type, sub_band_idx) in sub_band_types.iter().zip(sub_band_indices) { - let Some(window) = plan.sub_band_window(sub_band_idx) else { - continue; - }; - let rect = resolution_tile.sub_band_rect(*sub_band_type); - let grid = code_block_grid(&resolution_tile, rect, precinct.rect)?; + for (window, rect) in windows.iter().flatten().copied() { + let grid = code_block_grid(&resolution_tile, rect, precinct_rect)?; if grid.columns > 0 && grid.rows > 0 && window.intersects(grid.area.intersect(rect)) diff --git a/crates/j2k-native/src/j2c/tile.rs b/crates/j2k-native/src/j2c/tile.rs index e4c5fc0b5..e92922e6c 100644 --- a/crates/j2k-native/src/j2c/tile.rs +++ b/crates/j2k-native/src/j2c/tile.rs @@ -504,15 +504,18 @@ impl<'a> ResolutionTile<'a> { /// Return an iterator over the data of the precincts in this resolution /// tile. + pub(crate) fn precincts(&self) -> Option> { + self.precinct_grid().map(PrecinctGrid::iter) + } + + /// The validated precinct grid of this resolution tile, whose precincts + /// [`Self::precincts`] lists in order and [`PrecinctGrid::rect_at`] finds + /// by index. #[expect( clippy::similar_names, reason = "paired axis, subband, and marker names follow JPEG 2000 specification notation" )] - #[expect( - clippy::too_many_lines, - reason = "precinct iteration validates both coordinate grids before constructing the iterator" - )] - pub(crate) fn precincts(&self) -> Option> { + pub(crate) fn precinct_grid(&self) -> Option { let num_precincts_y = self.num_precincts_y(); let num_precincts_x = self.num_precincts_x(); @@ -620,51 +623,18 @@ impl<'a> ResolutionTile<'a> { self.component_tile.tile.rect.y1, )?; - let iter = (0..num_precincts_y).flat_map(move |y| { - let y0 = y * ppy_pow2 + y_start; - let mut r_x = r_x; - - let res = (0..num_precincts_x).map(move |x| { - let x0 = x * ppx_pow2 + x_start; - - #[expect( - clippy::cast_possible_truncation, - reason = "positions_fit checked every emitted coordinate against the u32 tile endpoint" - )] - let data = PrecinctData { - r_x: r_x as u32, - r_y: r_y as u32, - // A grid cell may end at 2^32. Image/subband endpoints - // are at most u32::MAX, so clipping here preserves every - // possible coefficient without overflowing the cell edge. - rect: IntRect::from_ltrb( - x0, - y0, - x0.saturating_add(ppx_pow2), - y0.saturating_add(ppy_pow2), - ), - idx: u64::from(num_precincts_x) * u64::from(y) + u64::from(x), - }; - - // If r_x is already aligned, we simply step by `precinct_x_step`. - // Otherwise (can only be the case for precincts in the first - // row or column), align to the next multiple. - if x + 1 < num_precincts_x { - r_x = (r_x + 1).next_multiple_of(precinct_x_step); - } - - data - }); - - // Same as for r_x. - if y + 1 < num_precincts_y { - r_y = (r_y + 1).next_multiple_of(precinct_y_step); - } - - res - }); - - Some(iter) + Some(PrecinctGrid { + num_x: num_precincts_x, + num_y: num_precincts_y, + ppx_pow2, + ppy_pow2, + x_start, + y_start, + first_r_x: r_x, + first_r_y: r_y, + x_step: precinct_x_step, + y_step: precinct_y_step, + }) } pub(crate) fn code_block_width(&self) -> u32 { @@ -704,6 +674,88 @@ impl<'a> ResolutionTile<'a> { } } +/// The precincts of one resolution tile: a grid of `num_x` by `num_y` cells +/// of `ppx_pow2` by `ppy_pow2` sub-band samples from `(x_start, y_start)`. +/// Each precinct's reference-grid position starts at the tile origin, aligned +/// past it when the grid is, and steps by whole precinct strides. +#[derive(Clone, Copy, Debug)] +pub(crate) struct PrecinctGrid { + num_x: u32, + num_y: u32, + ppx_pow2: u32, + ppy_pow2: u32, + x_start: u32, + y_start: u32, + first_r_x: u64, + first_r_y: u64, + x_step: u64, + y_step: u64, +} + +impl PrecinctGrid { + /// Every precinct, row by row. + pub(crate) fn iter(self) -> impl Iterator { + (0..self.num_y).flat_map(move |y| (0..self.num_x).map(move |x| self.cell(x, y))) + } + + /// Precinct `idx`, numbered row by row as [`Self::iter`] lists them, or + /// `None` past the last. + #[cfg(test)] + pub(crate) fn at(self, idx: u64) -> Option { + let x = u32::try_from(idx % u64::from(self.num_x.max(1))).ok()?; + let y = u32::try_from(idx / u64::from(self.num_x.max(1))).ok()?; + (x < self.num_x && y < self.num_y).then(|| self.cell(x, y)) + } + + /// The first position, or the first aligned stride past it and whole + /// strides from there: [`ResolutionTile::precinct_grid`] checked that + /// every emitted position fits the tile. + fn position(first: u64, step: u64, index: u32) -> u64 { + if index == 0 { + first + } else { + (first + 1).next_multiple_of(step) + u64::from(index - 1) * step + } + } + + /// The rectangle of precinct `idx` alone, numbered row by row as + /// [`Self::iter`] lists them, or `None` past the last. + pub(crate) fn rect_at(self, idx: usize) -> Option { + let columns = usize::try_from(self.num_x).ok()?; + let x = u32::try_from(idx.checked_rem(columns)?).ok()?; + let y = u32::try_from(idx / columns).ok()?; + (y < self.num_y).then(|| self.cell_rect(x, y)) + } + + fn cell_rect(self, x: u32, y: u32) -> IntRect { + let x0 = x * self.ppx_pow2 + self.x_start; + let y0 = y * self.ppy_pow2 + self.y_start; + // A grid cell may end at 2^32. Image/subband endpoints are at most + // u32::MAX, so clipping here preserves every possible coefficient + // without overflowing the cell edge. + IntRect::from_ltrb( + x0, + y0, + x0.saturating_add(self.ppx_pow2), + y0.saturating_add(self.ppy_pow2), + ) + } + + #[expect( + clippy::cast_possible_truncation, + reason = "precinct_grid checked every emitted coordinate against the u32 tile endpoint" + )] + fn cell(self, x: u32, y: u32) -> PrecinctData { + PrecinctData { + r_x: Self::position(self.first_r_x, self.x_step, x) as u32, + r_y: Self::position(self.first_r_y, self.y_step, y) as u32, + #[cfg(test)] + rect: self.cell_rect(x, y), + idx: u64::from(self.num_x) * u64::from(y) + u64::from(x), + } + } +} + #[cfg(test)] mod tests { use alloc::vec; @@ -826,6 +878,81 @@ mod tests { assert_eq!(precincts[0].r_y, 0); } + /// Precincts found by index are the ones the grid lists in order, and the + /// positions step by whole strides after the first. + #[test] + fn indexed_precincts_match_the_listed_order() { + for (sampling, exponents, rect) in [ + ((1, 1), (3, 4), IntRect::from_ltrb(0, 0, 203, 157)), + ((2, 1), (2, 3), IntRect::from_ltrb(37, 21, 290, 199)), + ((3, 2), (4, 2), IntRect::from_ltrb(5, 9, 77, 301)), + ] { + let component_info = ComponentInfo { + size_info: ComponentSizeInfo { + precision: 8, + signed: false, + horizontal_resolution: sampling.0, + vertical_resolution: sampling.1, + }, + coding_style: CodingStyleComponent { + flags: CodingStyleFlags::default(), + parameters: CodingStyleParameters { + num_decomposition_levels: 3, + num_resolution_levels: 4, + code_block_width: 2, + code_block_height: 2, + code_block_style: CodeBlockStyle::default(), + transformation: WaveletTransform::Reversible53, + precinct_exponents: vec![ + exponents, + (exponents.0 + 1, exponents.1 + 1), + (exponents.0 + 1, exponents.1 + 2), + (exponents.0 + 2, exponents.1 + 1), + ], + }, + }, + quantization_info: QuantizationInfo { + quantization_style: QuantizationStyle::NoQuantization, + guard_bits: 2, + step_sizes: vec![], + }, + roi_shift: 0, + }; + let tile = Tile { + #[cfg(feature = "std")] + packet_directory: None, + idx: 0, + tile_parts: vec![], + rect, + component_infos: vec![component_info], + progression_order: ProgressionOrder::ResolutionPositionComponentLayer, + progression_changes: vec![], + mct: false, + num_layers: 1, + }; + let component = ComponentTile::new(&tile, &tile.component_infos[0]); + for resolution in 0..4 { + let resolution_tile = ResolutionTile::new(component, resolution); + let grid = resolution_tile + .precinct_grid() + .expect("valid precinct grid"); + let listed: Vec<_> = grid.iter().collect(); + assert_eq!(listed.len() as u64, resolution_tile.num_precincts()); + assert!(listed.len() > 1); + for (index, precinct) in listed.iter().enumerate() { + let found = grid.at(index as u64).expect("indexed precinct"); + assert_eq!( + (found.r_x, found.r_y, found.rect, found.idx), + (precinct.r_x, precinct.r_y, precinct.rect, precinct.idx) + ); + assert_eq!(grid.rect_at(index), Some(precinct.rect)); + } + assert!(grid.at(listed.len() as u64).is_none()); + assert!(grid.rect_at(listed.len()).is_none()); + } + } + } + /// Test case for the example in B.4. #[test] #[expect( From 57f2b6f546ffa551c04cd79cfbdb9b2a14a8615d Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:25:30 -0400 Subject: [PATCH 11/53] perf(j2k-native): skip a scratch precinct's packet body in one step A packet whose precincts are parsed only to reach later packets borrows a scratch precinct whose code-blocks have no layer records, so its header skips every contribution. Skip its body as one run of the summed skipped bytes (PacketSkips keeps a saturating total) instead of walking every code-block of its sub-bands; a short body fails with the same error. Measured with the next commit (record lookup) against the commit before, on an 18-core ARM64 machine, 5 interleaved rounds: regions of the 18000 x 12000 no-PLT test image -4.7% (middle) and -3.7% (top-left) on 18 threads; PLT rows within 1%. --- crates/j2k-native/src/j2c/segment.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/crates/j2k-native/src/j2c/segment.rs b/crates/j2k-native/src/j2c/segment.rs index be3f77a33..03b4ee16b 100644 --- a/crates/j2k-native/src/j2c/segment.rs +++ b/crates/j2k-native/src/j2c/segment.rs @@ -81,6 +81,8 @@ impl<'a> SegmentSink<'_, 'a> { pub(super) struct PacketSkips { /// (code-block index, body bytes) for each contribution to skip. entries: Vec<(usize, usize)>, + /// The entries' bytes added up, saturating: more than any body holds. + bytes: usize, #[cfg(feature = "std")] discard_contributions: bool, } @@ -108,6 +110,7 @@ impl PacketSkips { DecodeAllocationBudget::from_live_bytes(live_bytes)?.reserve_new(&mut entries, capacity)?; Ok(Self { entries, + bytes: 0, #[cfg(feature = "std")] discard_contributions: false, }) @@ -119,6 +122,7 @@ impl PacketSkips { fn clear(&mut self) { self.entries.clear(); + self.bytes = 0; } pub(super) fn push(&mut self, code_block: usize, bytes: usize) -> PacketResult { @@ -129,6 +133,7 @@ impl PacketSkips { return Err("packet has more skipped contributions than code-blocks"); } self.entries.push((code_block, bytes)); + self.bytes = self.bytes.saturating_add(bytes); Ok(()) } } @@ -314,22 +319,24 @@ impl<'s> PacketRelevance<'s> { } /// Lend a parsed-only packet's precincts their scratch precincts, one per - /// sub-band, before its header is parsed. + /// sub-band, before its header is parsed. Returns whether it did: the + /// packet's code-blocks then have no layer records, so the header skips + /// every contribution. fn lend_scratch( relevance: Option<&mut Self>, packet: &ProgressionData, precinct_offset: usize, sub_bands: impl Iterator, storage: &mut DecompositionStorage<'_>, - ) -> PacketResult { + ) -> PacketResult { let Some(relevance) = relevance else { - return Ok(()); + return Ok(false); }; let Some(scratch) = relevance.scratch.as_mut() else { - return Ok(()); + return Ok(false); }; if relevance.selection.packet_use(packet) != Some(PrecinctUse::Parsed) { - return Ok(()); + return Ok(false); } for (slot, sub_band_idx) in sub_bands.enumerate() { let precinct = storage.sub_bands[sub_band_idx] @@ -348,7 +355,7 @@ impl<'s> PacketRelevance<'s> { ) .ok_or("parsed-only precinct cannot borrow its scratch precinct")?; } - Ok(()) + Ok(true) } /// Whether a packet must be parsed: always without a selection. A packet @@ -523,7 +530,7 @@ fn parse_inner<'a>( skips.clear(); // Even an empty packet starts its precinct: a precinct's first packet // is the one that borrows its scratch. - PacketRelevance::lend_scratch( + let in_scratch = PacketRelevance::lend_scratch( relevance.as_deref_mut(), &progression_data, precinct_offset, @@ -559,7 +566,12 @@ fn parse_inner<'a>( // Now read the packet body. let body_reader = tile_part.body(); - if !zero_length { + if !zero_length && in_scratch { + // Every contribution was skipped: the body is just their bytes. + body_reader + .skip_bytes(skips.bytes) + .ok_or("skipped packet body is shorter than its segment lengths")?; + } else if !zero_length { let mut skipped = skips.entries.iter().peekable(); for sub_band_idx in sub_band_iter { let sub_band = &storage.sub_bands[sub_band_idx]; From 833d45dfc942f805b54f128111316d3a406e4d23 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:27:39 -0400 Subject: [PATCH 12/53] perf(j2k-native): look a precinct's packet record up once for its layers record_offset binary-searches the run's precinct records for every packet. Position-major progressions read a precinct's layers in a row, so keep the last (component, resolution, precinct) and its offset; the selection is immutable while parsing, so the cached offset is the looked-up one. Measured with the previous commit; see there. --- crates/j2k-native/src/j2c/segment.rs | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/crates/j2k-native/src/j2c/segment.rs b/crates/j2k-native/src/j2c/segment.rs index 03b4ee16b..39c418420 100644 --- a/crates/j2k-native/src/j2c/segment.rs +++ b/crates/j2k-native/src/j2c/segment.rs @@ -405,6 +405,10 @@ fn parse_inner<'a>( )] walk: &mut PacketWalk<'_>, ) -> PacketResult { + // The last precinct's place among its sub-bands' records, by (component, + // resolution, precinct): position-major progressions read all of a + // precinct's layers in a row, so it is looked up once for them. + let mut last_record: Option<((u16, u8, u64), usize)> = None; while !tile_part.header().at_end() { let progression_data = progression_iterator .next() @@ -495,12 +499,24 @@ fn parse_inner<'a>( // precinct records. A region decode keeps none for unused // precincts. A packet for a precinct without a record gets an offset // past them all, so it fails below as one for a missing precinct. - let precinct_offset = relevance - .as_deref() - .map_or(Some(precinct_index), |relevance| { - relevance.selection.record_offset(&progression_data) - }) - .unwrap_or(usize::MAX); + let key = ( + progression_data.component, + progression_data.resolution, + progression_data.precinct, + ); + let precinct_offset = match last_record { + Some((last, offset)) if last == key => offset, + _ => { + let offset = relevance + .as_deref() + .map_or(Some(precinct_index), |relevance| { + relevance.selection.record_offset(&progression_data) + }) + .unwrap_or(usize::MAX); + last_record = Some((key, offset)); + offset + } + }; let packet_start = tile_part.packet_start_offset(); let body_reader = tile_part.body(); From c785f95e3428bd1b597c985585b584e16290e545 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:36:06 -0400 Subject: [PATCH 13/53] perf(j2k-native): read coding-pass codewords one prefix at a time Read Table B.4's codewords prefix by prefix (1, 1, 2, 5 and 7 bits) instead of peeking up to nine bits and re-reading the matched prefix. Values and reader positions match the peek chain on success (randomized test). Measured with the next commit against the commit before, on an 18-core ARM64 machine, 7 interleaved rounds of J2K_PROFILE_STAGES: packet-header parse -5% to -8% on the regions of the 18000 x 12000 no-PLT and HTJ2K test images; 5 rounds of whole decodes of the no-PLT image's regions: middle -3.2% and top-left -3.1% on 18 threads, middle -2.1% on one. --- crates/j2k-native/src/j2c/segment.rs | 59 +++++++--------------------- 1 file changed, 15 insertions(+), 44 deletions(-) diff --git a/crates/j2k-native/src/j2c/segment.rs b/crates/j2k-native/src/j2c/segment.rs index 39c418420..e24fc3025 100644 --- a/crates/j2k-native/src/j2c/segment.rs +++ b/crates/j2k-native/src/j2c/segment.rs @@ -907,53 +907,24 @@ fn resolve_code_block_inclusion( } fn decode_num_coding_passes(reader: &mut BitReader<'_>) -> Option { - // The longest codeword prefix has nine bits. Peek once at as many of them - // as the header holds: each shorter peek sees the first bits of it. - let (prefix, available) = peek_up_to_nine_bits(reader); - let peek = |bits: u8| (available >= bits).then(|| prefix >> (available - bits)); - if peek(9) == Some(0x1ff) { - reader.read_bits_with_stuffing(9)?; - u8::try_from(reader.read_bits_with_stuffing(7)? + 37).ok() - } else if peek(4) == Some(0x0f) { - reader.read_bits_with_stuffing(4)?; - u8::try_from(reader.read_bits_with_stuffing(5)? + 6).ok() - } else if peek(4) == Some(0b1110) { - reader.read_bits_with_stuffing(4)?; - Some(5) - } else if peek(4) == Some(0b1101) { - reader.read_bits_with_stuffing(4)?; - Some(4) - } else if peek(4) == Some(0b1100) { - reader.read_bits_with_stuffing(4)?; - Some(3) - } else if peek(2) == Some(0b10) { - reader.read_bits_with_stuffing(2)?; - Some(2) - } else if peek(1) == Some(0) { - reader.read_bits_with_stuffing(1)?; - Some(1) - } else { - None + // Table B.4, read a prefix at a time: 0 -> 1, 10 -> 2, 11xx -> 3 + xx + // (xx < 3), 1111 xxxxx -> 6 + xxxxx (xxxxx < 31) and 1111 11111 xxxxxxx + // -> 37 + xxxxxxx. Each codeword's bits are read once, in order. + if reader.read_bits_with_stuffing(1)? == 0 { + return Some(1); } -} - -/// The next bits, up to nine, and how many the header holds: a peek of `n` -/// bits succeeds exactly when `n` single-bit reads would. -fn peek_up_to_nine_bits(reader: &BitReader<'_>) -> (u32, u8) { - if let Some(bits) = reader.clone().read_bits_with_stuffing(9) { - return (bits, 9); + if reader.read_bits_with_stuffing(1)? == 0 { + return Some(2); } - let mut probe = reader.clone(); - let mut bits = 0_u32; - let mut available = 0_u8; - while available < 9 { - let Some(bit) = probe.read_bits_with_stuffing(1) else { - break; - }; - bits = (bits << 1) | bit; - available += 1; + let two = reader.read_bits_with_stuffing(2)?; + if two != 0b11 { + return u8::try_from(two + 3).ok(); + } + let five = reader.read_bits_with_stuffing(5)?; + if five != 0b1_1111 { + return u8::try_from(five + 6).ok(); } - (bits, available) + u8::try_from(reader.read_bits_with_stuffing(7)? + 37).ok() } fn read_lblock_increment(reader: &mut BitReader<'_>) -> Option { From d4a34b62f9f24f546c24b215edb5199b7283f61b Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:36:06 -0400 Subject: [PATCH 14/53] perf(j2k-native): record a single-segment classic contribution without walking its passes Without per-pass termination or selective bypass, every pass of a classic contribution belongs to the layer's one segment: push that segment directly (same index, pass count and length read) instead of grouping pass by pass. Measured with the previous commit; see there. --- crates/j2k-native/src/j2c/segment/classic.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/j2k-native/src/j2c/segment/classic.rs b/crates/j2k-native/src/j2c/segment/classic.rs index bf534fda7..6fb191c88 100644 --- a/crates/j2k-native/src/j2c/segment/classic.rs +++ b/crates/j2k-native/src/j2c/segment/classic.rs @@ -156,6 +156,17 @@ pub(super) fn parse_contribution_lengths( .ok_or("classic segment metadata allocation failed") }; + if !style.termination_on_each_pass && !style.selective_arithmetic_coding_bypass { + // Every pass of the contribution goes to the layer's one segment. + if added_coding_passes != 0 { + push_segment(code_block.non_empty_layer_count, added_coding_passes)?; + } + if first_length.is_some() { + return Err("classic contribution did not consume its prefetched length"); + } + code_block.number_of_coding_passes = cumulative_passes; + return Ok(()); + } let mut last_segment = segment_idx(previous_passes); let mut passes_in_segment = 0; for coding_pass in previous_passes..cumulative_passes { From 4e9030459ccde18ca69a299aaf9cb04371f61138 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:40:18 -0400 Subject: [PATCH 15/53] perf(j2k-native): lend scratch precincts without recounting their tree nodes ParseScratch::lend counted each tag tree's nodes, then TagTree::place counted them again; place now takes the count. The scratch code-blocks are filled row by row instead of dividing each index by the grid width. Measured against the commit before, on an 18-core ARM64 machine, 7 interleaved rounds of J2K_PROFILE_STAGES: packet-header parse -0.6% to -1.5% on the regions of the 18000 x 12000 no-PLT and HTJ2K test images (-1.1% HTJ2K). --- crates/j2k-native/src/j2c/build.rs | 24 +++++++++++++++++------- crates/j2k-native/src/j2c/tag_tree.rs | 12 +++++++++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/crates/j2k-native/src/j2c/build.rs b/crates/j2k-native/src/j2c/build.rs index 5a49770f4..700ad4d2d 100644 --- a/crates/j2k-native/src/j2c/build.rs +++ b/crates/j2k-native/src/j2c/build.rs @@ -636,9 +636,9 @@ impl ParseScratch { let columns = target.code_inclusion_tree.width(); let count = (columns as usize).checked_mul(target.code_inclusion_tree.height() as usize)?; let inclusion_nodes = target.code_inclusion_tree.node_count(); + let zero_bitplane_nodes = target.zero_bitplane_tree.node_count(); if count > self.code_blocks[slot] - || inclusion_nodes.checked_add(target.zero_bitplane_tree.node_count())? - > self.nodes[slot] + || inclusion_nodes.checked_add(zero_bitplane_nodes)? > self.nodes[slot] { return None; } @@ -646,14 +646,24 @@ impl ParseScratch { let blocks = code_blocks.get_mut(start..start.checked_add(count)?)?; // A rectangle that meets no window: no scratch code-block is read. let unread = IntRect::from_ltrb(0, 0, 0, 0); - for (block, index) in blocks.iter_mut().zip(0u32..) { - *block = CodeBlock::new(unread, index % columns, index / columns, 0..0); + // Row-major over the grid: code-block `index` sits at + // (`index % columns`, `index / columns`). + if columns != 0 { + for (row, y_idx) in blocks.chunks_exact_mut(columns as usize).zip(0u32..) { + for (block, x_idx) in row.iter_mut().zip(0u32..) { + *block = CodeBlock::new(unread, x_idx, y_idx, 0..0); + } + } } let root = self.node_start[slot]; - target.code_inclusion_tree.place(root, tag_tree_nodes)?; target - .zero_bitplane_tree - .place(root.checked_add(inclusion_nodes)?, tag_tree_nodes)?; + .code_inclusion_tree + .place(root, inclusion_nodes, tag_tree_nodes)?; + target.zero_bitplane_tree.place( + root.checked_add(inclusion_nodes)?, + zero_bitplane_nodes, + tag_tree_nodes, + )?; target.code_blocks = start..start + count; *borrower = Some(precinct); Some(()) diff --git a/crates/j2k-native/src/j2c/tag_tree.rs b/crates/j2k-native/src/j2c/tag_tree.rs index d80289cca..330502361 100644 --- a/crates/j2k-native/src/j2c/tag_tree.rs +++ b/crates/j2k-native/src/j2c/tag_tree.rs @@ -72,9 +72,15 @@ impl TagTree { } /// Start the tree afresh on the nodes from `root`, as [`Self::new`] - /// would build it there. Fails if they don't fit in `nodes`. - pub(crate) fn place(&mut self, root: usize, nodes: &mut [TagNode]) -> Option<()> { - let end = root.checked_add(self.node_count())?; + /// would build it there. `node_count` is [`Self::node_count`], which the + /// caller has at hand. Fails if they don't fit in `nodes`. + pub(crate) fn place( + &mut self, + root: usize, + node_count: usize, + nodes: &mut [TagNode], + ) -> Option<()> { + let end = root.checked_add(node_count)?; nodes.get_mut(root..end)?.fill(TagNode::default()); self.root = root; Some(()) From 89a6895b8bc7f029df8240be8b4c2f05ceb2d7f2 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:49:33 -0400 Subject: [PATCH 16/53] perf(j2k-native): count PLT lengths without decoding them twice PLT and PLM markers sized their length vector by decoding every length once, then decoded them again to store them. The decoding walk cannot fail in its arithmetic (a 7-bit shift into cleared bits), so its count is the number of bytes with a clear high bit, and its only error is a last byte that leaves a length unfinished. Count that way; the storing pass is unchanged (randomized test against the walk). Measured with the next commit (unread resolutions) against the commit before this one, on an 18-core ARM64 machine, 5 interleaved rounds, on 18 threads: the 18000 x 12000 PLT test image's middle, top-left and bottom-right regions -1.9%, -0.6% and -0.5%, the image reduced four levels -0.4%; the HTJ2K one reduced four levels -1.8%; other rows within noise. --- .../codestream/auxiliary/packet_lengths.rs | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/crates/j2k-native/src/j2c/codestream/auxiliary/packet_lengths.rs b/crates/j2k-native/src/j2c/codestream/auxiliary/packet_lengths.rs index 23f8ff3d8..e84ab0c6b 100644 --- a/crates/j2k-native/src/j2c/codestream/auxiliary/packet_lengths.rs +++ b/crates/j2k-native/src/j2c/codestream/auxiliary/packet_lengths.rs @@ -32,7 +32,7 @@ pub(crate) fn plm_marker( let mut packet_count = 0_usize; visit_plm_chunks(payload, |chunk| { packet_count = packet_count - .checked_add(visit_packet_lengths(chunk, "PLM", |_| {})?) + .checked_add(count_packet_lengths(chunk, "PLM")?) .ok_or(ValidationError::ImageTooLarge)?; Ok(()) })?; @@ -83,7 +83,7 @@ fn decode_packet_lengths_with_limit( max_owned_bytes: usize, marker: &'static str, ) -> Result> { - let packet_count = visit_packet_lengths(data, marker, |_| {})?; + let packet_count = count_packet_lengths(data, marker)?; validate_output_bytes(packet_count, max_owned_bytes)?; let mut packet_lengths = Vec::new(); @@ -114,6 +114,17 @@ fn visit_plm_chunks(payload: &[u8], mut visit: impl FnMut(&[u8]) -> Result<()>) Ok(()) } +/// The lengths in `data`, as [`visit_packet_lengths`] counts them: each ends +/// at a byte with a clear high bit, and the last byte must end one. Its +/// value arithmetic cannot fail (a 7-bit shift, then an add into the cleared +/// low bits), so counting the ending bytes is enough. +fn count_packet_lengths(data: &[u8], marker: &'static str) -> Result { + if data.last().is_some_and(|byte| byte & 0x80 != 0) { + return Err(MarkerError::ParseFailure(marker).into()); + } + Ok(data.iter().filter(|&&byte| byte & 0x80 == 0).count()) +} + fn visit_packet_lengths( data: &[u8], marker: &'static str, @@ -151,6 +162,32 @@ mod tests { use super::*; use crate::error::DecodeError; + #[test] + fn packet_length_counts_match_the_decoding_walk() { + let mut state = 0x2545_f491_4f6c_dd1d_u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for _ in 0..2000 { + let len = usize::try_from(next() % 16).expect("small length"); + let data: Vec = (0..len) + .map(|_| match next() % 3 { + 0 => 0x80 | u8::try_from(next() >> 57).expect("seven bits"), + _ => u8::try_from(next() >> 56).expect("top byte"), + }) + .collect(); + let walked = visit_packet_lengths(&data, "PLT", |_| {}); + let counted = count_packet_lengths(&data, "PLT"); + assert_eq!(walked.is_ok(), counted.is_ok(), "{data:02x?}"); + if let (Ok(walked), Ok(counted)) = (walked, counted) { + assert_eq!(walked, counted, "{data:02x?}"); + } + } + } + #[test] fn plt_output_limit_is_checked_before_reservation() { // Lplt=6: one sequence byte followed by three one-byte packet lengths. From 88d26ee24696f48532089f27e11a35c6e6f9f5bd Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:51:45 -0400 Subject: [PATCH 17/53] perf(j2k-native): mark the precincts of unread resolutions unused without their geometry When the ROI plan reads none of a resolution's sub-bands (a reduced-resolution decode's finer levels), region selection marks its precincts unused without computing each precinct's rectangle and code-block grid. The index check the rectangle lookup made is kept: PrecinctGrid::rect_count is the number of indices rect_at answers. Measured with the previous commit (PLT length count); see there. --- crates/j2k-native/src/j2c/build/selection.rs | 17 +++++++++++++++++ crates/j2k-native/src/j2c/tile.rs | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/crates/j2k-native/src/j2c/build/selection.rs b/crates/j2k-native/src/j2c/build/selection.rs index 464af68b3..d58fcf846 100644 --- a/crates/j2k-native/src/j2c/build/selection.rs +++ b/crates/j2k-native/src/j2c/build/selection.rs @@ -427,6 +427,10 @@ fn push_within_capacity(values: &mut Vec, value: T) -> Result<()> { Ok(()) } +#[expect( + clippy::too_many_lines, + reason = "one pass marks every component's and resolution's precincts in the order the use table indexes them" +)] fn select( tile: &Tile<'_>, storage: &DecompositionStorage<'_>, @@ -481,6 +485,19 @@ fn select( .ok_or(DecodingError::InvalidPrecinct)?; let precinct_count = usize::try_from(resolution_tile.num_precincts()) .map_err(|_| ValidationError::ImageTooLarge)?; + if windows.iter().all(Option::is_none) { + // The plan reads none of this resolution's sub-bands (a + // reduced decode's finer levels): every precinct is unused, + // and only the index check of its rectangle remains. + let rects = precincts.rect_count(); + for precinct_idx in 0..precinct_count { + if precinct_idx >= rects { + return Err(DecodingError::InvalidPrecinct.into()); + } + push_within_capacity(&mut uses, PrecinctUse::Unused)?; + } + continue; + } for precinct_idx in 0..precinct_count { let precinct_rect = precincts .rect_at(precinct_idx) diff --git a/crates/j2k-native/src/j2c/tile.rs b/crates/j2k-native/src/j2c/tile.rs index e92922e6c..0aed2a516 100644 --- a/crates/j2k-native/src/j2c/tile.rs +++ b/crates/j2k-native/src/j2c/tile.rs @@ -718,6 +718,12 @@ impl PrecinctGrid { } } + /// How many indices [`Self::rect_at`] accepts: it answers exactly the + /// indices below this. + pub(crate) fn rect_count(self) -> usize { + usize::try_from(u64::from(self.num_x) * u64::from(self.num_y)).unwrap_or(usize::MAX) + } + /// The rectangle of precinct `idx` alone, numbered row by row as /// [`Self::iter`] lists them, or `None` past the last. pub(crate) fn rect_at(self, idx: usize) -> Option { @@ -949,6 +955,7 @@ mod tests { } assert!(grid.at(listed.len() as u64).is_none()); assert!(grid.rect_at(listed.len()).is_none()); + assert_eq!(grid.rect_count(), listed.len()); } } } From 6682aae36b60ca259a712e052f055843b76b45cb Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:06:45 -0400 Subject: [PATCH 18/53] perf(j2k-native): decode small images serially when forking costs more An Auto full-image decode whose estimated serial time (1 ns per component sample plus 1 ns per coded HT byte or 72 ns per classic byte) is below 25 us per pool thread runs Serial: Tier-1, IDWT, MCT and pack on the calling thread. Explicit parallelism and region decodes are unchanged, and a non-Auto request no longer asks rayon for its thread count (which would start the pool). Tests keep their parallel routes unless they opt in. Measured against the previous commit on an 18-core ARM64 machine, 5 interleaved rounds: HT lossless 128^2 -56% on 18 threads and -20% on 6; HT 2 bpp 256^2 -26% on 18; classic 256^2 +0.5% on 18, HT 512^2 +0.3% on 6; region rows (not affected) within +-1.6%. --- crates/j2k-native/src/image.rs | 92 ++++++++++++++++++- crates/j2k-native/src/j2c/decode/state.rs | 8 ++ .../j2k-native/src/tests/workspace_reuse.rs | 47 ++++++++++ 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/crates/j2k-native/src/image.rs b/crates/j2k-native/src/image.rs index 0c6e23bb1..6ba141fa7 100644 --- a/crates/j2k-native/src/image.rs +++ b/crates/j2k-native/src/image.rs @@ -489,7 +489,8 @@ impl<'a> Image<'a> { pub fn decode_with_context(&self, decoder_context: &mut DecoderContext<'a>) -> Result { self.with_packet_cache_retry(|| { let retained_image_bytes = self.retained_header_metadata_bytes()?; - let parallel = decoder_context.cpu_decode_parallelism() == CpuDecodeParallelism::Auto; + let parallel = + self.decode_parallelism(decoder_context, None) == CpuDecodeParallelism::Auto; let (mct, first_ll, scratch, mut decoded_image, retained_image_bytes) = self.decode_image_for_rgb8(decoder_context, None, retained_image_bytes)?; let component_owner_capacity = decoded_image.decoded_components.capacity(); @@ -804,7 +805,8 @@ impl<'a> Image<'a> { decoder_context: &mut DecoderContext<'a>, ) -> Result<()> { self.with_packet_cache_retry(|| { - let parallel = decoder_context.cpu_decode_parallelism() == CpuDecodeParallelism::Auto; + let parallel = + self.decode_parallelism(decoder_context, None) == CpuDecodeParallelism::Auto; let retained = self.retained_header_metadata_bytes()?; let (mct, first_ll, scratch, mut decoded_image, _retained_image_bytes) = self.decode_image_for_rgb8(decoder_context, None, retained)?; @@ -935,6 +937,92 @@ impl<'a> Image<'a> { round_irreversible_output: bool, retained_baseline_bytes: usize, row_sink: Option<&mut dyn crate::ComponentRowSink>, + ) -> Result { + let requested = decoder_context.cpu_decode_parallelism(); + let parallelism = self.decode_parallelism(decoder_context, output_region); + decoder_context.set_cpu_decode_parallelism(parallelism); + let decoded = self.run_decode_rows_with_parallelism( + decoder_context, + output_region, + ht_decoder, + round_irreversible_output, + retained_baseline_bytes, + row_sink, + ); + decoder_context.set_cpu_decode_parallelism(requested); + decoded + } + + /// The parallelism a decode with `decoder_context` runs with: an `Auto` + /// full-image decode whose estimated serial time is shorter than waking + /// the pool's threads costs runs serially. The estimate counts the + /// samples and the coded bytes, which classic code-blocks take far longer + /// to decode than HT ones. + pub(crate) fn decode_parallelism( + &self, + decoder_context: &DecoderContext<'_>, + output_region: Option<(u32, u32, u32, u32)>, + ) -> CpuDecodeParallelism { + /// Estimated serial nanoseconds per component sample and per coded + /// byte of HT and of classic code-blocks, and the fork cost per pool + /// thread that a decode must repay. + const SAMPLE_NS: u64 = 1; + const HT_BYTE_NS: u64 = 1; + const CLASSIC_BYTE_NS: u64 = 72; + const FORK_NS_PER_THREAD: u64 = 25_000; + let requested = decoder_context.cpu_decode_parallelism(); + if requested != CpuDecodeParallelism::Auto || output_region.is_some() { + return requested; + } + // Only an `Auto` decode asks for the pool, which starts it. + #[cfg(feature = "parallel")] + let threads = rayon::current_num_threads(); + #[cfg(not(feature = "parallel"))] + let threads = 1_usize; + #[cfg(test)] + let threads = if decoder_context.tile_decode_context.serial_small_decodes { + threads + } else { + 1 + }; + if threads < 2 { + return requested; + } + let samples = u64::from(self.width()) + .saturating_mul(u64::from(self.height())) + .saturating_mul(u64::try_from(self.header.component_infos.len()).unwrap_or(u64::MAX)); + let byte_ns = if self.header.component_infos.iter().all(|component| { + component + .coding_style + .parameters + .code_block_style + .uses_high_throughput_block_coding() + }) { + HT_BYTE_NS + } else { + CLASSIC_BYTE_NS + }; + let serial_ns = samples.saturating_mul(SAMPLE_NS).saturating_add( + u64::try_from(self.codestream.len()) + .unwrap_or(u64::MAX) + .saturating_mul(byte_ns), + ); + let fork_ns = FORK_NS_PER_THREAD.saturating_mul(u64::try_from(threads).unwrap_or(u64::MAX)); + if serial_ns < fork_ns { + CpuDecodeParallelism::Serial + } else { + requested + } + } + + fn run_decode_rows_with_parallelism( + &self, + decoder_context: &mut DecoderContext<'a>, + output_region: Option<(u32, u32, u32, u32)>, + ht_decoder: Option<&mut dyn HtCodeBlockDecoder>, + round_irreversible_output: bool, + retained_baseline_bytes: usize, + row_sink: Option<&mut dyn crate::ComponentRowSink>, ) -> Result { let mut ht_decoder = ht_decoder; let mut row_sink = row_sink; diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index 8b17a70ed..dc5be4e3c 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -275,6 +275,11 @@ pub(crate) struct TileDecodeContext { /// pack apart, so the fused pass can be compared against them. #[cfg(test)] pub(crate) staged_rgb8_pack: bool, + /// Test-only switch that lets an `Auto` decode of a small image run + /// serially, as every other build does. Tests leave it off so their + /// small fixtures still exercise the parallel routes. + #[cfg(test)] + pub(crate) serial_small_decodes: bool, /// The raw, decoded samples for each channel. pub(crate) channel_data: Vec, /// Optional output window for region-local decode storage. @@ -336,6 +341,8 @@ impl TileDecodeContext { let separate_last_idwt_level = self.separate_last_idwt_level; #[cfg(test)] let staged_rgb8_pack = self.staged_rgb8_pack; + #[cfg(test)] + let serial_small_decodes = self.serial_small_decodes; *self = Self::default(); self.output_region = output_region; self.round_irreversible_output = round_irreversible_output; @@ -353,6 +360,7 @@ impl TileDecodeContext { self.reject_band_index = reject_band_index; self.separate_last_idwt_level = separate_last_idwt_level; self.staged_rgb8_pack = staged_rgb8_pack; + self.serial_small_decodes = serial_small_decodes; } } diff --git a/crates/j2k-native/src/tests/workspace_reuse.rs b/crates/j2k-native/src/tests/workspace_reuse.rs index a2d82e4db..9c328c34c 100644 --- a/crates/j2k-native/src/tests/workspace_reuse.rs +++ b/crates/j2k-native/src/tests/workspace_reuse.rs @@ -287,6 +287,53 @@ fn parallel_workspace_measurements(context: &DecoderContext<'_>) -> (usize, usiz ) } +/// An `Auto` decode of an image too small to repay forking runs serially, +/// with the same bytes, and leaves the context's policy as it was. +#[cfg(feature = "parallel")] +#[test] +fn small_auto_decodes_run_serially_with_the_same_bytes() { + let bytes = parallel_gray_fixture(257, 263, true, false); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("small image"); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(8) + .build() + .expect("test pool"); + pool.install(|| { + let mut parallel = DecoderContext::default(); + let expected = image + .decode_with_context(&mut parallel) + .expect("parallel decode"); + assert!( + parallel + .tile_decode_context + .debug_counters + .ht_parallel_tasks + > 0 + ); + let mut small = DecoderContext::default(); + small.tile_decode_context.serial_small_decodes = true; + let actual = image.decode_with_context(&mut small).expect("small decode"); + assert_eq!( + small.tile_decode_context.debug_counters.ht_parallel_tasks, + 0 + ); + assert_eq!( + small.cpu_decode_parallelism(), + crate::CpuDecodeParallelism::Auto + ); + assert_eq!(actual.data, expected.data); + // A larger image still forks. + let large = parallel_gray_fixture(1024, 1024, true, false); + let large = Image::new(&large, &DecodeSettings::default()).expect("large image"); + let mut context = DecoderContext::default(); + context.tile_decode_context.serial_small_decodes = true; + large + .decode_with_context(&mut context) + .expect("large decode"); + assert!(context.tile_decode_context.debug_counters.ht_parallel_tasks > 0); + }); +} + #[cfg(feature = "parallel")] #[test] fn ht_task_workspaces_match_serial_and_reuse_capacity() { From d2538169ec71f14008aa2a6d986f825550170bc0 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:44:14 -0400 Subject: [PATCH 19/53] perf(j2k-native): run a parallel decode_into on the pool from its start A decode_into called from outside the Rayon pool entered the pool once per parallel stage: each component's Tier-1, the IDWT and the MCT and pack were injected from the caller's thread, which slept until a pool thread had finished each of them. An Auto decode now enters the pool once, through rayon::scope, so its later stages start from a pool thread. It stays where it is when called from a pool thread, with a one-thread pool, with explicit parallelism (checked first, so it never starts the pool), with stage profiling on (a pool thread would keep its rows), and when its estimated serial time is under 1.25 times the fork cost: a decode only just worth forking spends much of its time in serial parts. A test-only switch keeps per-thread test counters on the test thread. Measured against the parent (built twice, as A and an A/A control) on an 18-core ARM64 machine, 5 interleaved rounds, median of per-round ratios, full-image RGB8 decodes: at 6 threads HT 512^2 -15%, HT lossless 256^2 -14% and 512^2 -13%, classic 192^2 -15% and 512^2 -9%, HT 4096x2048 mosaics -4% to -5%; at 18 threads HT lossless 512^2 -14%, HT 512^2 -11%, classic 512^2 -8% and 256^2 -5%, HT mosaics -5% to -8%, classic mosaics -2% to -3%. One-thread decodes, decodes that already run serially, region decodes and concurrent region decodes are unchanged within the A/A spread. Output and heap peak are unchanged. --- crates/j2k-native/src/image.rs | 77 +++++++++++++++---- crates/j2k-native/src/j2c/decode/state.rs | 8 ++ .../j2k-native/src/tests/workspace_reuse.rs | 32 ++++++++ 3 files changed, 102 insertions(+), 15 deletions(-) diff --git a/crates/j2k-native/src/image.rs b/crates/j2k-native/src/image.rs index 6ba141fa7..fb3a8f45d 100644 --- a/crates/j2k-native/src/image.rs +++ b/crates/j2k-native/src/image.rs @@ -803,6 +803,46 @@ impl<'a> Image<'a> { &self, buf: &mut [u8], decoder_context: &mut DecoderContext<'a>, + ) -> Result<()> { + #[cfg(feature = "parallel")] + if self.enters_pool(decoder_context) { + // One entry into the pool for the whole decode: its parallel + // stages then start from a pool thread instead of each waking one + // from the caller's thread and waiting for it there. + return rayon::scope(|_| self.decode_into_here(buf, decoder_context)); + } + self.decode_into_here(buf, decoder_context) + } + + /// Whether [`Self::decode_into`] moves onto the pool: an `Auto` decode + /// called from outside a pool of two or more threads whose estimated + /// serial time is at least 1.25 times the fork cost, unless stage + /// profiling is on (a pool thread would keep its rows). With one thread + /// the move only adds a hop between threads, and a decode only just worth + /// forking spends much of its time in serial parts, which then run on a + /// pool thread. + #[cfg(feature = "parallel")] + pub(crate) fn enters_pool(&self, decoder_context: &DecoderContext<'_>) -> bool { + #[cfg(test)] + if !decoder_context.tile_decode_context.enter_pool { + return false; + } + // Explicit parallelism is checked first so that it never starts the pool. + if decoder_context.cpu_decode_parallelism() != CpuDecodeParallelism::Auto + || rayon::current_thread_index().is_some() + || crate::profile::profile_stages_enabled() + { + return false; + } + let threads = rayon::current_num_threads(); + let (serial_ns, fork_ns) = self.full_decode_cost_ns(threads); + threads >= 2 && serial_ns.saturating_mul(4) >= fork_ns.saturating_mul(5) + } + + fn decode_into_here( + &self, + buf: &mut [u8], + decoder_context: &mut DecoderContext<'a>, ) -> Result<()> { self.with_packet_cache_retry(|| { let parallel = @@ -955,21 +995,12 @@ impl<'a> Image<'a> { /// The parallelism a decode with `decoder_context` runs with: an `Auto` /// full-image decode whose estimated serial time is shorter than waking - /// the pool's threads costs runs serially. The estimate counts the - /// samples and the coded bytes, which classic code-blocks take far longer - /// to decode than HT ones. + /// the pool's threads costs runs serially. pub(crate) fn decode_parallelism( &self, decoder_context: &DecoderContext<'_>, output_region: Option<(u32, u32, u32, u32)>, ) -> CpuDecodeParallelism { - /// Estimated serial nanoseconds per component sample and per coded - /// byte of HT and of classic code-blocks, and the fork cost per pool - /// thread that a decode must repay. - const SAMPLE_NS: u64 = 1; - const HT_BYTE_NS: u64 = 1; - const CLASSIC_BYTE_NS: u64 = 72; - const FORK_NS_PER_THREAD: u64 = 25_000; let requested = decoder_context.cpu_decode_parallelism(); if requested != CpuDecodeParallelism::Auto || output_region.is_some() { return requested; @@ -988,6 +1019,26 @@ impl<'a> Image<'a> { if threads < 2 { return requested; } + let (serial_ns, fork_ns) = self.full_decode_cost_ns(threads); + if serial_ns < fork_ns { + CpuDecodeParallelism::Serial + } else { + requested + } + } + + /// A full-image decode's estimated serial nanoseconds, and what waking + /// `threads` pool threads costs it. The estimate counts the samples and + /// the coded bytes, which classic code-blocks take far longer to decode + /// than HT ones. + pub(crate) fn full_decode_cost_ns(&self, threads: usize) -> (u64, u64) { + /// Estimated serial nanoseconds per component sample and per coded + /// byte of HT and of classic code-blocks, and the fork cost per pool + /// thread that a decode must repay. + const SAMPLE_NS: u64 = 1; + const HT_BYTE_NS: u64 = 1; + const CLASSIC_BYTE_NS: u64 = 72; + const FORK_NS_PER_THREAD: u64 = 25_000; let samples = u64::from(self.width()) .saturating_mul(u64::from(self.height())) .saturating_mul(u64::try_from(self.header.component_infos.len()).unwrap_or(u64::MAX)); @@ -1008,11 +1059,7 @@ impl<'a> Image<'a> { .saturating_mul(byte_ns), ); let fork_ns = FORK_NS_PER_THREAD.saturating_mul(u64::try_from(threads).unwrap_or(u64::MAX)); - if serial_ns < fork_ns { - CpuDecodeParallelism::Serial - } else { - requested - } + (serial_ns, fork_ns) } fn run_decode_rows_with_parallelism( diff --git a/crates/j2k-native/src/j2c/decode/state.rs b/crates/j2k-native/src/j2c/decode/state.rs index dc5be4e3c..948d376d1 100644 --- a/crates/j2k-native/src/j2c/decode/state.rs +++ b/crates/j2k-native/src/j2c/decode/state.rs @@ -280,6 +280,11 @@ pub(crate) struct TileDecodeContext { /// small fixtures still exercise the parallel routes. #[cfg(test)] pub(crate) serial_small_decodes: bool, + /// Test-only switch that lets a parallel `decode_into` called from outside + /// the pool move onto it, as every other build does. Tests leave it off so + /// their per-thread counters see the decode. + #[cfg(test)] + pub(crate) enter_pool: bool, /// The raw, decoded samples for each channel. pub(crate) channel_data: Vec, /// Optional output window for region-local decode storage. @@ -343,6 +348,8 @@ impl TileDecodeContext { let staged_rgb8_pack = self.staged_rgb8_pack; #[cfg(test)] let serial_small_decodes = self.serial_small_decodes; + #[cfg(test)] + let enter_pool = self.enter_pool; *self = Self::default(); self.output_region = output_region; self.round_irreversible_output = round_irreversible_output; @@ -361,6 +368,7 @@ impl TileDecodeContext { self.separate_last_idwt_level = separate_last_idwt_level; self.staged_rgb8_pack = staged_rgb8_pack; self.serial_small_decodes = serial_small_decodes; + self.enter_pool = enter_pool; } } diff --git a/crates/j2k-native/src/tests/workspace_reuse.rs b/crates/j2k-native/src/tests/workspace_reuse.rs index 9c328c34c..e75effd51 100644 --- a/crates/j2k-native/src/tests/workspace_reuse.rs +++ b/crates/j2k-native/src/tests/workspace_reuse.rs @@ -287,6 +287,38 @@ fn parallel_workspace_measurements(context: &DecoderContext<'_>) -> (usize, usiz ) } +/// A parallel `decode_into` from outside the pool runs on it with the same +/// bytes; a serial one and one from a pool thread stay where they are. +#[cfg(feature = "parallel")] +#[test] +fn decode_into_enters_the_pool_with_the_same_bytes() { + let bytes = parallel_gray_fixture(1024, 1024, true, false); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let len = usize::try_from(image.width() * image.height()).expect("pixels"); + let mut expected = alloc::vec![0_u8; len]; + let mut here = DecoderContext::default(); + image + .decode_into(&mut expected, &mut here) + .expect("decode here"); + assert!(!image.enters_pool(&here)); + let mut pooled = DecoderContext::default(); + pooled.tile_decode_context.enter_pool = true; + let threads = rayon::current_num_threads(); + let (serial_ns, fork_ns) = image.full_decode_cost_ns(threads); + assert_eq!( + image.enters_pool(&pooled), + !crate::profile::profile_stages_enabled() && threads >= 2 && serial_ns * 4 >= fork_ns * 5 + ); + let mut actual = alloc::vec![0_u8; len]; + image + .decode_into(&mut actual, &mut pooled) + .expect("decode on the pool"); + assert_eq!(actual, expected); + rayon::scope(|_| assert!(!image.enters_pool(&pooled))); + pooled.set_cpu_decode_parallelism(crate::CpuDecodeParallelism::Serial); + assert!(!image.enters_pool(&pooled)); +} + /// An `Auto` decode of an image too small to repay forking runs serially, /// with the same bytes, and leaves the context's policy as it was. #[cfg(feature = "parallel")] From 5882fe982515dcfdeb91d9a7f46972a83c3e9155 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 01:05:24 -0400 Subject: [PATCH 20/53] perf(j2k-native): split long classic code-block rows on the tile route The tile route queues one task per code-block row, largest first. Classic rows at coarse resolution levels are few and very unequal (an LL or level-4 row can carry many times the work of a level-1 row), so the queue ends on a long row while the other workers wait. Classic rows whose estimated work exceeds total / (8 x workers) are now split into runs of consecutive blocks of about equal work. The parts of a row share its output through a mutex: each block decodes into the worker's block scratch and is copied in under the lock (in a stale buffer each part first zeroes its own columns). HT rows stay whole. The extra queue entries, the mutex vector and one block of scratch per worker are charged to the decode budget up front. Unit tests cover the run splitting and the part counts. Interleaved A/B against the parent: full-image classic decodes are unchanged within +-1.5% at 6, 18 and default threads (their 79-94 rows per component already balance). On tile-route reduced-resolution decodes of an 18000 x 12000 classic image at 18 threads (measured with the band route switched off), 1/16 scale takes 15% less time and 1/8 scale 3.5% less. Output is byte-identical. --- .../src/j2c/decode/subband/tile_parallel.rs | 451 +++++++++++++++--- 1 file changed, 394 insertions(+), 57 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index cfb892bc3..d91394e3d 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -248,26 +248,17 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( } let workers = rayon::current_num_threads().min(plan.strips.len()); - // A stale buffer adds a zeroing task for each gap around the strips. - let task_count = if storage.stale_coefficients { - 2 * plan.strips.len() + 1 - } else { - plan.strips.len() - }; - let mut tasks: Vec> = Vec::new(); + let counts = TaskCounts::new(&plan, workers, storage.stale_coefficients)?; let route_budget = (|| -> Result<()> { // Payloads are gathered only once the tile takes the route. gather_payloads(&mut plan, storage, &mut budget)?; - tasks - .try_reserve_exact(task_count) - .map_err(|_| DecodingError::HostAllocationFailed)?; // `plan_tile` charged one task per strip. - budget.include_elements::>(task_count - plan.strips.len())?; - budget.include_capacity_overage::>(task_count, tasks.capacity())?; - // Each worker allocates its clipped-block scratch on first use. + budget.include_elements::>(counts.tasks - plan.strips.len())?; + budget.include_elements::>(counts.shared)?; + // Each worker allocates its block scratch on first use. budget.include_elements::( workers - .checked_mul(plan.max_clipped_samples) + .checked_mul(counts.scratch_samples) .ok_or(ValidationError::ImageTooLarge)?, )?; if has_later_components { @@ -284,9 +275,8 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( } return Err(error); } - let task_capacity = tasks.capacity(); let Some(growths) = - decode_with_retained_bank(&plan, workers, tasks, tile_ctx, storage, &mut budget)? + decode_with_retained_bank(&plan, workers, counts, tile_ctx, storage, &mut budget)? else { // The per-sub-band path has its own smaller-footprint fallbacks. return Ok(TileRoute::PerSubBand); @@ -295,7 +285,7 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( if let Some(post_decode) = tile_ctx.tile_route_post_decode.take() { post_decode(storage)?; } - record_route_peak(tile_ctx, storage, &plan, workers, task_capacity)?; + record_route_peak(tile_ctx, storage, &plan, workers, counts)?; count_decoded_blocks(tile_ctx, &plan, workers, growths); Ok(TileRoute::Decoded) } @@ -305,7 +295,7 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( fn decode_with_retained_bank<'a>( plan: &TilePlan<'a>, workers: usize, - tasks: Vec>, + counts: TaskCounts, tile_ctx: &mut TileDecodeContext, storage: &mut DecompositionStorage<'a>, budget: &mut DecodeAllocationBudget, @@ -314,7 +304,7 @@ fn decode_with_retained_bank<'a>( CodeBlockCoding::Classic => decode_with_bank( plan, workers, - tasks, + counts, &mut tile_ctx.classic_task_workspaces, storage, budget, @@ -325,7 +315,7 @@ fn decode_with_retained_bank<'a>( CodeBlockCoding::HighThroughput => decode_with_bank( plan, workers, - tasks, + counts, &mut tile_ctx.ht_task_workspaces, storage, budget, @@ -378,7 +368,7 @@ fn record_route_peak( storage: &DecompositionStorage<'_>, plan: &TilePlan<'_>, workers: usize, - task_capacity: usize, + counts: TaskCounts, ) -> Result<()> { let route_bytes = [ plan.bands.capacity() * size_of::(), @@ -387,9 +377,10 @@ fn record_route_peak( plan.strips.capacity() * size_of::(), plan.band_sub_bands.capacity() * size_of::(), plan.arena.capacity(), - task_capacity * size_of::>(), + counts.tasks * size_of::>(), + counts.shared * size_of::>(), workers - .saturating_mul(plan.max_clipped_samples) + .saturating_mul(counts.scratch_samples) .saturating_mul(size_of::()), ] .into_iter() @@ -406,7 +397,7 @@ fn record_route_peak( fn decode_with_bank<'a, W: TaskScratch>( plan: &TilePlan<'a>, workers: usize, - tasks: Vec>, + counts: TaskCounts, bank: &mut Vec>, storage: &mut DecompositionStorage<'a>, budget: &mut DecodeAllocationBudget, @@ -429,7 +420,7 @@ fn decode_with_bank<'a, W: TaskScratch>( &mut coefficients, stale, &mut bank[..workers], - tasks, + counts, |job, output, stride, slot| decode_block(job, output, stride, &mut slot.workspace), ); storage.coefficients = coefficients; @@ -635,7 +626,7 @@ fn plan_tile<'a>( return Ok(None); } plan.arena_len = arena_len; - budget.include_elements::>(plan.strips.len())?; + budget.include_elements::>(plan.strips.len())?; Ok(Some(plan)) } @@ -849,9 +840,7 @@ fn plan_band_strips( return Ok(false); } next_x = placement.target_x + placement.width; - work = work - .saturating_add(job.payload_length as usize) - .saturating_add((job.width as usize * job.height as usize) / 8); + work = work.saturating_add(job_work(job)); end += 1; } if y_offset @@ -888,6 +877,11 @@ fn plan_band_strips( Ok(true) } +/// A block's estimated decode cost, as the route weighs strips and parts. +fn job_work(job: &BlockJob<'_>) -> usize { + (job.payload_length as usize).saturating_add((job.width as usize * job.height as usize) / 8) +} + fn strips_are_disjoint(strips: &mut [Strip], coefficient_len: usize) -> bool { strips.sort_unstable_by_key(|strip| strip.start); let mut previous_end = 0usize; @@ -1023,9 +1017,151 @@ fn prepare_uniform_task_workspaces( Ok(growths) } -/// A strip's output, or with no strip, a range no strip decodes that a -/// stale buffer must zero. -type StripTask<'coefficients, 'plan> = (&'coefficients mut [f32], Option<&'plan Strip>); +/// Parts per worker that a classic tile's code-block rows are split into, +/// so that the queue ends in short tasks: a worker that finds it empty waits +/// for the others' last tasks only, and a strip of a whole row can take a +/// couple of milliseconds. +const CLASSIC_PARTS_PER_WORKER: usize = 8; + +/// One entry of the route's queue. +enum StripTask<'coefficients, 'plan, 'shared> { + /// A range no strip decodes, which a stale buffer must zero. + Zero(&'coefficients mut [f32]), + /// A whole strip, decoded in place into its output. + Strip(&'coefficients mut [f32], &'plan Strip), + /// Jobs of a strip whose output its parts share, with their work. + Part( + &'shared Mutex<&'coefficients mut [f32]>, + &'plan Strip, + Range, + usize, + ), +} + +impl StripTask<'_, '_, '_> { + fn work(&self) -> usize { + match self { + Self::Zero(_) => 0, + Self::Strip(_, strip) => strip.work, + Self::Part(_, _, _, work) => *work, + } + } +} + +/// The sizes of the route's queue, as budgeted before the decode. +#[derive(Clone, Copy)] +struct TaskCounts { + /// Queue entries, at most. + tasks: usize, + /// Strips split into parts, which share their output. + shared: usize, + /// Samples of each worker's block scratch. + scratch_samples: usize, + /// The work a part of a split strip aims at; `None` keeps whole strips. + part_work: Option, +} + +impl TaskCounts { + fn new(plan: &TilePlan<'_>, workers: usize, stale: bool) -> Result { + // HT rows decode quickly enough whole. + let part_work = (plan.coding == CodeBlockCoding::Classic).then(|| { + let total = plan + .strips + .iter() + .fold(0usize, |total, strip| total.saturating_add(strip.work)); + (total / workers.saturating_mul(CLASSIC_PARTS_PER_WORKER).max(1)).max(1) + }); + let mut counts = Self { + tasks: 0, + shared: 0, + scratch_samples: plan.max_clipped_samples, + part_work, + }; + for strip in &plan.strips { + let parts = counts.parts(strip); + counts.tasks = counts.tasks.saturating_add(parts); + counts.shared += usize::from(parts > 1); + } + if stale { + // A zeroing task for each gap around the strips. + counts.tasks = counts.tasks.saturating_add(plan.strips.len() + 1); + } + if counts.shared > 0 { + let block = (plan.max_width as usize) + .checked_mul(plan.max_height as usize) + .ok_or(ValidationError::ImageTooLarge)?; + counts.scratch_samples = counts.scratch_samples.max(block); + } + Ok(counts) + } + + /// How many parts `strip` is split into. + fn parts(&self, strip: &Strip) -> usize { + self.part_work.map_or(1, |part| { + strip.work.div_ceil(part).clamp(1, strip.jobs.len().max(1)) + }) + } +} + +/// Split `strip` into `parts` runs of consecutive jobs of about equal work, +/// each passed to `emit` with its work. +fn split_strip( + plan: &TilePlan<'_>, + strip: &Strip, + parts: usize, + emit: impl FnMut(Range, usize) -> Result<()>, +) -> Result<()> { + let works = strip.jobs.clone().map(|index| { + plan.jobs + .get(index) + .map(job_work) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) + }); + split_runs(strip.jobs.start, works, strip.work, parts, emit) +} + +/// Split consecutive jobs, numbered from `first` and with `works` summing to +/// `total`, into at most `parts` runs of about equal work: a run ends at the +/// first job that takes the work done to its share of `total`. Each run is +/// passed to `emit` with its work. +fn split_runs( + first: usize, + works: impl Iterator>, + total: usize, + parts: usize, + mut emit: impl FnMut(Range, usize) -> Result<()>, +) -> Result<()> { + let mut works = works.peekable(); + let mut start = first; + let mut end = first; + let mut done = 0usize; + let mut part_work = 0usize; + let mut cuts = 1usize; + while let Some(work) = works.next() { + let work = work?; + end += 1; + done = done.saturating_add(work); + part_work = part_work.saturating_add(work); + let last = works.peek().is_none(); + if last || (cuts < parts && done.saturating_mul(parts) >= cuts.saturating_mul(total)) { + emit(start..end, part_work)?; + start = end; + part_work = 0; + cuts += 1; + } + } + Ok(()) +} + +/// Queue `task`. The budget covers [`TaskCounts`], so the queue never grows +/// past the capacity reserved from it. +fn push_task(tasks: &mut Vec, task: T) -> Result<()> { + if tasks.len() == tasks.capacity() { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + tasks.push(task); + Ok(()) +} /// Decode the strips on one worker per workspace, each block through /// `decode_block`. @@ -1034,9 +1170,17 @@ fn decode_strips<'plan, 'coefficients, W: Send>( coefficients: &'coefficients mut [f32], stale: bool, workspaces: &mut [W], - mut tasks: Vec>, + counts: TaskCounts, decode_block: impl Fn(&BlockJob<'_>, &mut [f32], usize, &mut W) -> Result<()> + Sync, ) -> Result<()> { + let mut shared: Vec> = Vec::new(); + shared + .try_reserve_exact(counts.shared) + .map_err(|_| DecodingError::HostAllocationFailed)?; + let mut tasks: Vec> = Vec::new(); + tasks + .try_reserve_exact(counts.tasks) + .map_err(|_| DecodingError::HostAllocationFailed)?; // Strips are sorted by start offset, so carving them front to back yields // disjoint mutable outputs. In a stale buffer each strip zeroes its own // rows on its worker, where the decode then finds them in cache; the @@ -1050,49 +1194,77 @@ fn decode_strips<'plan, 'coefficients, W: Send>( rest = tail; consumed = strip.start + strip.len; if stale && !gap.is_empty() { - tasks.push((gap, None)); + push_task(&mut tasks, StripTask::Zero(gap))?; + } + if counts.parts(strip) > 1 { + if shared.len() == shared.capacity() { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + } + shared.push(Mutex::new(output)); + } else { + push_task(&mut tasks, StripTask::Strip(output, strip))?; } - tasks.push((output, Some(strip))); } if stale && !rest.is_empty() { - tasks.push((rest, None)); + push_task(&mut tasks, StripTask::Zero(rest))?; + } + let mut outputs = shared.iter(); + for strip in &plan.strips { + let parts = counts.parts(strip); + if parts > 1 { + let output = outputs + .next() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + split_strip(plan, strip, parts, |jobs, work| { + push_task(&mut tasks, StripTask::Part(output, strip, jobs, work)) + })?; + } } - // Largest rows first so stragglers are small; zeroing tasks last. - tasks - .sort_unstable_by_key(|(_, strip)| core::cmp::Reverse(strip.map_or(0, |strip| strip.work))); + // Largest tasks first so stragglers are small; zeroing tasks last. + tasks.sort_unstable_by_key(|task| core::cmp::Reverse(task.work())); let queue = Mutex::new(tasks.into_iter()); let failed = AtomicBool::new(false); // Each worker owns one already-budgeted workspace for the whole tile. - // A single queue pop per strip keeps work balanced across unequal cores + // A single queue pop per task keeps work balanced across unequal cores // without forking a Rayon job and borrowing/returning scratch for each row. workspaces .par_iter_mut() .with_max_len(1) .try_for_each(|workspace| -> Result<()> { - let mut clipped_scratch = Vec::new(); + let mut scratch = Vec::new(); while !failed.load(Ordering::Relaxed) { let next = queue .lock() .map_err(|_| DecodingError::CodeBlockDecodeFailure)? .next(); - let Some((output, strip)) = next else { + let Some(task) = next else { break; }; - if stale { - output.fill(0.0); - } - let Some(strip) = strip else { - continue; + let decoded = match task { + StripTask::Zero(output) => { + output.fill(0.0); + Ok(()) + } + StripTask::Strip(output, strip) => { + if stale { + output.fill(0.0); + } + decode_strip(plan, strip, output, workspace, &mut scratch, &decode_block) + } + StripTask::Part(output, strip, jobs, _) => decode_part( + plan, + strip, + jobs, + output, + stale, + counts.scratch_samples, + workspace, + &mut scratch, + &decode_block, + ), }; - if let Err(error) = decode_strip( - plan, - strip, - output, - workspace, - &mut clipped_scratch, - &decode_block, - ) { + if let Err(error) = decoded { failed.store(true, Ordering::Relaxed); return Err(error); } @@ -1101,6 +1273,75 @@ fn decode_strips<'plan, 'coefficients, W: Send>( }) } +/// Decode jobs `jobs` of `strip`, whose output its parts share: each block +/// into scratch, then into the output under its lock. In a stale buffer the +/// part first zeroes its columns, from its first block to the next part's. +#[expect( + clippy::too_many_arguments, + reason = "a part carries its strip's shared output and the worker's scratch explicitly" +)] +fn decode_part( + plan: &TilePlan<'_>, + strip: &Strip, + jobs: Range, + output: &Mutex<&mut [f32]>, + stale: bool, + scratch_samples: usize, + workspace: &mut W, + scratch: &mut Vec, + decode_block: &impl Fn(&BlockJob<'_>, &mut [f32], usize, &mut W) -> Result<()>, +) -> Result<()> { + let lock = || { + output + .lock() + .map_err(|_| DecodingError::CodeBlockDecodeFailure) + }; + if stale { + let column = |index: usize| { + if index == strip.jobs.start { + Some(0) + } else if index == strip.jobs.end { + Some(strip.stride) + } else { + plan.jobs + .get(index) + .map(|job| job.placement.target_x as usize) + } + }; + let (Some(start), Some(end)) = (column(jobs.start), column(jobs.end)) else { + return Err(DecodingError::CodeBlockDecodeFailure.into()); + }; + for row in lock()?.chunks_mut(strip.stride) { + row.get_mut(start..end) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + .fill(0.0); + } + } + if scratch.len() < scratch_samples { + try_resize_decode_elements(scratch, scratch_samples, 0.0)?; + } + for job in plan + .jobs + .get(jobs) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + { + // The strip starts at the block row's first stored row. + let placement = BlockPlacement { + target_y: 0, + ..job.placement + }; + let block = scratch + .get_mut(..job.width as usize * job.height as usize) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if !placement.is_whole(job.width, job.height) { + block.fill(0.0); + } + decode_block(job, block, job.width as usize, workspace)?; + placement.copy_from_block(block, job.width, &mut lock()?, strip.stride)?; + } + Ok(()) +} + fn decode_strip( plan: &TilePlan<'_>, strip: &Strip, @@ -1447,7 +1688,14 @@ impl BandRoute<'_> { tile_ctx: &mut TileDecodeContext, storage: &DecompositionStorage<'_>, ) -> Result<()> { - record_route_peak(tile_ctx, storage, &self.plan, self.workers + 1, 0)?; + // Band slots, not route tasks, hold this plan's outputs. + let counts = TaskCounts { + tasks: 0, + shared: 0, + scratch_samples: self.plan.max_clipped_samples, + part_work: None, + }; + record_route_peak(tile_ctx, storage, &self.plan, self.workers + 1, counts)?; count_decoded_blocks(tile_ctx, &self.plan, self.workers + 1, self.growths); Ok(()) } @@ -1455,7 +1703,96 @@ impl BandRoute<'_> { #[cfg(test)] mod tests { - use super::{route_fallback, CodeBlockCoding, TileRoute}; + use alloc::vec::Vec; + + use super::{route_fallback, split_runs, CodeBlockCoding, Strip, TaskCounts, TileRoute}; + + #[test] + fn split_runs_cover_the_strip_in_balanced_parts() { + let ramp: Vec = (1..=40).collect(); + for (works, parts) in [ + (&[5_usize; 32][..], 8), + (&[1, 1, 1, 100, 1, 1, 1][..], 3), + (&[10, 20, 30, 40][..], 8), + (&[7][..], 4), + (&[3, 0, 0, 3][..], 2), + (&[0, 0, 0][..], 2), + (&ramp[..], 5), + ] { + let total: usize = works.iter().sum(); + let largest = works.iter().copied().max().unwrap_or(0); + let mut runs = Vec::new(); + split_runs( + 10, + works.iter().map(|&work| Ok(work)), + total, + parts, + |jobs, work| { + runs.push((jobs, work)); + Ok(()) + }, + ) + .unwrap(); + assert!(runs.len() <= parts, "{works:?} into {parts}: {runs:?}"); + let mut next = 10; + for (jobs, work) in &runs { + assert_eq!(jobs.start, next, "{works:?} into {parts}: {runs:?}"); + assert!(jobs.end > jobs.start, "{works:?} into {parts}: {runs:?}"); + assert_eq!( + *work, + works[jobs.start - 10..jobs.end - 10].iter().sum::() + ); + // A run passes its share by less than its last job. + assert!( + *work <= total.div_ceil(parts) + largest, + "{works:?}: {runs:?}" + ); + next = jobs.end; + } + assert_eq!(next, 10 + works.len(), "{works:?} into {parts}: {runs:?}"); + } + + // Equal jobs split into equal runs. + let mut runs = Vec::new(); + split_runs( + 0, + [5_usize; 32].into_iter().map(Ok), + 160, + 8, + |jobs, work| { + runs.push((jobs, work)); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(runs.len(), 8); + assert!(runs + .iter() + .all(|(jobs, work)| jobs.len() == 4 && *work == 20)); + } + + #[test] + fn strips_split_into_at_most_one_part_per_job() { + let strip = |jobs, work| Strip { + start: 0, + len: 0, + stride: 0, + jobs, + work, + }; + let counts = |part_work| TaskCounts { + tasks: 0, + shared: 0, + scratch_samples: 0, + part_work, + }; + assert_eq!(counts(None).parts(&strip(0..32, 1_000_000)), 1); + assert_eq!(counts(Some(100)).parts(&strip(0..32, 1000)), 10); + assert_eq!(counts(Some(100)).parts(&strip(0..32, 1001)), 11); + assert_eq!(counts(Some(10)).parts(&strip(0..3, 1000)), 3); + assert_eq!(counts(Some(100)).parts(&strip(0..32, 0)), 1); + assert_eq!(counts(Some(100)).parts(&strip(5..5, 1000)), 1); + } #[test] fn tile_route() { From 4035a3d144f53453851cdf40b320f6a294be0091 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:56:58 -0400 Subject: [PATCH 21/53] perf(j2k-native): keep classic tiles with fewer rows than workers on the tile route A classic tile with fewer rows of code blocks than pool threads fell back to the per-sub-band path, which decodes the sub-bands of a small tile one after another. The tile route now keeps such tiles: the row splitting from the parent commit already balances their few long rows across workers. The band route keeps its per-sub-band fallback. Measured against the parent (built twice, as A and an A/A control) on an 18-core ARM64 machine, 5 interleaved rounds, median of per-round ratios: lossless classic 128^2 at 18 threads -51%, classic 2 bpp 192^2 at 18 threads -18%; every other row (classic and lossless 128^2-768^2 at 1/6/18 threads, HT 128^2-768^2, classic and HT mosaics, 8 concurrent region decodes, PLT and no-PLT regions) within the A/A spread. Output is byte-identical. The heap peak of lossless 128^2 at 18 threads rises from 0.31 to 1.12 MB, the same as the parent at 16 threads, where the tile already took the route. --- .../src/j2c/decode/subband/tile_parallel.rs | 64 ++++++++++++++----- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs index d91394e3d..0d3b8f90c 100644 --- a/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/tile_parallel.rs @@ -62,11 +62,11 @@ const MIN_PARALLEL_CODED_BYTES: usize = 0; /// average 2,000 to 15,000. pub(crate) const MIN_BAND_WORK: usize = 1024; -/// Where a tile of `coding` blocks with `strips` rows of code blocks and -/// `coded_bytes` of payload decodes on `threads` workers: on the route -/// (`None`), serially, or per sub-band. A classic tile with fewer rows than -/// workers leaves some idle, while the per-sub-band path splits each row's -/// blocks among them. +/// Where a component of `coding` blocks with `strips` rows of code blocks +/// and `coded_bytes` of payload decodes on the band route with `threads` +/// workers: on the route (`None`), serially, or per sub-band. A classic +/// component with fewer rows than workers leaves some idle there, while the +/// per-sub-band path splits each row's blocks among them. fn route_fallback( coding: CodeBlockCoding, strips: usize, @@ -86,6 +86,21 @@ fn route_fallback( } } +/// Where a tile of `coding` blocks with `strips` rows of code blocks and +/// `coded_bytes` of payload decodes: on the tile route (`None`) or serially. +/// Unlike the band route, a classic tile with fewer rows than workers stays on +/// the tile route, which splits its long rows into parts of about equal work; +/// the per-sub-band path would decode its sub-bands one after another. +fn tile_route_fallback( + coding: CodeBlockCoding, + strips: usize, + coded_bytes: usize, + min_coded_bytes: usize, +) -> Option { + (too_few_coded_bytes(coding, coded_bytes, min_coded_bytes) || strips < 2) + .then_some(TileRoute::Serial) +} + /// Whether `coded_bytes` of `coding` blocks are too few to fork for. fn too_few_coded_bytes( coding: CodeBlockCoding, @@ -208,13 +223,8 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( // serially before anything is planned. let (required, segment_bytes, rows) = required_code_blocks(tile, components.clone(), storage, header, coding)?; - if let Some(route) = route_fallback( - coding, - rows, - segment_bytes, - rayon::current_num_threads(), - MIN_PARALLEL_CODED_BYTES, - ) { + if let Some(route) = tile_route_fallback(coding, rows, segment_bytes, MIN_PARALLEL_CODED_BYTES) + { return Ok(route); } let mut budget = DecodeAllocationBudget::for_storage(storage)?; @@ -237,11 +247,10 @@ pub(in crate::j2c::decode) fn plan_tile_route<'a>( } Err(error) => return Err(error), }; - if let Some(route) = route_fallback( + if let Some(route) = tile_route_fallback( coding, plan.strips.len(), plan.coded_bytes, - rayon::current_num_threads(), MIN_PARALLEL_CODED_BYTES, ) { return Ok(route); @@ -1705,7 +1714,10 @@ impl BandRoute<'_> { mod tests { use alloc::vec::Vec; - use super::{route_fallback, split_runs, CodeBlockCoding, Strip, TaskCounts, TileRoute}; + use super::{ + route_fallback, split_runs, tile_route_fallback, CodeBlockCoding, Strip, TaskCounts, + TileRoute, + }; #[test] fn split_runs_cover_the_strip_in_balanced_parts() { @@ -1796,6 +1808,28 @@ mod tests { #[test] fn tile_route() { + let (ht, classic) = (CodeBlockCoding::HighThroughput, CodeBlockCoding::Classic); + // (coding, strips, coded bytes, expected fallback) with an 8 KiB cutoff: + // a classic tile with fewer rows than workers stays on the route. + for (coding, strips, coded_bytes, expected) in [ + (ht, 1, usize::MAX, Some(TileRoute::Serial)), + (ht, 16, 8191, Some(TileRoute::Serial)), + (ht, 2, 8192, None), + (classic, 16, 1023, Some(TileRoute::Serial)), + (classic, 16, 1024, None), + (classic, 3, 1024, None), + (classic, 1, usize::MAX, Some(TileRoute::Serial)), + ] { + assert_eq!( + tile_route_fallback(coding, strips, coded_bytes, 8192), + expected, + "{coding:?} {strips} strips, {coded_bytes} bytes" + ); + } + } + + #[test] + fn band_route() { let (ht, classic) = (CodeBlockCoding::HighThroughput, CodeBlockCoding::Classic); // (coding, strips, coded bytes, threads, expected fallback) with an 8 KiB cutoff. for (coding, strips, coded_bytes, threads, expected) in [ From 138665d7771d7e6126784e00626029ba474c5b37 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:42:08 -0400 Subject: [PATCH 22/53] perf(j2k-native): shorten the HT cleanup VLC decode chain The cleanup pass decodes a quad pair through three dependent table lookups: the two VLC codewords, then the U-VLC pair. Between them the codeword length, the context of the next quad and the U-VLC index were each rebuilt from the table symbol, and the reverse VLC reader refilled whenever fewer than 32 bits were left, a branch taken about every other pair that resolves only after the U-VLC lookup. - The VLC tables are relaid out at compile time from the existing tables. An entry holds the codeword length alone in its low six bits, so the reservoir shifts by the entry directly, the context bits it passes to the next quad in place, the U-VLC offset flag at the bit the U-VLC index needs, and the original symbol above them. The original tables, and the adapter API that exposes them, are unchanged. - The pair loops top the reservoir up without that branch: the eight bytes below the cursor are ORed in above the buffered bits and the whole bytes that fit are consumed. A stuffed byte among them, or fewer than eight left, takes the existing byte loop. Bits left above the count are the next byte's own bits, so ORing them in again is exact. A test reads random segments, stuffing-heavy and past their end, through both refills and compares every pair-sized step. On a 10-core ARM64 machine, interleaved against the base, the 4096x2048 RGB HT lossless decode is 6.1% faster at 1 thread, 7.5% at 4, 3.4% at 10 and 4.2% at the default; HT 2 bpp is 0.8 to 1.4% faster; 512x512 and 256x256 HT tiles 1.2% and 1.1% faster at 1 thread. Region decodes of an 18000x12000 HT image and 8 concurrent region decodes are unchanged within noise. Output is unchanged. --- .../src/j2c/ht_block_decode/cleanup.rs | 120 ++++++++++++------ .../src/j2c/ht_block_decode/readers.rs | 62 +++++++++ .../src/j2c/ht_block_decode/readers/tests.rs | 49 +++++++ 3 files changed, 195 insertions(+), 36 deletions(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs b/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs index ecbb58bbf..38e06ba2d 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs @@ -52,13 +52,59 @@ const fn uvlc_wide1() -> [u32; 4096] { table } -/// VLC table index for context `c_q` and the next VLC bits. `c_q` holds -/// only bits 7-9, so the mask never changes the index; it only drops the -/// bounds check on the 1024-entry tables. +/// VLC decoding of the initial quad row ([`VLC_TABLE0`]) and of later rows +/// ([`VLC_TABLE1`]), laid out for the serial decode chain. Each entry is +/// `length | context << 7 | u_offset << 10 | symbol << 16`: the codeword +/// length alone in the low six bits, so a 64-bit shift by the entry consumes +/// the codeword; the bits 7-9 the symbol contributes to the context of the +/// quad after it; the U-VLC offset flag; and the original table symbol. +const VLC_DECODE0: [u32; 1024] = vlc_decode_table(&VLC_TABLE0, true); +const VLC_DECODE1: [u32; 1024] = vlc_decode_table(&VLC_TABLE1, false); + +const fn vlc_decode_table(table: &[u16; 1024], initial_row: bool) -> [u32; 1024] { + let mut decode = [0; 1024]; + let mut index = 0; + while index < 1024 { + let symbol = table[index] as u32; + let context = if initial_row { + ((symbol & 0x10) << 3) | ((symbol & 0xE0) << 2) + } else { + ((symbol & 0x40) << 2) | ((symbol & 0x80) << 1) + }; + decode[index] = (symbol & 0x7) | context | ((symbol & 0x8) << 7) | (symbol << 16); + index += 1; + } + decode +} + +/// The [`VLC_DECODE0`]/[`VLC_DECODE1`] entry for context `c_q` and the next +/// VLC bits. `c_q` holds only bits 7-9, so the mask never changes the index; +/// it only drops the bounds check on the 1024-entry tables. #[expect(clippy::inline_always, reason = "per-quad hot path")] #[inline(always)] -fn vlc_index(c_q: u32, vlc_val: u32) -> usize { - ((c_q + (vlc_val & 0x7F)) & 0x3FF) as usize +fn vlc_entry(table: &[u32; 1024], c_q: u32, vlc_val: u32) -> u32 { + table[((c_q | (vlc_val & 0x7F)) & 0x3FF) as usize] +} + +/// The context bits a decoded VLC entry passes to the next quad. +#[expect(clippy::inline_always, reason = "per-quad hot path")] +#[inline(always)] +fn entry_context(entry: u32) -> u32 { + entry & 0x380 +} + +/// The original VLC table symbol of a decoded entry. +#[expect(clippy::inline_always, reason = "per-quad hot path")] +#[inline(always)] +fn entry_symbol(entry: u32) -> u32 { + entry >> 16 +} + +/// The U-VLC mode (`u_offset` flags in bits 6 and 7) of a quad pair. +#[expect(clippy::inline_always, reason = "per-pair hot path")] +#[inline(always)] +fn pair_uvlc_mode(entry0: u32, entry1: u32) -> u32 { + ((entry0 >> 4) & 0x40) | ((entry1 >> 3) & 0x80) } /// Decodes one initial-row quad pair's symbols as `[t0, u0, t1, u1]`. @@ -78,33 +124,34 @@ pub(super) fn decode_initial_pair_symbols( c_q: &mut u32, second_quad: bool, ) -> Option<[u32; 4]> { - let mut vlc_val = vlc.fetch_with::(); - let mut t0 = u32::from(VLC_TABLE0[vlc_index(*c_q, vlc_val)]); + let mut vlc_val = vlc.fetch_topped_up::(); + let mut e0 = vlc_entry(&VLC_DECODE0, *c_q, vlc_val); if *c_q == 0 { *run -= 2; - t0 = if *run == -1 { t0 } else { 0 }; + e0 = if *run == -1 { e0 } else { 0 }; if *run < 0 { *run = mel.get_run_with::()?; } } - *c_q = ((t0 & 0x10) << 3) | ((t0 & 0xE0) << 2); - vlc_val = vlc.advance(t0 & 0x7); + *c_q = entry_context(e0); + vlc_val = vlc.advance_entry(e0); - let mut t1 = u32::from(VLC_TABLE0[vlc_index(*c_q, vlc_val)]); + let mut e1 = vlc_entry(&VLC_DECODE0, *c_q, vlc_val); if *c_q == 0 && second_quad { *run -= 2; - t1 = if *run == -1 { t1 } else { 0 }; + e1 = if *run == -1 { e1 } else { 0 }; if *run < 0 { *run = mel.get_run_with::()?; } } if !second_quad { - t1 = 0; + e1 = 0; } - *c_q = ((t1 & 0x10) << 3) | ((t1 & 0xE0) << 2); - vlc_val = vlc.advance(t1 & 0x7); + *c_q = entry_context(e1); + vlc_val = vlc.advance_entry(e1); + let (t0, t1) = (entry_symbol(e0), entry_symbol(e1)); - let mut uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); + let mut uvlc_mode = pair_uvlc_mode(e0, e1); if uvlc_mode == 0xC0 { *run -= 2; if *run == -1 { @@ -212,48 +259,49 @@ pub(super) fn decode_pair_symbols( above: impl Fn(usize) -> u32, second_quad: bool, ) -> Option<[u32; 4]> { - *c_q |= (above(0) & 0xA0) << 2; - *c_q |= (above(2) & 0x20) << 4; - - let mut vlc_val = vlc.fetch_with::(); - let mut t0 = u32::from(VLC_TABLE1[vlc_index(*c_q, vlc_val)]); + // The row above's part of each quad's context, off the decode chain. + let above_first = ((above(0) & 0xA0) << 2) | ((above(2) & 0x20) << 4); + let above_second = (above(0) & 0x80) | ((above(2) & 0xA0) << 2) | ((above(4) & 0x20) << 4); + let above_next = above(2) & 0x80; + *c_q |= above_first; + + let mut vlc_val = vlc.fetch_topped_up::(); + let mut e0 = vlc_entry(&VLC_DECODE1, *c_q, vlc_val); if *c_q == 0 { *run -= 2; - t0 = if *run == -1 { t0 } else { 0 }; + e0 = if *run == -1 { e0 } else { 0 }; if *run < 0 { *run = mel.get_run_with::()?; } } - *c_q = ((t0 & 0x40) << 2) | ((t0 & 0x80) << 1); - *c_q |= above(0) & 0x80; - *c_q |= (above(2) & 0xA0) << 2; - *c_q |= (above(4) & 0x20) << 4; - vlc_val = vlc.advance(t0 & 0x7); + *c_q = entry_context(e0) | above_second; + vlc_val = vlc.advance_entry(e0); - let mut t1 = u32::from(VLC_TABLE1[vlc_index(*c_q, vlc_val)]); + let mut e1 = vlc_entry(&VLC_DECODE1, *c_q, vlc_val); if *c_q == 0 && second_quad { *run -= 2; - t1 = if *run == -1 { t1 } else { 0 }; + e1 = if *run == -1 { e1 } else { 0 }; if *run < 0 { *run = mel.get_run_with::()?; } } if !second_quad { - t1 = 0; + e1 = 0; } - *c_q = ((t1 & 0x40) << 2) | ((t1 & 0x80) << 1); - *c_q |= above(2) & 0x80; - vlc_val = vlc.advance(t1 & 0x7); + *c_q = entry_context(e1) | above_next; + vlc_val = vlc.advance_entry(e1); + let (t0, t1) = (entry_symbol(e0), entry_symbol(e1)); - // The mode is at most 0xC0, so the mask only drops the bounds check. - let uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); - let wide = UVLC_WIDE1[(((uvlc_mode << 4) | (vlc_val & 0x3FF)) & 0xFFF) as usize]; + // The `u_offset` flags land in index bits 10 and 11, so the index is + // below 4096 and needs no bounds check. + let wide = UVLC_WIDE1[((e0 & 0x400) | ((e1 & 0x400) << 1) | (vlc_val & 0x3FF)) as usize]; if wide & WIDE_UVLC_FALLBACK == 0 { let _ = vlc.advance(wide & 0xF); return Some([t0, (wide >> 8) & 0xFFF, t1, wide >> 20]); } + let uvlc_mode = pair_uvlc_mode(e0, e1); let mut uvlc_entry = u32::from(UVLC_TABLE1[((uvlc_mode + (vlc_val & 0x3F)) & 0xFF) as usize]); vlc_val = vlc.advance(uvlc_entry & 0x7); uvlc_entry >>= 3; diff --git a/crates/j2k-native/src/j2c/ht_block_decode/readers.rs b/crates/j2k-native/src/j2c/ht_block_decode/readers.rs index 422ad5506..e9eea3504 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/readers.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/readers.rs @@ -123,6 +123,10 @@ impl<'a> MelDecoder<'a> { } } +/// Bit 7 of every byte of a word, where the SWAR stuffing checks keep +/// their per-byte flags. +const BYTE_HIGH_BITS: u64 = 0x8080_8080_8080_8080; + /// `Clone` (not `Copy`) keeps the by-value refill hand-offs explicit. #[derive(Clone)] pub(super) struct ForwardBitReader<'a, const PAD: u8> { @@ -334,6 +338,48 @@ impl<'a> ReverseBitReader<'a> { self.fetch_with::() } + /// [`Self::fetch_with`] for the per-pair loops, without its data-dependent + /// refill branch: unless a byte below the cursor is stuffed or fewer than + /// eight remain, OR the eight bytes below the cursor in above the + /// reservoir's bits and consume the whole bytes that fit, leaving 56 to 63 + /// bits. Bits above the count then hold the next byte's leading bits, the + /// same bits the next top-up ORs in again. + #[expect( + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_sign_loss, + clippy::inline_always, + reason = "low reservoir word; nonnegative live cursor; per-pair hot path" + )] + #[inline(always)] + pub(super) fn fetch_topped_up(&mut self) -> u32 { + if self.remaining >= 8 && self.pos >= 7 { + let last = self.pos as usize; + if let Some(&bytes) = self.data.get(last - 7..=last).and_then(<[u8]>::first_chunk) { + // `data[last]` is consumed first, so it lands in the low byte. + // The per-byte flags are as in `fill`. + let word = u64::from_be_bytes(bytes); + let above_8f = word + & BYTE_HIGH_BITS + & ((word & 0x7070_7070_7070_7070) + 0x7070_7070_7070_7070); + let low_ones = + ((word & 0x7F7F_7F7F_7F7F_7F7F) + 0x0101_0101_0101_0101) & BYTE_HIGH_BITS; + let after_above_8f = (above_8f << 8) | (u64::from(self.unstuff) << 7); + if after_above_8f & low_ones == 0 { + let bytes = (63 - self.bits) >> 3; + self.tmp |= word << self.bits; + self.bits += 8 * bytes; + self.pos -= bytes as isize; + self.remaining -= bytes as usize; + // Whether the last byte consumed is above 0x8F. + self.unstuff = (after_above_8f >> (8 * bytes + 7)) & 1 != 0; + return self.tmp as u32; + } + } + } + self.fetch_with::() + } + /// [`Self::fetch`]. `INLINE_REFILL` inlines the byte refill too, so a /// loop around it makes no call and keeps the reader in registers. #[expect( @@ -362,6 +408,22 @@ impl<'a> ReverseBitReader<'a> { self.bits -= count; self.tmp as u32 } + + /// [`Self::advance`] past a VLC codeword whose decode-table `entry` holds + /// its length in bits 0-2 and zeros in bits 3-5, so the reservoir shift + /// takes the entry's low six bits directly. + #[expect( + clippy::cast_possible_truncation, + clippy::inline_always, + reason = "low reservoir word; per-quad hot path" + )] + #[inline(always)] + pub(super) fn advance_entry(&mut self, entry: u32) -> u32 { + debug_assert!(entry & 0x38 == 0 && entry & 0x7 <= self.bits); + self.tmp >>= entry & 0x3F; + self.bits -= entry & 0x7; + self.tmp as u32 + } } /// Padding after the unstuffed `MagSgn` bytes. It is at least one fetch window, diff --git a/crates/j2k-native/src/j2c/ht_block_decode/readers/tests.rs b/crates/j2k-native/src/j2c/ht_block_decode/readers/tests.rs index dd93fc8ca..b73ebd745 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/readers/tests.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/readers/tests.rs @@ -199,3 +199,52 @@ fn flat_magsgn_unstuffing_requires_reserved_capacity() { "long segments must not allocate" ); } + +/// The branch-free top-up reads the same bits as the reservoir reader's +/// refills, stuffing and the zeros past the segment included, for every +/// pair-sized step pattern. +#[test] +fn vlc_top_ups_match_reservoir_refills() { + let mut seed = 0x9E37_79B9_u32; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed + }; + // Bytes around the stuffing thresholds are over-represented. + let special = [0xFF_u8, 0x7F, 0x8F, 0x90, 0x00, 0xFE, 0xF7, 0x0F]; + for lcup in 2..=48_usize { + for _ in 0..40 { + let data = (0..lcup) + .map(|_| { + let draw = next(); + if draw % 3 == 0 { + special[(draw >> 8) as usize % special.len()] + } else { + (draw >> 16).to_le_bytes()[0] + } + }) + .collect::>(); + let scup = 2 + next() as usize % (lcup - 1); + let mut reference = ReverseBitReader::new_vlc(&data, lcup, scup); + let mut topped_up = reference.clone(); + // Pairs consume at most 30 bits between top-ups. + for _ in 0..40 { + let first = reference.fetch(); + assert_eq!(topped_up.fetch_topped_up::(), first, "{data:02X?}"); + let mut budget = 30; + while budget > 0 { + let count = (next() % 9).min(budget); + budget -= count; + let consumed = 30 - budget; + let expected = reference.advance(count); + // Only the 32 - `consumed` bits the reference still + // guarantees are compared; the top-up may hold more. + let mask = u32::try_from((1_u64 << (32 - consumed)) - 1).expect("mask"); + assert_eq!(topped_up.advance(count) & mask, expected & mask); + } + } + } + } +} From 7f9004a3ab76d0285495d800497984c168f42140 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 00:42:26 -0400 Subject: [PATCH 23/53] perf(j2k-native): read HT VLC bits from a flat unstuffed buffer The direct-output cleanup path, which decodes every cleanup-only block straight to dequantized samples, still read the VLC bits through the reverse reservoir reader. Its top-up checks the eight bytes below the cursor for stuffing on every quad pair, about a seventh of the pair loop's instructions. - The VLC bits are now unstuffed once per block, eight bytes at a time when none is stuffed, into a forward buffer in the order the reverse reader consumes them, then zero padded as that reader pads. It keeps the bytes a block can consume (at most 30 bits per quad pair), so the buffer is sized from the block's geometry: 1937 bytes for 64x64. - `FlatVlcReader` tops its reservoir up from that buffer with no checks and no branch, and hands each pair's first lookup the word from before the top-up: it already holds the seven bits that lookup needs, since a pair consumes at most 30 of the 56 a top-up leaves. - The pair decoders take either reader through `VlcBits`; the other HT paths keep the reverse reader. Tests compare the flat reader with the reverse reader on random stuffing-heavy segments and check the unstuffing limits. On a 10-core ARM64 machine, interleaved against the base and including the previous commit, the 4096x2048 RGB HT lossless decode is 9.8% faster at 1 thread, 7.1% at 4, 6.6% at 10 and 5.6% at the default; HT 2 bpp is 3.6% faster at 1 thread, 2.8% at 4, 1.8% at 10 and 2.6% at the default; 512x512 and 256x256 HT tiles 2.1% and 1.9% faster at 1 thread. 1024x1024 regions of an 18000x12000 HT image are 1.9 to 2.1% faster at 1 thread and 0.4 to 0.5% at 10; 8 concurrent region decodes are unchanged within noise. The HT decode workspace grows by the flat buffer. Output is unchanged. --- .../src/j2c/ht_block_decode/cleanup.rs | 18 +- .../src/j2c/ht_block_decode/magnitude.rs | 5 + .../src/j2c/ht_block_decode/magnitude/simd.rs | 22 +- .../src/j2c/ht_block_decode/pipeline.rs | 1 + .../j2c/ht_block_decode/pipeline/scratch.rs | 8 +- .../src/j2c/ht_block_decode/readers.rs | 220 ++++++++++++++++++ .../src/j2c/ht_block_decode/readers/tests.rs | 82 ++++++- .../src/j2c/ht_block_decode/state.rs | 7 + 8 files changed, 341 insertions(+), 22 deletions(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs b/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs index 38e06ba2d..e7a5c4079 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::super::ht_tables::{UVLC_TABLE0, UVLC_TABLE1, VLC_TABLE0, VLC_TABLE1}; -use super::readers::{MelDecoder, ReverseBitReader}; +use super::readers::{MelDecoder, ReverseBitReader, VlcBits}; pub(super) fn cleanup_symbol_stride(width: u32) -> usize { ((width + 2 + 7) & !7) as usize @@ -117,14 +117,14 @@ fn pair_uvlc_mode(entry0: u32, entry1: u32) -> u32 { reason = "VLC table fields are defined as 16-bit values and this per-pair decoder is a hot path" )] #[inline(always)] -pub(super) fn decode_initial_pair_symbols( +pub(super) fn decode_initial_pair_symbols( mel: &mut MelDecoder, - vlc: &mut ReverseBitReader, + vlc: &mut V, run: &mut i32, c_q: &mut u32, second_quad: bool, ) -> Option<[u32; 4]> { - let mut vlc_val = vlc.fetch_topped_up::(); + let mut vlc_val = vlc.top_up::(); let mut e0 = vlc_entry(&VLC_DECODE0, *c_q, vlc_val); if *c_q == 0 { *run -= 2; @@ -211,7 +211,7 @@ pub(super) fn decode_cleanup_first_row_pair( x: u32, width: u32, ) -> Option<[u32; 4]> { - let symbols = decode_initial_pair_symbols::(mel, vlc, run, c_q, x + 2 < width)?; + let symbols = decode_initial_pair_symbols::(mel, vlc, run, c_q, x + 2 < width)?; store_pair_symbols(scratch, row_offset, symbols); Some(symbols) } @@ -251,9 +251,9 @@ pub(super) fn decode_cleanup_symbols_first_row( reason = "VLC table fields are defined as 16-bit values and this per-pair decoder is a hot path" )] #[inline(always)] -pub(super) fn decode_pair_symbols( +pub(super) fn decode_pair_symbols( mel: &mut MelDecoder, - vlc: &mut ReverseBitReader, + vlc: &mut V, run: &mut i32, c_q: &mut u32, above: impl Fn(usize) -> u32, @@ -265,7 +265,7 @@ pub(super) fn decode_pair_symbols( let above_next = above(2) & 0x80; *c_q |= above_first; - let mut vlc_val = vlc.fetch_topped_up::(); + let mut vlc_val = vlc.top_up::(); let mut e0 = vlc_entry(&VLC_DECODE1, *c_q, vlc_val); if *c_q == 0 { *run -= 2; @@ -341,7 +341,7 @@ pub(super) fn decode_cleanup_row_pair( // `sstr >= 8`, so the row above never overlaps this pair's symbols. let prev = row_offset - sstr; let above = |k| u32::from(scratch[prev + k]); - let symbols = decode_pair_symbols::(mel, vlc, run, c_q, above, x + 2 < width)?; + let symbols = decode_pair_symbols::(mel, vlc, run, c_q, above, x + 2 < width)?; store_pair_symbols(scratch, row_offset, symbols); Some(symbols) } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs index dc402acf5..26e11d86b 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs @@ -3,6 +3,8 @@ use alloc::vec::Vec; use super::cleanup::{decode_cleanup_symbols_first_row, decode_cleanup_symbols_row}; +#[cfg(feature = "simd")] +use super::readers::unstuff_vlc; use super::readers::{unstuff_magsgn, FlatBitReader, MelDecoder, ReverseBitReader}; use super::stats::HtDecodeObserver; #[cfg(feature = "simd")] @@ -449,6 +451,7 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32, + vlc_buffer: &mut Vec, ) -> Option<()> { if !supports_cleanup_to_f32(missing_msbs, width) { return None; @@ -467,6 +470,7 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32( fn decode_first_row_to_f32( lanes: &Lanes, mel: &mut MelDecoder<'_>, - vlc: &mut ReverseBitReader<'_>, + vlc: &mut FlatVlcReader<'_>, run: &mut i32, magsgn: &mut FlatBitReader<'_>, row: &mut RowOutput<'_>, @@ -674,7 +674,7 @@ fn decode_first_row_to_f32( *run -= 4; [0, 1, 0, 1] } else { - decode_initial_pair_symbols::(mel, vlc, run, &mut c_q, true)? + decode_initial_pair_symbols::(mel, vlc, run, &mut c_q, true)? }; *row.symbols.get_mut(index)? = symbols.map(|symbol| symbol as u16); let [t0, u0, t1, u1] = symbols; @@ -706,7 +706,7 @@ fn decode_first_row_to_f32( fn decode_row_to_f32( lanes: &Lanes, mel: &mut MelDecoder<'_>, - vlc: &mut ReverseBitReader<'_>, + vlc: &mut FlatVlcReader<'_>, run: &mut i32, magsgn: &mut FlatBitReader<'_>, above: &[[u16; 4]], @@ -729,7 +729,7 @@ fn decode_row_to_f32( *run -= 4; [0; 4] } else { - decode_pair_symbols::(mel, vlc, run, &mut c_q, above, true)? + decode_pair_symbols::(mel, vlc, run, &mut c_q, above, true)? }; *row.symbols.get_mut(index)? = symbols.map(|symbol| symbol as u16); // As `with_kappa`: the row above's `v_n`, read before this pair's. @@ -772,6 +772,7 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32_rows< lcup: usize, scup: usize, flat: &[u8], + flat_vlc: &[u8], scratch: &mut [u16], decoded_data: &mut [f32], v_n_scratch: &mut [u32], @@ -783,7 +784,7 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32_rows< output: FinalOutput, ) -> Option<()> { let mut mel = MelDecoder::new(coded_data, lcup, scup); - let mut vlc = ReverseBitReader::new_vlc(coded_data, lcup, scup); + let mut vlc = FlatVlcReader::new(flat_vlc)?; let mut magsgn = FlatBitReader::new(flat)?; let mut run = mel.get_run_with::()?; let lanes = Lanes::new(simd); @@ -859,7 +860,8 @@ mod tests { cleanup_segment_suffix_length, cleanup_symbol_stride, decode_cleanup_symbols, }; use super::super::super::readers::{ - flat_magsgn_capacity, unstuff_magsgn, FlatBitReader, MelDecoder, ReverseBitReader, + flat_magsgn_capacity, flat_vlc_capacity, unstuff_magsgn, unstuff_vlc, FlatBitReader, + MelDecoder, ReverseBitReader, }; use super::super::super::stats::NoHtDecodeStats; use super::super::super::store_coefficient_row; @@ -1036,6 +1038,8 @@ mod tests { let mut buffer = Vec::with_capacity(flat_magsgn_capacity(samples).expect("MagSgn capacity")); let flat = unstuff_magsgn(&coded[..lcup - scup], &mut buffer, samples * 4)?; + let mut vlc_buffer = Vec::with_capacity(flat_vlc_capacity(width, height)); + let flat_vlc = unstuff_vlc(coded, lcup, scup, &mut vlc_buffer)?; let mut scratch = vec![0xA5A5; sstr * (height.div_ceil(2) as usize + 1)]; let mut v_n = vec![0xDEAD_BEEF; width.div_ceil(2) as usize + 2]; let mut output = guarded_output(width, height, stride); @@ -1048,12 +1052,12 @@ mod tests { }; if irreversible { dispatch!(level, simd => decode_cleanup_and_magnitude_sign_to_f32_rows::( - simd, coded, lcup, scup, flat, &mut scratch, rows, &mut v_n, missing_msbs, width, + simd, coded, lcup, scup, flat, flat_vlc, &mut scratch, rows, &mut v_n, missing_msbs, width, height, stride, sstr, conversion, ))?; } else { dispatch!(level, simd => decode_cleanup_and_magnitude_sign_to_f32_rows::( - simd, coded, lcup, scup, flat, &mut scratch, rows, &mut v_n, missing_msbs, width, + simd, coded, lcup, scup, flat, flat_vlc, &mut scratch, rows, &mut v_n, missing_msbs, width, height, stride, sstr, conversion, ))?; } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs index 6fb88aad3..53d6788f3 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs @@ -212,5 +212,6 @@ pub(super) fn decode_cleanup_to_f32_impl( scratch, v_n_scratch, &mut scratch_buffers.magsgn, + &mut scratch_buffers.vlc, ) } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/pipeline/scratch.rs b/crates/j2k-native/src/j2c/ht_block_decode/pipeline/scratch.rs index 129182296..4cfff890d 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/pipeline/scratch.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/pipeline/scratch.rs @@ -3,7 +3,7 @@ //! Fallible HT phase scratch reservation. use super::super::cleanup::cleanup_symbol_stride; -use super::super::readers::flat_magsgn_capacity; +use super::super::readers::{flat_magsgn_capacity, flat_vlc_capacity}; use super::super::significance::sigma_stride; use super::super::state::HtBlockDecodeScratch; use crate::error::{Result, ValidationError}; @@ -18,6 +18,7 @@ struct HtWorkspaceLayout { sigma_symbols: usize, previous_row_symbols: usize, magsgn_bytes: usize, + vlc_bytes: usize, } fn workspace_layout(width: u32, height: u32) -> Result { @@ -49,6 +50,7 @@ fn workspace_layout(width: u32, height: u32) -> Result { sigma_symbols: sigma_len, previous_row_symbols: previous_row_len, magsgn_bytes: flat_magsgn_capacity(coefficient_len)?, + vlc_bytes: flat_vlc_capacity(width, height), }) } @@ -61,6 +63,7 @@ pub(crate) fn ht_decode_workspace_bytes(width: u32, height: u32) -> Result(&mut bytes, layout.sigma_symbols)?; include_elements::(&mut bytes, layout.previous_row_symbols)?; include_elements::(&mut bytes, layout.magsgn_bytes)?; + include_elements::(&mut bytes, layout.vlc_bytes)?; Ok(bytes) } @@ -88,5 +91,6 @@ pub(crate) fn prepare_scratch( try_reserve_decode_elements(&mut scratch.v_n, layout.v_n_words)?; try_reserve_decode_elements(&mut scratch.sigma, layout.sigma_symbols)?; try_reserve_decode_elements(&mut scratch.prev_row_sig, layout.previous_row_symbols)?; - try_reserve_decode_elements(&mut scratch.magsgn, layout.magsgn_bytes) + try_reserve_decode_elements(&mut scratch.magsgn, layout.magsgn_bytes)?; + try_reserve_decode_elements(&mut scratch.vlc, layout.vlc_bytes) } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/readers.rs b/crates/j2k-native/src/j2c/ht_block_decode/readers.rs index e9eea3504..9cc51fd39 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/readers.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/readers.rs @@ -588,3 +588,223 @@ pub(super) fn read_u32_pair(values: &[u16], index: usize) -> u32 { #[cfg(test)] mod tests; + +/// Largest cleanup suffix (MEL + VLC) in bytes, Clause 7.1.4. +pub(super) const MAX_CLEANUP_SUFFIX: usize = 4079; + +/// Padding after the unstuffed VLC bytes: one top-up window past the data, +/// and the window a clamped top-up reads, both all zeros. +const FLAT_VLC_PADDING: usize = 16; + +/// Bytes of VLC data a `width` x `height` block can consume: a quad pair takes +/// at most 30 bits (two seven-bit codewords and a 16-bit U-VLC pair), and the +/// pairs of a quad row cover four columns. +pub(super) fn flat_vlc_consumable(width: u32, height: u32) -> usize { + let pairs = (width.div_ceil(4) as usize).saturating_mul(height.div_ceil(2) as usize); + pairs + .saturating_mul(30) + .div_ceil(8) + .saturating_add(1) + .min(MAX_CLEANUP_SUFFIX) +} + +/// Capacity of the flat VLC buffer of a `width` x `height` block. +pub(super) fn flat_vlc_capacity(width: u32, height: u32) -> usize { + flat_vlc_consumable(width, height) + FLAT_VLC_PADDING +} + +/// The VLC bits of a cleanup segment, in the order [`ReverseBitReader`] +/// consumes them, as a forward byte buffer read least significant bit first +/// and padded with zeros: the bits of [`ReverseBitReader::new_vlc`] and its +/// byte loop (a byte after one above 0x8F whose low seven bits are all ones +/// carries seven bits), then the zeros it reads past the segment. `out` is +/// reserved for its block ([`flat_vlc_capacity`]); a block cannot read more +/// than the bytes before the padding, so a longer segment stops there. +#[cfg_attr( + not(any(test, feature = "simd")), + expect(dead_code, reason = "only SIMD HT decoding reads flat VLC bits") +)] +#[expect( + clippy::cast_possible_truncation, + reason = "the low byte of the bit accumulator is emitted on purpose" +)] +pub(super) fn unstuff_vlc<'a>( + data: &[u8], + lcup: usize, + scup: usize, + out: &'a mut alloc::vec::Vec, +) -> Option<&'a [u8]> { + let consumable = out.capacity().checked_sub(FLAT_VLC_PADDING)?; + out.clear(); + let first = *data.get(lcup.checked_sub(2)?)?; + let mut acc = u64::from(first >> 4); + let mut bits = 4 - u32::from((acc & 0x7) == 0x7); + let mut unstuff = first > 0x8F; + let bytes = data.get(lcup.checked_sub(scup)?..lcup - 2)?; + let mut index = bytes.len(); + while index > 0 && out.len() < consumable { + if let Some(&chunk) = index + .checked_sub(8) + .and_then(|start| bytes.get(start..index)) + .and_then(<[u8]>::first_chunk::<8>) + { + // Eight bytes with no stuffed one, the last in the low byte, as + // in `ReverseBitReader::fill`: all 64 bits are data, appended + // after the `bits` pending bits. + let word = u64::from_be_bytes(chunk); + let above_8f = + word & BYTE_HIGH_BITS & ((word & 0x7070_7070_7070_7070) + 0x7070_7070_7070_7070); + let low_ones = + ((word & 0x7F7F_7F7F_7F7F_7F7F) + 0x0101_0101_0101_0101) & BYTE_HIGH_BITS; + let after_above_8f = (above_8f << 8) | (u64::from(unstuff) << 7); + if after_above_8f & low_ones == 0 { + out.extend_from_slice(&(acc | (word << bits)).to_le_bytes()); + acc = if bits == 0 { 0 } else { word >> (64 - bits) }; + unstuff = above_8f >> 63 != 0; + index -= 8; + continue; + } + } + index -= 1; + let byte = bytes[index]; + let stuffed = unstuff && (byte & 0x7F) == 0x7F; + let value = if stuffed { byte & 0x7F } else { byte }; + acc |= u64::from(value) << bits; + bits += 8 - u32::from(stuffed); + unstuff = byte > 0x8F; + if bits >= 8 { + out.push(acc as u8); + acc >>= 8; + bits -= 8; + } + } + if bits > 0 { + out.push(acc as u8); + } + out.truncate(consumable); + out.resize(out.len() + FLAT_VLC_PADDING, 0); + Some(out.as_slice()) +} + +/// VLC bits read from an [`unstuff_vlc`] buffer through a 64-bit reservoir. +/// Every top-up is branch-free: it ORs the eight bytes at the cursor in +/// above the buffered bits and consumes the whole bytes that fit. Bits above +/// the count hold the next byte's leading bits, the same bits the next +/// top-up ORs in again. +#[cfg_attr( + not(any(test, feature = "simd")), + expect(dead_code, reason = "only SIMD HT decoding reads flat VLC bits") +)] +pub(super) struct FlatVlcReader<'a> { + data: &'a [u8], + /// The last eight-byte window inside the buffer; every window from there + /// on reads only padding. + last_window: usize, + pos: usize, + tmp: u64, + bits: u32, +} + +#[cfg_attr( + not(any(test, feature = "simd")), + expect(dead_code, reason = "only SIMD HT decoding reads flat VLC bits") +)] +impl<'a> FlatVlcReader<'a> { + pub(super) fn new(data: &'a [u8]) -> Option { + let mut reader = Self { + last_window: data.len().checked_sub(8)?, + data, + pos: 0, + tmp: 0, + bits: 0, + }; + // Start with 56 bits buffered, as every later top-up leaves them. + let _ = reader.top_up::(); + Some(reader) + } +} + +/// The VLC bit source of the cleanup pair decoders. +pub(super) trait VlcBits { + /// Top the reservoir up to at least 33 bits and return a word whose low + /// seven bits are the next ones; read later bits through the advances. + fn top_up(&mut self) -> u32; + /// Consume `count` bits and return the new low reservoir word. + fn advance(&mut self, count: u32) -> u32; + /// Consume the codeword of a VLC decode-table `entry`, whose length is + /// in bits 0-2 with zeros in bits 3-5. + fn advance_entry(&mut self, entry: u32) -> u32; +} + +impl VlcBits for ReverseBitReader<'_> { + #[expect(clippy::inline_always, reason = "per-pair hot path")] + #[inline(always)] + fn top_up(&mut self) -> u32 { + self.fetch_topped_up::() + } + + #[expect(clippy::inline_always, reason = "per-quad hot path")] + #[inline(always)] + fn advance(&mut self, count: u32) -> u32 { + ReverseBitReader::advance(self, count) + } + + #[expect(clippy::inline_always, reason = "per-quad hot path")] + #[inline(always)] + fn advance_entry(&mut self, entry: u32) -> u32 { + ReverseBitReader::advance_entry(self, entry) + } +} + +impl VlcBits for FlatVlcReader<'_> { + #[expect( + clippy::cast_possible_truncation, + clippy::inline_always, + reason = "low reservoir word; per-pair hot path" + )] + #[inline(always)] + fn top_up(&mut self) -> u32 { + // Past the data every window is padding, so clamping it is exact. + let start = self.pos.min(self.last_window); + let word = self + .data + .get(start..) + .and_then(<[u8]>::first_chunk::<8>) + .map_or(0, |bytes| u64::from_le_bytes(*bytes)); + // The word before the top-up: a pair consumes at most 30 bits, so + // 26 or more of the 56 a top-up leaves are still buffered, more than + // the seven bits of a pair's first lookup, which then does not wait + // for the top-up. + let peek = self.tmp as u32; + self.tmp |= word << self.bits; + self.pos += ((63 - self.bits) >> 3) as usize; + self.bits |= 0x38; + peek + } + + #[expect( + clippy::cast_possible_truncation, + clippy::inline_always, + reason = "low reservoir word; per-quad hot path" + )] + #[inline(always)] + fn advance(&mut self, count: u32) -> u32 { + debug_assert!(count <= self.bits); + self.tmp >>= count; + self.bits -= count; + self.tmp as u32 + } + + #[expect( + clippy::cast_possible_truncation, + clippy::inline_always, + reason = "low reservoir word; per-quad hot path" + )] + #[inline(always)] + fn advance_entry(&mut self, entry: u32) -> u32 { + debug_assert!(entry & 0x38 == 0 && entry & 0x7 <= self.bits); + self.tmp >>= entry & 0x3F; + self.bits -= entry & 0x7; + self.tmp as u32 + } +} diff --git a/crates/j2k-native/src/j2c/ht_block_decode/readers/tests.rs b/crates/j2k-native/src/j2c/ht_block_decode/readers/tests.rs index b73ebd745..bb43b58f5 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/readers/tests.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/readers/tests.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - flat_magsgn_capacity, unstuff_magsgn, FlatBitReader, ForwardBitReader, MelDecoder, - ReverseBitReader, + flat_magsgn_capacity, flat_vlc_capacity, unstuff_magsgn, unstuff_vlc, FlatBitReader, + FlatVlcReader, ForwardBitReader, MelDecoder, ReverseBitReader, VlcBits, MAX_CLEANUP_SUFFIX, }; #[test] @@ -200,6 +200,84 @@ fn flat_magsgn_unstuffing_requires_reserved_capacity() { ); } +/// The flat VLC buffer and its reader read the same bits as the reservoir +/// reader's refills, stuffing and the zeros past the segment included, for +/// every pair-sized step pattern, as long as no more than the consumable +/// bytes are read. +#[test] +fn flat_vlc_matches_reservoir_refills() { + let mut seed = 0x9E37_79B9_u32; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed + }; + // Bytes around the stuffing thresholds are over-represented. + let special = [0xFF_u8, 0x7F, 0x8F, 0x90, 0x00, 0xFE, 0xF7, 0x0F]; + // 40 pairs of at most 30 bits: 150 bytes. + let (width, height) = (16, 20); + let mut buffer = Vec::with_capacity(flat_vlc_capacity(width, height)); + for lcup in 2..=48_usize { + for _ in 0..40 { + let data = (0..lcup) + .map(|_| { + let draw = next(); + if draw % 3 == 0 { + special[(draw >> 8) as usize % special.len()] + } else { + (draw >> 16).to_le_bytes()[0] + } + }) + .collect::>(); + let scup = 2 + next() as usize % (lcup - 1); + let mut reference = ReverseBitReader::new_vlc(&data, lcup, scup); + let flat = unstuff_vlc(&data, lcup, scup, &mut buffer).expect("flat VLC"); + let mut flat = FlatVlcReader::new(flat).expect("flat VLC reader"); + for _ in 0..40 { + let first = reference.fetch(); + let peek = flat.top_up::(); + assert_eq!( + peek & 0x7F, + first & 0x7F, + "lcup {lcup} scup {scup} {data:02X?}" + ); + let mut budget = 30; + while budget > 0 { + let count = (next() % 9).min(budget); + budget -= count; + let consumed = 30 - budget; + let expected = reference.advance(count); + // Only the bits both readers guarantee are compared: the + // reference keeps at least 32 after a fetch, the flat + // reader 26 after a pair. + let valid = 26_u32.saturating_sub(consumed).max(2); + let mask = u32::try_from((1_u64 << valid) - 1).expect("mask"); + let word = VlcBits::advance(&mut flat, count); + assert_eq!(word & mask, expected & mask, "lcup {lcup} scup {scup}"); + } + } + } + } +} + +#[test] +fn flat_vlc_unstuffing_keeps_what_the_buffer_holds_and_needs_a_valid_suffix() { + let data = [0x12, 0x34, 0x56, 0x78, 0x9A]; + let mut short = Vec::with_capacity(15); + assert!(unstuff_vlc(&data, data.len(), 3, &mut short).is_none()); + let mut buffer = Vec::with_capacity(flat_vlc_capacity(4, 4)); + assert!(unstuff_vlc(&data, data.len(), data.len() + 1, &mut buffer).is_none()); + let flat = unstuff_vlc(&data, data.len(), 3, &mut buffer).expect("flat VLC"); + assert_eq!(flat.len(), 2 + 16); + // A segment longer than the buffer keeps only the bytes a block can read. + let mut tight = Vec::with_capacity(17); + let flat = unstuff_vlc(&data, data.len(), 3, &mut tight).expect("flat VLC"); + assert_eq!(flat.len(), 1 + 16); + assert_eq!(super::flat_vlc_consumable(64, 64), 1921); + assert_eq!(super::flat_vlc_consumable(4096, 4096), MAX_CLEANUP_SUFFIX); +} + /// The branch-free top-up reads the same bits as the reservoir reader's /// refills, stuffing and the zeros past the segment included, for every /// pair-sized step pattern. diff --git a/crates/j2k-native/src/j2c/ht_block_decode/state.rs b/crates/j2k-native/src/j2c/ht_block_decode/state.rs index c29fe8e24..0b984ae43 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/state.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/state.rs @@ -183,6 +183,8 @@ pub(crate) struct HtBlockDecodeScratch { pub(super) prev_row_sig: Vec, /// The `MagSgn` segment unstuffed into a flat, padded bit buffer. pub(super) magsgn: Vec, + /// The VLC bits of the cleanup segment unstuffed into a flat buffer. + pub(super) vlc: Vec, } impl HtBlockDecodeScratch { @@ -193,6 +195,7 @@ impl HtBlockDecodeScratch { sigma: Vec::new(), prev_row_sig: Vec::new(), magsgn: Vec::new(), + vlc: Vec::new(), } } @@ -207,6 +210,7 @@ impl HtBlockDecodeScratch { include_capacity::(&mut bytes, self.sigma.capacity())?; include_capacity::(&mut bytes, self.prev_row_sig.capacity())?; include_capacity::(&mut bytes, self.magsgn.capacity())?; + include_capacity::(&mut bytes, self.vlc.capacity())?; Ok(bytes) } } @@ -229,6 +233,7 @@ pub(super) struct HtBlockDecodeScratchCapacities { pub(super) sigma: usize, pub(super) prev_row_sig: usize, pub(super) magsgn: usize, + pub(super) vlc: usize, } #[cfg(test)] @@ -240,6 +245,7 @@ impl HtBlockDecodeScratch { sigma: self.sigma.capacity(), prev_row_sig: self.prev_row_sig.capacity(), magsgn: self.magsgn.capacity(), + vlc: self.vlc.capacity(), } } @@ -249,5 +255,6 @@ impl HtBlockDecodeScratch { self.sigma.fill(u16::MAX); self.prev_row_sig.fill(u16::MAX); self.magsgn.fill(u8::MAX); + self.vlc.fill(u8::MAX); } } From 188609672f2ff84d044f72478cef9fba42d56d5a Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 01:04:46 -0400 Subject: [PATCH 24/53] perf(j2k-native): fold HT pair v_n in vector registers The direct-output cleanup path folds each quad pair's bottom-row `v_n` into the next row's context. It extracted the four lanes to scalars, combined them with the previous pair's last lane, and stored two words plus a carry, about ten instructions per pair in a loop that is close to its instruction-throughput limit. The pair decoder now returns the bottom row as a vector. The fold slides the previous pair's bottom row in by one lane, ORs, and stores lanes 0 and 2 as one 64-bit word; the carry stays a vector until the end of the row. On a 10-core ARM64 machine, interleaved against the base and including the earlier commits, the 4096x2048 RGB HT lossless decode is 10.6% faster at 1 thread, 8.1% at 4, 6.1% at 10 and 6.4% at the default; HT 2 bpp is 3.7% faster at 1 thread, 3.2% at 4, 2.6% at 10 and 3.1% at the default; 512x512 and 256x256 HT tiles 2.7% and 2.3% faster at 1 thread. 1024x1024 regions of an 18000x12000 HT image are 2.2 to 2.6% faster at 1 thread and 0.1 to 0.3% at 10; 8 concurrent region decodes are unchanged within noise. Output is unchanged. --- .../src/j2c/ht_block_decode/magnitude/simd.rs | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs index f0663282a..e709186a7 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs @@ -8,7 +8,9 @@ //! instead of one after another. Each lane computes exactly what //! [`super::decode_mag_sgn_sample_with_vn`] computes for that sample. -use fearless_simd::{f32x4, f32x8, i32x8, u32x4, u32x8, u8x16, u8x32, Bytes, Simd, SimdBase}; +use fearless_simd::{ + f32x4, f32x8, i32x8, u32x4, u32x8, u64x2, u8x16, u8x32, Bytes, Simd, SimdBase, +}; use super::super::cleanup::{ decode_cleanup_first_row_pair, decode_cleanup_row_pair, decode_cleanup_symbols_first_row, @@ -207,7 +209,7 @@ fn decode_two_quads_to_f32( quads: [(u32, u32); 2], missing_msbs: u32, output: FinalOutput, -) -> Option<(f32x4, f32x4, [u32; 4])> { +) -> Option<(f32x4, f32x4, u32x4)> { let [(inf0, uq0), (inf1, uq1)] = quads; if uq0 > missing_msbs + 2 || uq1 > missing_msbs + 2 { return None; @@ -217,7 +219,7 @@ fn decode_two_quads_to_f32( if (inf0 | inf1) & 0xF0 == 0 { let zero = final_samples::(lanes, u32x8::splat(simd, 0), output); let (zero, _) = simd.split_f32x8(zero); - return Some((zero, zero, [0; 4])); + return Some((zero, zero, u32x4::splat(simd, 0))); } let (samples, v_n) = decode_quad_pair(lanes, magsgn, quads, 29 - missing_msbs); @@ -227,7 +229,7 @@ fn decode_two_quads_to_f32( Some(( samples0.unzip_low(samples1), samples0.unzip_high(samples1), - v_n0.unzip_high(v_n1).to_array(), + v_n0.unzip_high(v_n1), )) } @@ -613,10 +615,13 @@ struct RowOutput<'o> { /// Decode a pair's two quads straight to final samples, store both rows, /// and fold its bottom-row `v_n` into `row.v_n` as the stored-row walkers do. +/// `carry` holds the previous pair's bottom row; its last lane is the `v_n` +/// carried into this pair's first column pair. #[expect( + clippy::cast_possible_truncation, clippy::inline_always, clippy::too_many_arguments, - reason = "per-pair direct-output hot step" + reason = "the two 32-bit halves of the folded word; per-pair direct-output hot step" )] #[inline(always)] fn decode_pair_to_f32( @@ -627,9 +632,9 @@ fn decode_pair_to_f32( quads: [(u32, u32); 2], missing_msbs: u32, output: FinalOutput, - carry: &mut u32, + carry: &mut u32x4, ) -> Option<()> { - let (samples0, samples1, [c0, c1, c2, c3]) = decode_two_quads_to_f32::( + let (samples0, samples1, bottom) = decode_two_quads_to_f32::( lanes, magsgn, quads, @@ -640,8 +645,12 @@ fn decode_pair_to_f32( if let Some(second) = row.second.get_mut(index) { samples1.store_array(second); } - *row.v_n.get_mut(index)? = [*carry | c0, c1 | c2]; - *carry = c3; + // `[carry, c0, c1, c2] | [c0, c1, c2, c3]`: lanes 0 and 2 are the pair's + // `[carry | c0, c1 | c2]`, extracted as one 64-bit word. + let folded = carry.slide_within_blocks::<3>(bottom) | bottom; + let [v_n, _] = folded.unzip_low(folded).bitcast::>().to_array(); + *row.v_n.get_mut(index)? = [v_n as u32, (v_n >> 32) as u32]; + *carry = bottom; Some(()) } @@ -665,7 +674,7 @@ fn decode_first_row_to_f32( output: FinalOutput, ) -> Option { let mut c_q = 0; - let mut carry = 0; + let mut carry = u32x4::splat(lanes.simd, 0); for index in 0..row.first.len() { // Two quads in zero context inside one MEL run of zeros code no VLC, // U-VLC or MagSgn bits: skip to the symbols they decode to (the @@ -690,7 +699,7 @@ fn decode_first_row_to_f32( &mut carry, )?; } - Some(carry) + Some(carry.to_array()[3]) } /// Decode a non-initial quad row straight to final samples and return its @@ -715,7 +724,7 @@ fn decode_row_to_f32( output: FinalOutput, ) -> Option { let mut c_q = 0; - let mut carry = 0; + let mut carry = u32x4::splat(lanes.simd, 0); for index in 0..row.first.len() { let [above0, above1] = above.get(index..)?.first_chunk::<2>()?; let above = |k: usize| u32::from(if k < 4 { above0[k] } else { above1[0] }); @@ -751,7 +760,7 @@ fn decode_row_to_f32( &mut carry, )?; } - Some(carry) + Some(carry.to_array()[3]) } /// Cleanup-only SIMD rows that publish final dequantized coefficients From 01665b4996aef9e45ee526158f7d8f20c3ecd342 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 01:27:04 -0400 Subject: [PATCH 25/53] perf(j2k-native): fuse cleanup and MagSgn for HT refinement blocks Code blocks with SigProp or MagRef passes decoded their cleanup pass in two sweeps: every quad row's VLC/MEL symbols, then a separate MagSgn pass over the stored symbols. Cleanup-only blocks already use the fused row decoder, where the latency-bound VLC chain of one pair overlaps the MagSgn work of the previous one. Refinement-bearing blocks now take the same fused phase; they keep the zeroed symbol scratch the sigma build reads past the block. The split MagSgn phase stays as the test reference. MagRef visited all 32 samples of each 8x4 group through nested column and row branches. It now walks the set bits of the group's significance word, whose bit order (4 * column + row) is the scan order the refinement bits follow. On a 10-core ARM64 machine, interleaved against the base and including the earlier commits, 512x512 and 256x256 HT tiles encoded at 2 bpp (most of their blocks carry refinement passes) decode 9.9% and 8.2% faster at 1 thread. The 4096x2048 RGB HT lossless decode stays 10.7% faster at 1 thread and HT 2 bpp 4.2%; 1024x1024 regions of an 18000x12000 HT image 2.0 to 2.1% at 1 thread; 8 concurrent region decodes unchanged within noise. Output is unchanged. --- .../src/j2c/ht_block_decode/magnitude.rs | 3 + .../src/j2c/ht_block_decode/magnitude/simd.rs | 1 + .../src/j2c/ht_block_decode/pipeline.rs | 63 ++++++------------- .../src/j2c/ht_block_decode/refinement.rs | 35 ++++------- 4 files changed, 36 insertions(+), 66 deletions(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs index 26e11d86b..7798f7761 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs @@ -61,6 +61,9 @@ fn decode_mag_sgn_sample_with_vn( (((ms_val << 31) | magnitude) & significant, v_n) } +/// The `MagSgn` pass alone, from already decoded cleanup symbols: the +/// reference the fused cleanup/`MagSgn` rows are tested against. +#[cfg(test)] #[inline(never)] #[expect( clippy::too_many_arguments, diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs index e709186a7..6141e6af8 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs @@ -422,6 +422,7 @@ fn decode_stored_row( } /// SIMD twin of the `MagSgn` row loop in [`super::decode_magnitude_sign_phase`]. +#[cfg(test)] #[expect( clippy::inline_always, clippy::too_many_arguments, diff --git a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs index 53d6788f3..2bab8aaf5 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs @@ -1,11 +1,9 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::cleanup::{ - cleanup_segment_suffix_length, cleanup_symbol_stride, decode_cleanup_symbols, -}; +use super::cleanup::{cleanup_segment_suffix_length, cleanup_symbol_stride}; +use super::magnitude::decode_cleanup_and_magnitude_sign_phase; #[cfg(feature = "simd")] use super::magnitude::decode_cleanup_and_magnitude_sign_to_f32; -use super::magnitude::{decode_cleanup_and_magnitude_sign_phase, decode_magnitude_sign_phase}; use super::refinement::apply_magnitude_refinement_phase; use super::significance::{ apply_significance_propagation_phase, build_sigma_from_cleanup_phase, sigma_stride, @@ -26,7 +24,6 @@ pub(crate) const PHASE_LIMIT_MAGREF: u8 = 2; #[expect( clippy::inline_always, clippy::too_many_arguments, - clippy::too_many_lines, reason = "the monomorphized HT phase pipeline has a stable hot signature and preserves normative pass order" )] #[inline(always)] @@ -77,44 +74,24 @@ pub(super) fn decode_impl( zeroed_u16_scratch(&mut scratch_buffers.cleanup, sstr * (quad_rows + 1))? }; - if cleanup_only { - decode_cleanup_and_magnitude_sign_phase( - cleanup_data, - lcup, - scup, - decoded_data, - missing_msbs, - width, - height, - stride, - sstr, - scratch, - v_n_scratch, - &mut scratch_buffers.magsgn, - observer, - )?; - } else { - let phase_start = observer.phase_start(); - decode_cleanup_symbols(cleanup_data, lcup, scup, width, height, sstr, scratch)?; - observer.add_cleanup_us(phase_start); - - let phase_start = observer.phase_start(); - decode_magnitude_sign_phase( - cleanup_data, - lcup, - scup, - scratch, - decoded_data, - missing_msbs, - width, - height, - stride, - sstr, - v_n_scratch, - &mut scratch_buffers.magsgn, - )?; - observer.add_mag_sgn_us(phase_start); - } + // Refinement-bearing blocks decode their cleanup pass the same way; they + // only keep a zeroed symbol scratch, whose rows past the block the sigma + // build reads. + decode_cleanup_and_magnitude_sign_phase( + cleanup_data, + lcup, + scup, + decoded_data, + missing_msbs, + width, + height, + stride, + sstr, + scratch, + v_n_scratch, + &mut scratch_buffers.magsgn, + observer, + )?; if PHASE_LIMIT == PHASE_LIMIT_CLEANUP { return Some(()); diff --git a/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs b/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs index 2f366ab2f..064805a06 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs @@ -41,31 +41,20 @@ pub(super) fn apply_magnitude_refinement_phase( // the cleanup pass, nothing is written for them. let in_block = (0x1111_1111u32 * rows_in_block) & (u32::MAX >> (32 - 4 * (width - i).min(8))); - let mut col_mask = 0xFu32; + // Significant samples take refinement bits in bit order: column + // by column, top to bottom (bit `4 * column + row`). + let base = dpp + i as usize; + let mut pending = sig; let mut cwd_mut = cwd; - - if sig != 0 { - for j in 0..8 { - if (sig & col_mask) != 0 { - let mut dp = dpp + i as usize + j; - let mut sample_mask = 0x1111_1111u32 & col_mask; - - for _ in 0..4 { - if (sig & sample_mask) != 0 { - let mut sym = cwd_mut & 1; - sym = (1 - sym) << (p - 1); - sym |= half; - if (in_block & sample_mask) != 0 { - decoded_data[dp] ^= sym; - } - cwd_mut >>= 1; - } - sample_mask <<= 1; - dp += stride as usize; - } - } - col_mask <<= 4; + while pending != 0 { + let bit = pending.trailing_zeros(); + pending &= pending - 1; + if in_block & (1 << bit) != 0 { + let sym = ((1 - (cwd_mut & 1)) << (p - 1)) | half; + decoded_data + [base + (bit >> 2) as usize + (bit & 3) as usize * stride as usize] ^= sym; } + cwd_mut >>= 1; } magref.advance(sig.count_ones()); From d6ae72a47eb412c0bdcef7b45a0f87bfdd9f96dd Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 01:43:19 -0400 Subject: [PATCH 26/53] perf(j2k-native): decode HT SigProp by column and refinement-block VLC bits from a flat buffer Two changes to code blocks that carry refinement passes. SigProp decoded each 4x4 group one candidate sample at a time, with a branch on every refinement bit (a one makes the sample significant and spreads candidacy to its later neighbors). It now decodes a column of four samples per lookup: a 4096-entry table indexed by the column's candidates, its still-insignificant samples and the next four bits gives the newly significant samples and the bits used, and the new samples make their neighbors in the next column candidates. A test checks the column decode against the candidate scan on random groups. The fused cleanup/MagSgn rows these blocks now use read their VLC bits through the reverse reservoir reader, checking for stuffing on every pair. They now unstuff the VLC bits once per block into the flat buffer the direct-output rows use, and inline the MEL and VLC refills as those rows do. The scalar rows keep the reverse reader. On a 10-core ARM64 machine, interleaved against the base and including the earlier commits, 512x512 and 256x256 HT tiles encoded at 2 bpp decode 11.1% and 10.9% faster at 1 thread. The 4096x2048 RGB HT lossless decode stays 10.7% faster at 1 thread and HT 2 bpp 4.3%; 1024x1024 regions of an 18000x12000 HT image 2.1 to 2.5% at 1 thread; 8 concurrent region decodes unchanged within noise. Output is unchanged. --- .../src/j2c/ht_block_decode/cleanup.rs | 30 +++--- .../src/j2c/ht_block_decode/magnitude.rs | 12 ++- .../src/j2c/ht_block_decode/magnitude/simd.rs | 14 +-- .../src/j2c/ht_block_decode/pipeline.rs | 1 + .../src/j2c/ht_block_decode/significance.rs | 99 +++++++++++++++---- .../src/j2c/ht_block_decode/tests.rs | 53 +++++++++- 6 files changed, 167 insertions(+), 42 deletions(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs b/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs index e7a5c4079..6e53e1238 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/cleanup.rs @@ -201,9 +201,9 @@ fn store_pair_symbols(scratch: &mut [u16], row_offset: usize, symbols: [u32; 4]) reason = "this per-pair decoder is a hot path" )] #[inline(always)] -pub(super) fn decode_cleanup_first_row_pair( +pub(super) fn decode_cleanup_first_row_pair( mel: &mut MelDecoder, - vlc: &mut ReverseBitReader, + vlc: &mut V, run: &mut i32, scratch: &mut [u16], row_offset: usize, @@ -211,7 +211,8 @@ pub(super) fn decode_cleanup_first_row_pair( x: u32, width: u32, ) -> Option<[u32; 4]> { - let symbols = decode_initial_pair_symbols::(mel, vlc, run, c_q, x + 2 < width)?; + let symbols = + decode_initial_pair_symbols::(mel, vlc, run, c_q, x + 2 < width)?; store_pair_symbols(scratch, row_offset, symbols); Some(symbols) } @@ -220,9 +221,9 @@ pub(super) fn decode_cleanup_first_row_pair( /// shared MEL/VLC state; the trailing sentinel pair is written after the row. #[expect(clippy::inline_always, reason = "this per-row decoder is a hot path")] #[inline(always)] -pub(super) fn decode_cleanup_symbols_first_row( +pub(super) fn decode_cleanup_symbols_first_row( mel: &mut MelDecoder, - vlc: &mut ReverseBitReader, + vlc: &mut V, run: &mut i32, scratch: &mut [u16], width: u32, @@ -232,7 +233,9 @@ pub(super) fn decode_cleanup_symbols_first_row( let mut x = 0u32; while x < width { - decode_cleanup_first_row_pair(mel, vlc, run, scratch, row_offset, &mut c_q, x, width)?; + decode_cleanup_first_row_pair::( + mel, vlc, run, scratch, row_offset, &mut c_q, x, width, + )?; x += 4; row_offset += 4; } @@ -327,9 +330,9 @@ pub(super) fn decode_pair_symbols( reason = "this per-pair decoder is a hot path" )] #[inline(always)] -pub(super) fn decode_cleanup_row_pair( +pub(super) fn decode_cleanup_row_pair( mel: &mut MelDecoder, - vlc: &mut ReverseBitReader, + vlc: &mut V, run: &mut i32, scratch: &mut [u16], row_offset: usize, @@ -341,7 +344,8 @@ pub(super) fn decode_cleanup_row_pair( // `sstr >= 8`, so the row above never overlaps this pair's symbols. let prev = row_offset - sstr; let above = |k| u32::from(scratch[prev + k]); - let symbols = decode_pair_symbols::(mel, vlc, run, c_q, above, x + 2 < width)?; + let symbols = + decode_pair_symbols::(mel, vlc, run, c_q, above, x + 2 < width)?; store_pair_symbols(scratch, row_offset, symbols); Some(symbols) } @@ -351,9 +355,9 @@ pub(super) fn decode_cleanup_row_pair( /// state; the trailing sentinel pair is written after the row. #[expect(clippy::inline_always, reason = "this per-row decoder is a hot path")] #[inline(always)] -pub(super) fn decode_cleanup_symbols_row( +pub(super) fn decode_cleanup_symbols_row( mel: &mut MelDecoder, - vlc: &mut ReverseBitReader, + vlc: &mut V, run: &mut i32, scratch: &mut [u16], width: u32, @@ -365,7 +369,9 @@ pub(super) fn decode_cleanup_symbols_row( let mut row_offset = (y >> 1) as usize * sstr; while x < width { - decode_cleanup_row_pair(mel, vlc, run, scratch, row_offset, sstr, &mut c_q, x, width)?; + decode_cleanup_row_pair::( + mel, vlc, run, scratch, row_offset, sstr, &mut c_q, x, width, + )?; x += 4; row_offset += 4; } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs index 7798f7761..bcfa85389 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs @@ -3,9 +3,9 @@ use alloc::vec::Vec; use super::cleanup::{decode_cleanup_symbols_first_row, decode_cleanup_symbols_row}; -#[cfg(feature = "simd")] -use super::readers::unstuff_vlc; use super::readers::{unstuff_magsgn, FlatBitReader, MelDecoder, ReverseBitReader}; +#[cfg(feature = "simd")] +use super::readers::{unstuff_vlc, FlatVlcReader}; use super::stats::HtDecodeObserver; #[cfg(feature = "simd")] use crate::math::{dispatch, Level}; @@ -351,6 +351,7 @@ pub(super) fn decode_cleanup_and_magnitude_sign_phase( scratch: &mut [u16], v_n_scratch: &mut [u32], magsgn_buffer: &mut Vec, + vlc_buffer: &mut Vec, observer: &mut impl HtDecodeObserver, ) -> Option<()> { let quad_rows = height.div_ceil(2) as usize; @@ -364,7 +365,6 @@ pub(super) fn decode_cleanup_and_magnitude_sign_phase( v_n_scratch[..v_n_width].fill(0); let mut mel = MelDecoder::new(coded_data, lcup, scup); - let mut vlc = ReverseBitReader::new_vlc(coded_data, lcup, scup); let consumable = (width as usize) .checked_mul(height as usize)? .checked_mul(4)?; @@ -374,6 +374,9 @@ pub(super) fn decode_cleanup_and_magnitude_sign_phase( #[cfg(feature = "simd")] if uses_simd_rows(missing_msbs) { + // As the direct-output rows: the VLC bits unstuffed once per block. + let flat_vlc = unstuff_vlc(coded_data, lcup, scup, vlc_buffer)?; + let mut vlc = FlatVlcReader::new(flat_vlc)?; return dispatch!(Level::new(), simd => simd::decode_cleanup_and_magnitude_sign_rows( simd, &mut mel, @@ -392,6 +395,9 @@ pub(super) fn decode_cleanup_and_magnitude_sign_phase( )); } + #[cfg(not(feature = "simd"))] + let _ = vlc_buffer; + let mut vlc = ReverseBitReader::new_vlc(coded_data, lcup, scup); let phase_start = observer.phase_start(); decode_cleanup_symbols_first_row(&mut mel, &mut vlc, &mut run, scratch, width)?; observer.add_cleanup_us(phase_start); diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs index 6141e6af8..43525fbba 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs @@ -16,7 +16,7 @@ use super::super::cleanup::{ decode_cleanup_first_row_pair, decode_cleanup_row_pair, decode_cleanup_symbols_first_row, decode_cleanup_symbols_row, decode_initial_pair_symbols, decode_pair_symbols, }; -use super::super::readers::{FlatBitReader, FlatVlcReader, MelDecoder, ReverseBitReader}; +use super::super::readers::{FlatBitReader, FlatVlcReader, MelDecoder, VlcBits}; use super::super::stats::HtDecodeObserver; use super::decode_magnitude_sign_pair_from_cleanup; @@ -470,10 +470,10 @@ pub(super) fn decode_magnitude_sign_rows( reason = "fused row walker inlined into the dispatched SIMD level" )] #[inline(always)] -fn decode_fused_row( +fn decode_fused_row( lanes: &Lanes, mel: &mut MelDecoder<'_>, - vlc: &mut ReverseBitReader<'_>, + vlc: &mut V, run: &mut i32, magsgn: &mut FlatBitReader<'_>, scratch: &mut [u16], @@ -498,12 +498,12 @@ fn decode_fused_row( let mut x = 0u32; while x < width { let quads = if y == 0 { - let [t0, u0, t1, u1] = decode_cleanup_first_row_pair( + let [t0, u0, t1, u1] = decode_cleanup_first_row_pair::( mel, vlc, run, scratch, row_offset, &mut c_q, x, width, )?; [(t0, u0), (t1, u1)] } else { - let symbols = decode_cleanup_row_pair( + let symbols = decode_cleanup_row_pair::( mel, vlc, run, scratch, row_offset, sstr, &mut c_q, x, width, )?; with_kappa(symbols, v_n_scratch, cursor.vp) @@ -538,10 +538,10 @@ fn decode_fused_row( reason = "inlined into the dispatched SIMD level" )] #[inline(always)] -pub(super) fn decode_cleanup_and_magnitude_sign_rows( +pub(super) fn decode_cleanup_and_magnitude_sign_rows( simd: S, mel: &mut MelDecoder<'_>, - vlc: &mut ReverseBitReader<'_>, + vlc: &mut V, run: &mut i32, magsgn: &mut FlatBitReader<'_>, scratch: &mut [u16], diff --git a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs index 2bab8aaf5..663efa830 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs @@ -90,6 +90,7 @@ pub(super) fn decode_impl( scratch, v_n_scratch, &mut scratch_buffers.magsgn, + &mut scratch_buffers.vlc, observer, )?; diff --git a/crates/j2k-native/src/j2c/ht_block_decode/significance.rs b/crates/j2k-native/src/j2c/ht_block_decode/significance.rs index 708222619..80fa12f64 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/significance.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/significance.rs @@ -2,11 +2,89 @@ use super::readers::{read_u32_pair, ForwardBitReader}; +/// The samples each newly significant sample of a 4x4 group makes a SigProp +/// candidate, as bits `4 * column + row`: itself, the sample below it, and +/// the three nearest of the next column. The reference for +/// [`SIGPROP_COLUMN`], and the order the encoder visits. +#[cfg(test)] pub(super) const SIGPROP_SPREAD_MASKS: [u32; 16] = [ 0x33, 0x76, 0xEC, 0xC8, 0x330, 0x760, 0xEC0, 0xC80, 0x3300, 0x7600, 0xEC00, 0xC800, 0x33000, 0x76000, 0xEC000, 0xC8000, ]; +/// SigProp decoding of one four-sample column of a stripe, indexed by +/// `candidates | insignificant << 4 | bits << 8`: the column's candidates, +/// its samples not yet significant, and the next four refinement bits. A +/// candidate takes one bit, top to bottom; a one makes it significant and +/// the insignificant sample below it a candidate. Each entry is +/// `significant | bits_used << 4`. +const SIGPROP_COLUMN: [u8; 4096] = sigprop_column_table(); + +#[expect( + clippy::cast_possible_truncation, + reason = "entries hold four significance bits and a bit count up to four" +)] +const fn sigprop_column_table() -> [u8; 4096] { + let mut table = [0; 4096]; + let mut index = 0; + while index < 4096 { + let mut candidates = index & 0xF; + let insignificant = (index >> 4) & 0xF; + let bits = index >> 8; + let mut used = 0; + let mut significant = 0; + let mut row = 0; + while row < 4 { + if candidates & (1 << row) != 0 { + if (bits >> used) & 1 != 0 { + significant |= 1 << row; + candidates |= (1 << (row + 1)) & insignificant; + } + used += 1; + } + row += 1; + } + table[index] = (significant | (used << 4)) as u8; + index += 1; + } + table +} + +/// The significance bits of one 4x4 SigProp group: `candidates` are the +/// samples next to a sample significant before this pass, `insignificant` +/// the samples that can still become significant (bits `4 * column + row`), +/// and `cwd` the group's refinement bits. Newly significant samples make +/// their later neighbors candidates; the scan is column by column, top to +/// bottom. Returns the newly significant samples and the bits used. +#[expect(clippy::inline_always, reason = "per-group hot path")] +#[inline(always)] +pub(super) fn decode_group_significance( + candidates: u32, + insignificant: u32, + cwd: u32, +) -> (u32, u32) { + let mut bits = cwd; + let mut used = 0; + let mut significant = 0; + // Candidates the previous column's new samples add to this one: the + // samples beside them and diagonally next to them. + let mut spread = 0; + for column in 0..4 { + let shift = 4 * column; + let column_insignificant = (insignificant >> shift) & 0xF; + let column_candidates = ((candidates >> shift) & 0xF) | (spread & column_insignificant); + let index = column_candidates | (column_insignificant << 4) | ((bits & 0xF) << 8); + let entry = u32::from(SIGPROP_COLUMN[index as usize]); + let column_significant = entry & 0xF; + let column_used = entry >> 4; + bits >>= column_used; + used += column_used; + significant |= column_significant << shift; + spread = column_significant | (column_significant << 1) | (column_significant >> 1); + } + (significant, used) +} + pub(crate) fn sigma_stride(width: u32) -> usize { ((width.div_ceil(4) + 2 + 7) & !7) as usize } @@ -138,24 +216,9 @@ pub(super) fn apply_significance_propagation_phase( let mut new_sig = 0u32; if mbr != 0 { let mut cwd = sigprop.fetch(); - let mut cnt = 0u32; - let inv_sig = !cs & col_pattern; - let mut candidates = mbr; - let mut processed = 0u32; - - while candidates != 0 { - let bit = candidates.trailing_zeros(); - let sample_mask = 1u32 << bit; - candidates &= !sample_mask; - processed |= sample_mask; - - if (cwd & 1) != 0 { - new_sig |= sample_mask; - candidates |= SIGPROP_SPREAD_MASKS[bit as usize] & inv_sig & !processed; - } - cwd >>= 1; - cnt += 1; - } + let (significant, mut cnt) = decode_group_significance(mbr, !cs & col_pattern, cwd); + new_sig = significant; + cwd >>= cnt; if new_sig != 0 { let value = 3u32 << (p - 2); diff --git a/crates/j2k-native/src/j2c/ht_block_decode/tests.rs b/crates/j2k-native/src/j2c/ht_block_decode/tests.rs index f85520074..4827e0798 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/tests.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/tests.rs @@ -11,8 +11,8 @@ use super::readers::{ForwardBitReader, ReverseBitReader}; use super::refinement::apply_magnitude_refinement_phase; use super::segments::{CombinedCodeBlockData, HtCodeBlockSegments}; use super::significance::{ - apply_significance_propagation_phase, build_sigma_from_cleanup_phase, sigma_stride, - SIGPROP_SPREAD_MASKS, + apply_significance_propagation_phase, build_sigma_from_cleanup_phase, + decode_group_significance, sigma_stride, SIGPROP_SPREAD_MASKS, }; use super::state::{zeroed_u16_scratch, zeroed_u32_scratch, HtBlockDecodeScratch}; use super::stats::{HtBlockDecodeStats, NoHtDecodeStats}; @@ -388,6 +388,55 @@ fn sigprop_spread_masks_follow_column_major_scan_order() { } } +/// One SigProp group decoded a candidate at a time, in scan order: the +/// reference for the column-table decode. +fn group_significance_by_candidate(candidates: u32, insignificant: u32, cwd: u32) -> (u32, u32) { + let (mut candidates, mut cwd) = (candidates, cwd); + let (mut significant, mut processed, mut used) = (0u32, 0u32, 0u32); + while candidates != 0 { + let bit = candidates.trailing_zeros(); + let sample = 1u32 << bit; + candidates &= !sample; + processed |= sample; + if cwd & 1 != 0 { + significant |= sample; + candidates |= SIGPROP_SPREAD_MASKS[bit as usize] & insignificant & !processed; + } + cwd >>= 1; + used += 1; + } + (significant, used) +} + +#[test] +fn sigprop_group_columns_match_candidate_scan() { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for case in 0..200_000 { + let word = next(); + let insignificant = (word & 0xFFFF) as u32; + // Sparse and dense candidate sets, always among the insignificant + // samples, as the decoder builds them. + let dense = if case % 2 == 0 { + 0 + } else { + (word >> 48) as u32 + }; + let candidates = (((word >> 16) as u32 & (word >> 32) as u32) | dense) & insignificant; + let cwd = next() as u32; + assert_eq!( + decode_group_significance(candidates, insignificant, cwd), + group_significance_by_candidate(candidates, insignificant, cwd), + "candidates={candidates:#06x} insignificant={insignificant:#06x} cwd={cwd:#010x}" + ); + } +} + #[test] fn combined_data_exposes_borrowed_segment_slices() { let combined = CombinedCodeBlockData { From f4e550564256b71681ce58f389a26b9e2c3e6dc9 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:05:05 -0400 Subject: [PATCH 27/53] perf(j2k-native): decode HT refinement blocks' cleanup through the direct rows The direct-output row walkers (pair loop with the MEL-run shortcut, flat VLC and MagSgn buffers, vector v_n fold) served only cleanup-only blocks; refinement-bearing blocks decoded their cleanup pass through the older fused row loop, about 12 to 21% slower on the same blocks. The direct rows are now generic over what they publish per sample: `FinalOutput` keeps the dequantized f32 conversion, `SignMagnitude` stores the sign-magnitude words that SigProp and MagRef refine. Blocks whose width is a multiple of four (and within the SIMD bit budget) take the direct rows unless the phases are timed apart; others keep the fused loop. Tests check the sign-magnitude rows against the scalar rows on encoded and arbitrary blocks. On a 10-core ARM64 machine, interleaved against the base and including the earlier commits, 512x512 and 256x256 HT tiles encoded at 2 bpp decode 14.8% and 13.9% faster at 1 thread. The 4096x2048 RGB HT lossless decode stays 10.7% faster at 1 thread and HT 2 bpp 4.3%; 1024x1024 regions of an 18000x12000 HT image 2.1 to 2.3% at 1 thread and unchanged at 10; 8 concurrent region decodes unchanged within noise. Output is unchanged. --- .../src/j2c/ht_block_decode/magnitude.rs | 66 +++++- .../src/j2c/ht_block_decode/magnitude/simd.rs | 224 +++++++++++++----- .../src/j2c/ht_block_decode/pipeline.rs | 63 +++-- 3 files changed, 274 insertions(+), 79 deletions(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs index bcfa85389..ac89adb3c 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude.rs @@ -480,13 +480,13 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32 { coefficient_shift: 31 - u32::from(coded_bitplanes), midpoint_scale: crate::math::pow2i(-(31_i32 - i32::from(coded_bitplanes))), dequantization_step, }; - dispatch!(Level::new(), simd => simd::decode_cleanup_and_magnitude_sign_to_f32_rows::( + dispatch!(Level::new(), simd => simd::decode_cleanup_and_magnitude_sign_direct_rows( simd, coded_data, lcup, @@ -504,3 +504,65 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32, + vlc_buffer: &mut Vec, +) -> Option<()> { + if !supports_cleanup_to_f32(missing_msbs, width) { + return None; + } + let quad_rows = height.div_ceil(2) as usize; + if scratch.len() < sstr * (quad_rows + 1) { + return None; + } + let v_n_width = width.div_ceil(2) as usize + 2; + if v_n_scratch.len() < v_n_width { + return None; + } + v_n_scratch[..v_n_width].fill(0); + + let consumable = (width as usize) + .checked_mul(height as usize)? + .checked_mul(4)?; + let flat = unstuff_magsgn(&coded_data[..lcup - scup], magsgn_buffer, consumable)?; + let flat_vlc = unstuff_vlc(coded_data, lcup, scup, vlc_buffer)?; + + dispatch!(Level::new(), simd => simd::decode_cleanup_and_magnitude_sign_direct_rows( + simd, + coded_data, + lcup, + scup, + flat, + flat_vlc, + scratch, + decoded_data, + v_n_scratch, + missing_msbs, + width, + height, + stride, + sstr, + simd::SignMagnitude, + )) +} diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs index 43525fbba..8305812be 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs @@ -28,7 +28,7 @@ use super::decode_magnitude_sign_pair_from_cleanup; pub(super) const MAX_SIMD_MMSBP2: u32 = 25; #[derive(Clone, Copy)] -struct Lanes { +pub(super) struct Lanes { simd: S, sig_shift: u32x8, emb_k_shift: u32x8, @@ -37,13 +37,86 @@ struct Lanes { byte_offsets: u32x8, } +/// Final dequantized `f32` coefficients, the direct rows' output for +/// cleanup-only blocks. `IRREVERSIBLE_MIDPOINT` selects the 9/7 midpoint +/// reconstruction over the 5/3 integer one. #[derive(Clone, Copy)] -pub(super) struct FinalOutput { +pub(super) struct FinalOutput { pub(super) coefficient_shift: u32, pub(super) midpoint_scale: f32, pub(super) dequantization_step: f32, } +/// The sign-magnitude words the refinement passes refine, the direct rows' +/// output for refinement-bearing blocks. +#[derive(Clone, Copy)] +pub(super) struct SignMagnitude; + +/// What the direct rows store for each sample. +pub(super) trait DirectOutput: Copy { + type Sample: Copy; + type Row: Copy; + /// A pair's sign-magnitude samples (one quad per 128-bit half) as its + /// two output rows. + fn rows(self, lanes: &Lanes, samples: u32x8) -> (Self::Row, Self::Row); + /// A quad row of zero coefficients. + fn zero_row(self, lanes: &Lanes) -> Self::Row; + fn store(row: Self::Row, dst: &mut [Self::Sample; 4]); +} + +impl DirectOutput + for FinalOutput +{ + type Sample = f32; + type Row = f32x4; + + #[expect(clippy::inline_always, reason = "per-pair direct-output hot step")] + #[inline(always)] + fn rows(self, lanes: &Lanes, samples: u32x8) -> (f32x4, f32x4) { + let samples = final_samples::(lanes, samples, self); + let (samples0, samples1) = lanes.simd.split_f32x8(samples); + (samples0.unzip_low(samples1), samples0.unzip_high(samples1)) + } + + #[expect(clippy::inline_always, reason = "per-pair direct-output hot step")] + #[inline(always)] + fn zero_row(self, lanes: &Lanes) -> f32x4 { + let simd = lanes.simd; + let zero = final_samples::(lanes, u32x8::splat(simd, 0), self); + simd.split_f32x8(zero).0 + } + + #[expect(clippy::inline_always, reason = "per-pair direct-output hot step")] + #[inline(always)] + fn store(row: f32x4, dst: &mut [f32; 4]) { + row.store_array(dst); + } +} + +impl DirectOutput for SignMagnitude { + type Sample = u32; + type Row = u32x4; + + #[expect(clippy::inline_always, reason = "per-pair direct-output hot step")] + #[inline(always)] + fn rows(self, lanes: &Lanes, samples: u32x8) -> (u32x4, u32x4) { + let (samples0, samples1) = lanes.simd.split_u32x8(samples); + (samples0.unzip_low(samples1), samples0.unzip_high(samples1)) + } + + #[expect(clippy::inline_always, reason = "per-pair direct-output hot step")] + #[inline(always)] + fn zero_row(self, lanes: &Lanes) -> u32x4 { + u32x4::splat(lanes.simd, 0) + } + + #[expect(clippy::inline_always, reason = "per-pair direct-output hot step")] + #[inline(always)] + fn store(row: u32x4, dst: &mut [u32; 4]) { + row.store_array(dst); + } +} + impl Lanes { #[expect( clippy::inline_always, @@ -181,7 +254,7 @@ fn decode_two_quads( fn final_samples( lanes: &Lanes, samples: u32x8, - output: FinalOutput, + output: FinalOutput, ) -> f32x8 { let magnitude = samples & 0x7FFF_FFFF; let values = if IRREVERSIBLE_MIDPOINT { @@ -197,19 +270,17 @@ fn final_samples( values * output.dequantization_step } -/// Decode a full-width pair into final samples: both output rows and the -/// bottom-row `v_n` of its four columns. This is used only by whole-image -/// cleanup-only decode, where no refinement pass needs the intermediate -/// sign-magnitude words. +/// Decode a full-width pair into its two output rows and the bottom-row +/// `v_n` of its four columns. #[expect(clippy::inline_always, reason = "two-quad direct-output hot step")] #[inline(always)] -fn decode_two_quads_to_f32( +fn decode_two_quads_direct, S: Simd>( lanes: &Lanes, magsgn: &mut FlatBitReader<'_>, quads: [(u32, u32); 2], missing_msbs: u32, - output: FinalOutput, -) -> Option<(f32x4, f32x4, u32x4)> { + output: O, +) -> Option<(O::Row, O::Row, u32x4)> { let [(inf0, uq0), (inf1, uq1)] = quads; if uq0 > missing_msbs + 2 || uq1 > missing_msbs + 2 { return None; @@ -217,20 +288,14 @@ fn decode_two_quads_to_f32( let simd = lanes.simd; if (inf0 | inf1) & 0xF0 == 0 { - let zero = final_samples::(lanes, u32x8::splat(simd, 0), output); - let (zero, _) = simd.split_f32x8(zero); + let zero = output.zero_row(lanes); return Some((zero, zero, u32x4::splat(simd, 0))); } let (samples, v_n) = decode_quad_pair(lanes, magsgn, quads, 29 - missing_msbs); - let samples = final_samples::(lanes, samples, output); - let (samples0, samples1) = simd.split_f32x8(samples); + let (row0, row1) = output.rows(lanes, samples); let (v_n0, v_n1) = simd.split_u32x8(v_n); - Some(( - samples0.unzip_low(samples1), - samples0.unzip_high(samples1), - v_n0.unzip_high(v_n1), - )) + Some((row0, row1, v_n0.unzip_high(v_n1))) } /// Store a quad pair with no significant sample: zero output and zero `v_n`. @@ -605,10 +670,10 @@ pub(super) fn decode_cleanup_and_magnitude_sign_rows( /// One quad row of direct output, sliced to the row's quad pairs (`v_n` to /// one more), so the pair loop needs no bounds checks. -struct RowOutput<'o> { - first: &'o mut [[f32; 4]], +struct RowOutput<'o, T> { + first: &'o mut [[T; 4]], /// Empty for the last row of an odd-height block. - second: &'o mut [[f32; 4]], + second: &'o mut [[T; 4]], symbols: &'o mut [[u16; 4]], /// The row above's `v_n` column pairs on entry, this row's on exit. v_n: &'o mut [[u32; 2]], @@ -625,26 +690,21 @@ struct RowOutput<'o> { reason = "the two 32-bit halves of the folded word; per-pair direct-output hot step" )] #[inline(always)] -fn decode_pair_to_f32( +fn decode_pair_direct, S: Simd>( lanes: &Lanes, magsgn: &mut FlatBitReader<'_>, - row: &mut RowOutput<'_>, + row: &mut RowOutput<'_, O::Sample>, index: usize, quads: [(u32, u32); 2], missing_msbs: u32, - output: FinalOutput, + output: O, carry: &mut u32x4, ) -> Option<()> { - let (samples0, samples1, bottom) = decode_two_quads_to_f32::( - lanes, - magsgn, - quads, - missing_msbs, - output, - )?; - samples0.store_array(row.first.get_mut(index)?); + let (samples0, samples1, bottom) = + decode_two_quads_direct::(lanes, magsgn, quads, missing_msbs, output)?; + O::store(samples0, row.first.get_mut(index)?); if let Some(second) = row.second.get_mut(index) { - samples1.store_array(second); + O::store(samples1, second); } // `[carry, c0, c1, c2] | [c0, c1, c2, c3]`: lanes 0 and 2 are the pair's // `[carry | c0, c1 | c2]`, extracted as one 64-bit word. @@ -664,15 +724,15 @@ fn decode_pair_to_f32( reason = "16-bit symbol fields; row walker inlined into the dispatched SIMD level" )] #[inline(always)] -fn decode_first_row_to_f32( +fn decode_first_row_direct, S: Simd>( lanes: &Lanes, mel: &mut MelDecoder<'_>, vlc: &mut FlatVlcReader<'_>, run: &mut i32, magsgn: &mut FlatBitReader<'_>, - row: &mut RowOutput<'_>, + row: &mut RowOutput<'_, O::Sample>, missing_msbs: u32, - output: FinalOutput, + output: O, ) -> Option { let mut c_q = 0; let mut carry = u32x4::splat(lanes.simd, 0); @@ -689,7 +749,7 @@ fn decode_first_row_to_f32( *row.symbols.get_mut(index)? = symbols.map(|symbol| symbol as u16); let [t0, u0, t1, u1] = symbols; let quads = [(t0, u0), (t1, u1)]; - decode_pair_to_f32::( + decode_pair_direct::( lanes, magsgn, row, @@ -713,16 +773,16 @@ fn decode_first_row_to_f32( reason = "16-bit symbol fields; row walker inlined into the dispatched SIMD level" )] #[inline(always)] -fn decode_row_to_f32( +fn decode_row_direct, S: Simd>( lanes: &Lanes, mel: &mut MelDecoder<'_>, vlc: &mut FlatVlcReader<'_>, run: &mut i32, magsgn: &mut FlatBitReader<'_>, above: &[[u16; 4]], - row: &mut RowOutput<'_>, + row: &mut RowOutput<'_, O::Sample>, missing_msbs: u32, - output: FinalOutput, + output: O, ) -> Option { let mut c_q = 0; let mut carry = u32x4::splat(lanes.simd, 0); @@ -750,7 +810,7 @@ fn decode_row_to_f32( (t0, u0 + kappa(t0, v_n0 | v_n1)), (t1, u1 + kappa(t1, v_n1 | v_n2)), ]; - decode_pair_to_f32::( + decode_pair_direct::( lanes, magsgn, row, @@ -773,10 +833,7 @@ fn decode_row_to_f32( reason = "inlined into the dispatched SIMD level" )] #[inline(always)] -pub(super) fn decode_cleanup_and_magnitude_sign_to_f32_rows< - const IRREVERSIBLE_MIDPOINT: bool, - S: Simd, ->( +pub(super) fn decode_cleanup_and_magnitude_sign_direct_rows, S: Simd>( simd: S, coded_data: &[u8], lcup: usize, @@ -784,14 +841,14 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32_rows< flat: &[u8], flat_vlc: &[u8], scratch: &mut [u16], - decoded_data: &mut [f32], + decoded_data: &mut [O::Sample], v_n_scratch: &mut [u32], missing_msbs: u32, width: u32, height: u32, stride: u32, sstr: usize, - output: FinalOutput, + output: O, ) -> Option<()> { let mut mel = MelDecoder::new(coded_data, lcup, scup); let mut vlc = FlatVlcReader::new(flat_vlc)?; @@ -829,7 +886,7 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32_rows< v_n: v_n.get_mut(..=pairs)?, }; let carry = if y == 0 { - decode_first_row_to_f32::( + decode_first_row_direct::( &lanes, &mut mel, &mut vlc, @@ -841,7 +898,7 @@ pub(super) fn decode_cleanup_and_magnitude_sign_to_f32_rows< )? } else { let above = prev.get(..4 * pairs + 4)?.as_chunks::<4>().0; - decode_row_to_f32::( + decode_row_direct::( &lanes, &mut mel, &mut vlc, @@ -879,8 +936,8 @@ mod tests { decode_magnitude_sign_first_row_from_cleanup, decode_magnitude_sign_row_from_cleanup, }; use super::{ - decode_cleanup_and_magnitude_sign_rows, decode_cleanup_and_magnitude_sign_to_f32_rows, - decode_magnitude_sign_rows, FinalOutput, MAX_SIMD_MMSBP2, + decode_cleanup_and_magnitude_sign_direct_rows, decode_cleanup_and_magnitude_sign_rows, + decode_magnitude_sign_rows, FinalOutput, SignMagnitude, MAX_SIMD_MMSBP2, }; use crate::j2c::ht_block_encode::encode_code_block; @@ -1055,18 +1112,25 @@ mod tests { let mut output = guarded_output(width, height, stride); let len = output.len(); let rows = &mut output[GUARD..len - GUARD]; - let conversion = FinalOutput { - coefficient_shift: 31 - u32::from(coded_bitplanes), - midpoint_scale: crate::math::pow2i(-(31_i32 - i32::from(coded_bitplanes))), - dequantization_step: step, - }; + let coefficient_shift = 31 - u32::from(coded_bitplanes); + let midpoint_scale = crate::math::pow2i(-(31_i32 - i32::from(coded_bitplanes))); if irreversible { - dispatch!(level, simd => decode_cleanup_and_magnitude_sign_to_f32_rows::( + let conversion = FinalOutput:: { + coefficient_shift, + midpoint_scale, + dequantization_step: step, + }; + dispatch!(level, simd => decode_cleanup_and_magnitude_sign_direct_rows( simd, coded, lcup, scup, flat, flat_vlc, &mut scratch, rows, &mut v_n, missing_msbs, width, height, stride, sstr, conversion, ))?; } else { - dispatch!(level, simd => decode_cleanup_and_magnitude_sign_to_f32_rows::( + let conversion = FinalOutput:: { + coefficient_shift, + midpoint_scale, + dequantization_step: step, + }; + dispatch!(level, simd => decode_cleanup_and_magnitude_sign_direct_rows( simd, coded, lcup, scup, flat, flat_vlc, &mut scratch, rows, &mut v_n, missing_msbs, width, height, stride, sstr, conversion, ))?; @@ -1074,6 +1138,34 @@ mod tests { Some(output.iter().map(|sample| sample.to_bits()).collect()) } + /// The direct rows' sign-magnitude words (stride `width`), as the + /// refinement-bearing blocks decode their cleanup pass. + fn direct_sign_magnitude_rows( + coded: &[u8], + missing_msbs: u32, + width: u32, + height: u32, + level: Level, + ) -> Option> { + let lcup = coded.len(); + let scup = cleanup_segment_suffix_length(coded, lcup)?; + let samples = (width * height) as usize; + let sstr = cleanup_symbol_stride(width); + let mut buffer = + Vec::with_capacity(flat_magsgn_capacity(samples).expect("MagSgn capacity")); + let flat = unstuff_magsgn(&coded[..lcup - scup], &mut buffer, samples * 4)?; + let mut vlc_buffer = Vec::with_capacity(flat_vlc_capacity(width, height)); + let flat_vlc = unstuff_vlc(coded, lcup, scup, &mut vlc_buffer)?; + let mut scratch = vec![0xA5A5; sstr * (height.div_ceil(2) as usize + 1)]; + let mut v_n = vec![0; width.div_ceil(2) as usize + 2]; + let mut decoded = vec![0xDEAD_BEEF; samples]; + dispatch!(level, simd => decode_cleanup_and_magnitude_sign_direct_rows( + simd, coded, lcup, scup, flat, flat_vlc, &mut scratch, &mut decoded, &mut v_n, missing_msbs, + width, height, width, sstr, SignMagnitude, + ))?; + Some(decoded) + } + /// `coefficients` (`width` apart) converted as the per-row decode /// converts them, laid out as [`direct_rows`] returns them. fn converted_rows( @@ -1143,6 +1235,13 @@ mod tests { None, "{width}x{height} missing_msbs={missing_msbs} coded={coded:02x?}" ); + if width.is_multiple_of(4) { + assert_eq!( + direct_sign_magnitude_rows(coded, missing_msbs, width, height, level), + None, + "direct {width}x{height} missing_msbs={missing_msbs} coded={coded:02x?}" + ); + } } assert_direct_rows_match(coded, missing_msbs, coded_bitplanes, (width, height), None); return false; @@ -1175,6 +1274,13 @@ mod tests { expected, "fused {width}x{height} missing_msbs={missing_msbs} coded={coded:02x?}" ); + if width.is_multiple_of(4) { + assert_eq!( + direct_sign_magnitude_rows(coded, missing_msbs, width, height, level), + expected, + "direct {width}x{height} missing_msbs={missing_msbs} coded={coded:02x?}" + ); + } } expected.is_some() } diff --git a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs index 663efa830..d8a9aa6aa 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs @@ -3,7 +3,10 @@ use super::cleanup::{cleanup_segment_suffix_length, cleanup_symbol_stride}; use super::magnitude::decode_cleanup_and_magnitude_sign_phase; #[cfg(feature = "simd")] -use super::magnitude::decode_cleanup_and_magnitude_sign_to_f32; +use super::magnitude::{ + decode_cleanup_and_magnitude_sign_direct, decode_cleanup_and_magnitude_sign_to_f32, + supports_cleanup_to_f32, +}; use super::refinement::apply_magnitude_refinement_phase; use super::significance::{ apply_significance_propagation_phase, build_sigma_from_cleanup_phase, sigma_stride, @@ -76,23 +79,47 @@ pub(super) fn decode_impl( // Refinement-bearing blocks decode their cleanup pass the same way; they // only keep a zeroed symbol scratch, whose rows past the block the sigma - // build reads. - decode_cleanup_and_magnitude_sign_phase( - cleanup_data, - lcup, - scup, - decoded_data, - missing_msbs, - width, - height, - stride, - sstr, - scratch, - v_n_scratch, - &mut scratch_buffers.magsgn, - &mut scratch_buffers.vlc, - observer, - )?; + // build reads. Blocks the direct rows take (unless the phases are timed + // apart) use them, publishing sign-magnitude words. + #[cfg(feature = "simd")] + let direct = !observer.times_phases() && supports_cleanup_to_f32(missing_msbs, width); + #[cfg(not(feature = "simd"))] + let direct = false; + if direct { + #[cfg(feature = "simd")] + decode_cleanup_and_magnitude_sign_direct( + cleanup_data, + lcup, + scup, + decoded_data, + missing_msbs, + width, + height, + stride, + sstr, + scratch, + v_n_scratch, + &mut scratch_buffers.magsgn, + &mut scratch_buffers.vlc, + )?; + } else { + decode_cleanup_and_magnitude_sign_phase( + cleanup_data, + lcup, + scup, + decoded_data, + missing_msbs, + width, + height, + stride, + sstr, + scratch, + v_n_scratch, + &mut scratch_buffers.magsgn, + &mut scratch_buffers.vlc, + observer, + )?; + } if PHASE_LIMIT == PHASE_LIMIT_CLEANUP { return Some(()); From 55a9c0834f5169c5f56200c8a5d2f61e53456176 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:40:36 -0400 Subject: [PATCH 28/53] perf(j2k-native): refine whole HT MagRef groups in vector rows MagRef walked the set bits of each 8x4 group's significance word, one sample and one refinement bit at a time. In the refinement-bearing blocks of rate-controlled HT tiles about two thirds of a group's samples are significant, so the walk ran some twenty serial steps per group and ended on a hard-to-predict branch. A group that lies entirely inside the block is now refined one row of eight samples per vector: each significant sample's refinement bit sits at its rank among the group's significant samples in bit order (an exclusive prefix sum of the column counts plus the rows above it), and the update is masked to the significant samples. Groups on the block edge keep the per-sample walk. A test checks the phase against the per-sample scan on random blocks, strides and densities. On a 10-core ARM64 machine, interleaved against the base and including the earlier commits, 512x512 and 256x256 HT tiles encoded at 2 bpp decode 16.8% and 16.3% faster at 1 thread. The 4096x2048 RGB HT lossless decode stays 10.6% faster at 1 thread and HT 2 bpp 4.2%; 1024x1024 regions of an 18000x12000 HT image 2.2 to 2.4% at 1 thread and unchanged at 10; 8 concurrent region decodes unchanged within noise. Output is unchanged. --- .../src/j2c/ht_block_decode/refinement.rs | 263 ++++++++++++++++-- 1 file changed, 243 insertions(+), 20 deletions(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs b/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs index 064805a06..2b8bf8d6f 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs @@ -1,6 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +#[cfg(feature = "simd")] +use fearless_simd::{u32x4, u32x8, Simd, SimdBase}; + use super::readers::{read_u32_pair, ReverseBitReader}; +#[cfg(feature = "simd")] +use crate::math::{dispatch, Level}; #[expect( clippy::inline_always, @@ -21,13 +26,55 @@ pub(super) fn apply_magnitude_refinement_phase( if p < 2 { return None; } + let geometry = (width, height, stride as usize, mstr); + + #[cfg(feature = "simd")] + { + dispatch!(Level::new(), simd => refine_stripes( + refinement_data, + sigma, + decoded_data, + geometry, + p, + |data: &mut [u32], base, sig, cwd| refine_whole_group(simd, data, base, stride as usize, sig, cwd, p), + )) + } + #[cfg(not(feature = "simd"))] + { + refine_stripes( + refinement_data, + sigma, + decoded_data, + geometry, + p, + |data: &mut [u32], base, sig, cwd| { + refine_group_bits(data, base, stride as usize, sig, u32::MAX, cwd, p); + Some(()) + }, + ) + } +} +/// The MagRef scan over 8x4 groups of the block: `whole_group` refines a +/// group that lies entirely inside the block, the rest go sample by sample. +#[expect( + clippy::inline_always, + reason = "inlined into the dispatched SIMD level" +)] +#[inline(always)] +fn refine_stripes( + refinement_data: &[u8], + sigma: &[u16], + decoded_data: &mut [u32], + (width, height, stride, mstr): (u32, u32, usize, usize), + p: u32, + mut whole_group: impl FnMut(&mut [u32], usize, u32, u32) -> Option<()>, +) -> Option<()> { let mut magref = ReverseBitReader::new_mrp(refinement_data); - let half = 1u32 << (p - 2); for y in (0..height).step_by(4) { let mut cur_sig_idx = (y >> 2) as usize * mstr; - let dpp = (y * stride) as usize; + let dpp = y as usize * stride; // One bit per row of this stripe that lies inside the code block. let rows_in_block = (1u32 << (height - y).min(4)) - 1; @@ -35,26 +82,20 @@ pub(super) fn apply_magnitude_refinement_phase( let cwd = magref.fetch(); let sig = read_u32_pair(sigma, cur_sig_idx); cur_sig_idx += 2; - // The cleanup pass can mark samples of a quad that lies across - // the block edge as significant when the codestream is malformed. - // Those samples still consume their refinement bits, but, as in - // the cleanup pass, nothing is written for them. - let in_block = - (0x1111_1111u32 * rows_in_block) & (u32::MAX >> (32 - 4 * (width - i).min(8))); - // Significant samples take refinement bits in bit order: column - // by column, top to bottom (bit `4 * column + row`). let base = dpp + i as usize; - let mut pending = sig; - let mut cwd_mut = cwd; - while pending != 0 { - let bit = pending.trailing_zeros(); - pending &= pending - 1; - if in_block & (1 << bit) != 0 { - let sym = ((1 - (cwd_mut & 1)) << (p - 1)) | half; - decoded_data - [base + (bit >> 2) as usize + (bit & 3) as usize * stride as usize] ^= sym; + if sig != 0 { + if rows_in_block == 0xF && i + 8 <= width { + whole_group(decoded_data, base, sig, cwd)?; + } else { + // The cleanup pass can mark samples of a quad that lies + // across the block edge as significant when the + // codestream is malformed. Those samples still consume + // their refinement bits, but, as in the cleanup pass, + // nothing is written for them. + let in_block = (0x1111_1111u32 * rows_in_block) + & (u32::MAX >> (32 - 4 * (width - i).min(8))); + refine_group_bits(decoded_data, base, stride, sig, in_block, cwd, p); } - cwd_mut >>= 1; } magref.advance(sig.count_ones()); @@ -63,3 +104,185 @@ pub(super) fn apply_magnitude_refinement_phase( Some(()) } + +/// Refine one 8x4 group sample by sample. Significant samples take +/// refinement bits in bit order: column by column, top to bottom (bit +/// `4 * column + row`); only those in `in_block` are written. +#[expect(clippy::inline_always, reason = "per-group hot path")] +#[inline(always)] +fn refine_group_bits( + decoded_data: &mut [u32], + base: usize, + stride: usize, + sig: u32, + in_block: u32, + cwd: u32, + p: u32, +) { + let half = 1u32 << (p - 2); + let mut pending = sig; + let mut cwd_mut = cwd; + while pending != 0 { + let bit = pending.trailing_zeros(); + pending &= pending - 1; + if in_block & (1 << bit) != 0 { + let sym = ((1 - (cwd_mut & 1)) << (p - 1)) | half; + decoded_data[base + (bit >> 2) as usize + (bit & 3) as usize * stride] ^= sym; + } + cwd_mut >>= 1; + } +} + +/// [`refine_group_bits`] for a group inside the block, one row of eight +/// samples per vector and no per-sample branches: each sample's refinement +/// bit sits at its rank among the group's significant samples in bit order. +#[cfg(feature = "simd")] +#[expect(clippy::inline_always, reason = "per-group hot path")] +#[inline(always)] +fn refine_whole_group( + simd: S, + decoded_data: &mut [u32], + base: usize, + stride: usize, + sig: u32, + cwd: u32, + p: u32, +) -> Option<()> { + let per_column = |values: [u32; 8]| u32x8::from_slice(simd, &values); + let zero = u32x8::splat(simd, 0); + // Column `c`'s four significance bits (rows 0-3) in lane `c`. + let nibbles = (u32x8::splat(simd, sig) >> per_column([0, 4, 8, 12, 16, 20, 24, 28])) & 0xF; + let flags = [0, 1, 2, 3].map(|row| (nibbles >> row) & 1); + // Bits taken by the columns before each column: an exclusive prefix sum + // of the column counts, carried from the low four lanes to the high four. + let counts = flags[0] + flags[1] + flags[2] + flags[3]; + let pairs = counts + zero.slide_within_blocks::<3>(counts); + let inclusive = pairs + zero.slide_within_blocks::<2>(pairs); + let (low, high) = simd.split_u32x8(inclusive); + let high = high + u32x4::splat(simd, low.to_array()[3]); + let mut rank = simd.combine_u32x4(low, high) - counts; + + let bits = u32x8::splat(simd, cwd); + // The symbol of a zero refinement bit; a one clears bit `p - 1`. + let zero_bit_symbol = (1u32 << (p - 1)) | (1u32 << (p - 2)); + for (row, flag) in flags.into_iter().enumerate() { + let refinement = (bits >> rank) & 1; + let symbol = + (u32x8::splat(simd, zero_bit_symbol) - (refinement << (p - 1))) & (zero - flag); + let at = base + row * stride; + let samples = decoded_data.get_mut(at..at + 8)?; + (u32x8::from_slice(simd, samples) ^ symbol).store_slice(samples); + rank += flag; + } + Some(()) +} + +#[cfg(test)] +mod tests { + use alloc::vec::Vec; + + use super::apply_magnitude_refinement_phase; + use crate::j2c::ht_block_decode::significance::sigma_stride; + + /// The MagRef scan as first written: eight columns by four rows of + /// branches per group. + fn reference( + refinement: &[u8], + sigma: &[u16], + decoded: &mut [u32], + (width, height, stride, mstr): (u32, u32, u32, usize), + p: u32, + ) { + let mut magref = super::ReverseBitReader::new_mrp(refinement); + let half = 1u32 << (p - 2); + for y in (0..height).step_by(4) { + let mut cur = (y >> 2) as usize * mstr; + let dpp = (y * stride) as usize; + let rows_in_block = (1u32 << (height - y).min(4)) - 1; + for i in (0..width).step_by(8) { + let mut cwd = magref.fetch(); + let sig = super::read_u32_pair(sigma, cur); + cur += 2; + let in_block = + (0x1111_1111u32 * rows_in_block) & (u32::MAX >> (32 - 4 * (width - i).min(8))); + for j in 0..8 { + for row in 0..4 { + let bit = 4 * j + row; + if sig & (1 << bit) != 0 { + if in_block & (1 << bit) != 0 { + let sym = ((1 - (cwd & 1)) << (p - 1)) | half; + decoded[dpp + i as usize + j as usize + (row * stride) as usize] ^= + sym; + } + cwd >>= 1; + } + } + } + magref.advance(sig.count_ones()); + } + } + } + + #[test] + fn magnitude_refinement_matches_the_per_sample_scan() { + let mut state = 0x2545_F491_4F6C_DD1D_u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for (width, height, stride) in [ + (64_u32, 64_u32, 64_u32), + (64, 64, 70), + (24, 16, 24), + (13, 7, 16), + (8, 4, 8), + (5, 9, 5), + ] { + for case in 0..40 { + let mstr = sigma_stride(width); + let sigma_rows = height.div_ceil(4) as usize + 1; + let density = case % 4; + let sigma: Vec = (0..sigma_rows * mstr) + .map(|_| { + let word = next(); + let mut bits = word as u16; + for shift in 0..density { + bits &= (word >> (16 * (shift + 1))) as u16; + } + bits + }) + .collect(); + let refinement: Vec = (0..4096).map(|_| next() as u8).collect(); + let len = ((height - 1) * stride + width) as usize; + let original: Vec = (0..len).map(|_| (next() as u32) & 0x7FFF_FFF0).collect(); + let p = 5 + (case % 20) as u32; + let mut expected = original.clone(); + reference( + &refinement, + &sigma, + &mut expected, + (width, height, stride, mstr), + p, + ); + let mut decoded = original.clone(); + apply_magnitude_refinement_phase( + &refinement, + &sigma, + &mut decoded, + width, + height, + stride, + mstr, + p, + ) + .expect("magref"); + assert_eq!( + decoded, expected, + "{width}x{height} stride {stride} case {case}" + ); + } + } + } +} From 76b5654da20593666c130efa9dbb8694c39a518f Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:48:09 -0400 Subject: [PATCH 29/53] perf(j2k-native): write HT SigProp signs of whole groups in vector rows After a SigProp group's significance bits, its newly significant samples take one sign bit each, in bit order. The loop over the new samples ran two or three times per group and ended on a branch the predictor missed about once per group; it was the hottest point of the SigProp scan in rate-controlled HT tiles. A group that lies entirely inside the block now writes its new samples one row of four per vector: each new sample's sign bit sits at its rank among the group's new samples (an exclusive prefix sum of the column counts plus the rows above it), and the other samples keep their value. Groups on the block edge keep the per-sample loop. A test checks the phase against the original candidate scan and sign loop on random blocks, strides, densities and stripe-causal settings. On a 10-core ARM64 machine, interleaved against the base and including the earlier commits, 512x512 and 256x256 HT tiles encoded at 2 bpp decode 18.6% and 17.7% faster at 1 thread. The 4096x2048 RGB HT lossless decode stays 10.5% faster at 1 thread and HT 2 bpp 4.3%; 1024x1024 regions of an 18000x12000 HT image 2.3% at 1 thread and unchanged at 10; 8 concurrent region decodes unchanged within noise. Output is unchanged. --- .../src/j2c/ht_block_decode/significance.rs | 371 +++++++++++++++++- 1 file changed, 356 insertions(+), 15 deletions(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/significance.rs b/crates/j2k-native/src/j2c/ht_block_decode/significance.rs index 80fa12f64..d5cd1daa2 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/significance.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/significance.rs @@ -1,6 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +#[cfg(feature = "simd")] +use fearless_simd::{u32x4, Simd, SimdBase}; + use super::readers::{read_u32_pair, ForwardBitReader}; +#[cfg(feature = "simd")] +use crate::math::{dispatch, Level}; /// The samples each newly significant sample of a 4x4 group makes a SigProp /// candidate, as bits `4 * column + row`: itself, the sample below it, and @@ -142,12 +147,9 @@ pub(super) fn build_sigma_from_cleanup_phase( } #[expect( - clippy::cast_possible_truncation, - clippy::cast_possible_wrap, - clippy::cast_sign_loss, clippy::inline_always, clippy::too_many_arguments, - reason = "the hot significance scan uses bounded block coordinates and stores the low packed mask half" + reason = "the significance scan is a hot stable phase boundary with explicit geometry" )] #[inline(always)] pub(super) fn apply_significance_propagation_phase( @@ -161,6 +163,123 @@ pub(super) fn apply_significance_propagation_phase( stripe_causal: bool, p: u32, prev_row_sig: &mut [u16], +) -> Option<()> { + let geometry = (width, height, stride, mstr); + #[cfg(feature = "simd")] + { + dispatch!(Level::new(), simd => propagate_stripes( + refinement_data, + sigma, + decoded_data, + geometry, + stripe_causal, + p, + prev_row_sig, + |data: &mut [u32], base, new_sig, cwd, value| { + write_group_signs(simd, data, base, stride as usize, new_sig, cwd, value) + }, + )) + } + #[cfg(not(feature = "simd"))] + { + propagate_stripes( + refinement_data, + sigma, + decoded_data, + geometry, + stripe_causal, + p, + prev_row_sig, + |data: &mut [u32], base, new_sig, cwd, value| { + write_signs_by_bit(data, base, stride as usize, new_sig, cwd, value); + Some(()) + }, + ) + } +} + +/// The new samples of a SigProp group with their sign bits, in bit order +/// (`4 * column + row`), one sample at a time. +#[expect(clippy::inline_always, reason = "per-group hot path")] +#[inline(always)] +fn write_signs_by_bit( + decoded_data: &mut [u32], + base: usize, + stride: usize, + new_sig: u32, + cwd: u32, + value: u32, +) { + let mut sign_bits = new_sig; + let mut cwd = cwd; + while sign_bits != 0 { + let bit = sign_bits.trailing_zeros(); + sign_bits &= sign_bits - 1; + decoded_data[base + (bit >> 2) as usize + (bit & 3) as usize * stride] = + ((cwd & 1) << 31) | value; + cwd >>= 1; + } +} + +/// [`write_signs_by_bit`] for a group inside the block, one row of four +/// samples per vector: each new sample's sign bit sits at its rank among the +/// group's new samples in bit order, and the other samples keep their value. +/// The per-sample loop ends on a hard-to-predict branch; this has none. +#[cfg(feature = "simd")] +#[expect(clippy::inline_always, reason = "per-group hot path")] +#[inline(always)] +fn write_group_signs( + simd: S, + decoded_data: &mut [u32], + base: usize, + stride: usize, + new_sig: u32, + cwd: u32, + value: u32, +) -> Option<()> { + let zero = u32x4::splat(simd, 0); + // Column `c`'s four new-sample bits (rows 0-3) in lane `c`. + let nibbles = (u32x4::splat(simd, new_sig) >> u32x4::from_slice(simd, &[0, 4, 8, 12])) & 0xF; + let flags = [0, 1, 2, 3].map(|row| (nibbles >> row) & 1); + // Sign bits taken by the columns before each column. + let counts = flags[0] + flags[1] + flags[2] + flags[3]; + let pairs = counts + zero.slide_within_blocks::<3>(counts); + let inclusive = pairs + zero.slide_within_blocks::<2>(pairs); + let mut rank = inclusive - counts; + let bits = u32x4::splat(simd, cwd); + for (row, flag) in flags.into_iter().enumerate() { + let new_value = (((bits >> rank) & 1) << 31) | value; + let keep = flag - 1; + let at = base + row * stride; + let samples = decoded_data.get_mut(at..at + 4)?; + let old = u32x4::from_slice(simd, samples); + ((old & keep) | (new_value & !keep)).store_slice(samples); + rank += flag; + } + Some(()) +} + +/// The SigProp scan over 4x4 groups: `whole_group_signs` writes the new +/// samples of a group that lies entirely inside the block, given the group's +/// new samples, the bits after its significance bits, and the sample value. +#[expect( + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_sign_loss, + clippy::inline_always, + clippy::too_many_arguments, + reason = "the hot significance scan uses bounded block coordinates and stores the low packed mask half" +)] +#[inline(always)] +fn propagate_stripes( + refinement_data: &[u8], + sigma: &[u16], + decoded_data: &mut [u32], + (width, height, stride, mstr): (u32, u32, u32, usize), + stripe_causal: bool, + p: u32, + prev_row_sig: &mut [u16], + mut whole_group_signs: impl FnMut(&mut [u32], usize, u32, u32, u32) -> Option<()>, ) -> Option<()> { if prev_row_sig.len() < width.div_ceil(4) as usize + 8 { return None; @@ -223,18 +342,19 @@ pub(super) fn apply_significance_propagation_phase( if new_sig != 0 { let value = 3u32 << (p - 2); let block_base = dpp + x as usize; - let mut sign_bits = new_sig; - - while sign_bits != 0 { - let bit = sign_bits.trailing_zeros(); - let sample_mask = 1u32 << bit; - sign_bits &= !sample_mask; - - let offset = (bit >> 2) as usize + ((bit & 3) as usize * stride_us); - decoded_data[block_base + offset] = ((cwd & 1) << 31) | value; - cwd >>= 1; - cnt += 1; + if x + 4 <= width && y + 4 <= height { + whole_group_signs(decoded_data, block_base, new_sig, cwd, value)?; + } else { + write_signs_by_bit( + decoded_data, + block_base, + stride_us, + new_sig, + cwd, + value, + ); } + cnt += new_sig.count_ones(); } sigprop.advance(cnt); @@ -253,3 +373,224 @@ pub(super) fn apply_significance_propagation_phase( Some(()) } + +#[cfg(test)] +mod tests { + use alloc::{vec, vec::Vec}; + + use super::{apply_significance_propagation_phase, sigma_stride}; + use crate::j2c::ht_block_decode::readers::{read_u32_pair, ForwardBitReader}; + + /// The SigProp phase as first written (candidate scan and per-sample + /// sign loop), the reference for the column table and vector signs. + #[expect( + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_sign_loss, + clippy::too_many_arguments, + reason = "the test reference keeps the phase's original scalar arithmetic and signature" + )] + fn reference( + refinement_data: &[u8], + sigma: &[u16], + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, + mstr: usize, + stripe_causal: bool, + p: u32, + prev_row_sig: &mut [u16], + ) -> Option<()> { + if prev_row_sig.len() < width.div_ceil(4) as usize + 8 { + return None; + } + + prev_row_sig.fill(0); + let mut sigprop = ForwardBitReader::<0>::new(refinement_data); + let stride_us = stride as usize; + + for y in (0..height).step_by(4) { + let mut pattern = 0xFFFFu32; + if height - y < 4 { + pattern = 0x7777; + if height - y < 3 { + pattern = 0x3333; + if height - y < 2 { + pattern = 0x1111; + } + } + } + + let mut prev = 0u32; + let cur_row = (y >> 2) as usize * mstr; + let next_row = cur_row + mstr; + let dpp = (y * stride) as usize; + + for x in (0..width).step_by(4) { + let mut col_pattern = pattern; + let mut s = x as i32 + 4 - width as i32; + s = s.max(0); + col_pattern >>= (s * 4) as u32; + + let idx = (x >> 2) as usize; + let ps = u32::from(prev_row_sig[idx]) | (u32::from(prev_row_sig[idx + 1]) << 16); + let ns = read_u32_pair(sigma, next_row + idx); + let mut u = (ps & 0x8888_8888) >> 3; + if !stripe_causal { + u |= (ns & 0x1111_1111) << 3; + } + + let cs = read_u32_pair(sigma, cur_row + idx); + let mut mbr = cs; + mbr |= (cs & 0x7777_7777) << 1; + mbr |= (cs & 0xEEEE_EEEE) >> 1; + mbr |= u; + let t = mbr; + mbr |= t << 4; + mbr |= t >> 4; + mbr |= prev >> 12; + mbr &= col_pattern; + mbr &= !cs; + + let mut new_sig = 0u32; + if mbr != 0 { + let mut cwd = sigprop.fetch(); + let mut cnt = 0u32; + let inv_sig = !cs & col_pattern; + let mut candidates = mbr; + let mut processed = 0u32; + + while candidates != 0 { + let bit = candidates.trailing_zeros(); + let sample_mask = 1u32 << bit; + candidates &= !sample_mask; + processed |= sample_mask; + + if (cwd & 1) != 0 { + new_sig |= sample_mask; + candidates |= + super::SIGPROP_SPREAD_MASKS[bit as usize] & inv_sig & !processed; + } + cwd >>= 1; + cnt += 1; + } + + if new_sig != 0 { + let value = 3u32 << (p - 2); + let block_base = dpp + x as usize; + let mut sign_bits = new_sig; + + while sign_bits != 0 { + let bit = sign_bits.trailing_zeros(); + let sample_mask = 1u32 << bit; + sign_bits &= !sample_mask; + + let offset = (bit >> 2) as usize + ((bit & 3) as usize * stride_us); + decoded_data[block_base + offset] = ((cwd & 1) << 31) | value; + cwd >>= 1; + cnt += 1; + } + } + + sigprop.advance(cnt); + } + + let combined_sig = new_sig | (cs & 0xFFFF); + prev_row_sig[idx] = combined_sig as u16; + + let t = combined_sig; + let mut next_prev = combined_sig; + next_prev |= (t & 0x7777) << 1; + next_prev |= (t & 0xEEEE) >> 1; + prev = (next_prev | u) & 0xF000; + } + } + + Some(()) + } + + #[test] + fn significance_propagation_matches_the_candidate_scan() { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for (width, height, stride) in [ + (64_u32, 64_u32, 64_u32), + (64, 64, 70), + (24, 16, 24), + (13, 7, 16), + (4, 4, 4), + (5, 9, 5), + ] { + for case in 0..60 { + let mstr = sigma_stride(width); + let sigma_rows = height.div_ceil(4) as usize + 1; + let density = case % 4; + let mut sigma: Vec = (0..sigma_rows * mstr) + .map(|_| { + let word = next(); + let mut bits = word as u16; + for shift in 0..density { + bits &= (word >> (16 * (shift + 1))) as u16; + } + bits + }) + .collect(); + // As the sigma build leaves them: nothing past the block. + let groups = width.div_ceil(4) as usize; + for row in 0..sigma_rows { + for group in groups..mstr { + sigma[row * mstr + group] = 0; + } + } + for group in 0..mstr { + sigma[(sigma_rows - 1) * mstr + group] = 0; + } + let refinement: Vec = (0..4096).map(|_| next() as u8).collect(); + let len = ((height - 1) * stride + width) as usize; + let original: Vec = (0..len).map(|_| (next() as u32) & 0x7FFF_FFF0).collect(); + let p = 5 + (case % 20) as u32; + let stripe_causal = case % 3 == 0; + let mut expected = original.clone(); + let mut prev = vec![0u16; groups + 8]; + reference( + &refinement, + &sigma, + &mut expected, + width, + height, + stride, + mstr, + stripe_causal, + p, + &mut prev, + ) + .expect("reference sigprop"); + let mut decoded = original.clone(); + let mut prev = vec![0u16; groups + 8]; + apply_significance_propagation_phase( + &refinement, + &sigma, + &mut decoded, + width, + height, + stride, + mstr, + stripe_causal, + p, + &mut prev, + ) + .expect("sigprop"); + assert_eq!( + decoded, expected, + "{width}x{height} stride {stride} case {case}" + ); + } + } + } +} From bae8d4701af1a65d5685b8b8e3e7596f81e05d84 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 03:10:40 -0400 Subject: [PATCH 30/53] perf(j2k-native): fold HT SigProp spread and sign counts into the column table The SigProp column table gave each column's new samples and bits used; the scan then derived the candidates they add to the next column with three shifts and ORs, and counted the group's sign bits with a popcount after its last column. Both sat on the serial chain from one column's lookup to the next and from one group's bits to the next group's. Each entry now also holds that spread, already in the place where the next column's index takes its candidates, and the count of new samples; the bits used sit in the entry's low bits so the entry itself is the shift past them. A column feeds the next through a mask, and the group's bit count is a sum of entry fields. The existing test checks the table against the candidate-by-candidate scan, now with the sign count. On a 10-core ARM64 machine, interleaved against the base and including the earlier commits, 512x512 and 256x256 HT tiles encoded at 2 bpp decode 20.1% and 19.6% faster at 1 thread (1.8% from this change). The 4096x2048 RGB HT lossless decode stays 10.5% faster at 1 thread and HT 2 bpp 4.2%; 1024x1024 regions of an 18000x12000 HT image 2.4 to 2.6% at 1 thread and unchanged at 10; 8 concurrent region decodes unchanged within noise. Output is unchanged. --- .../src/j2c/ht_block_decode/pipeline.rs | 1 + .../src/j2c/ht_block_decode/refinement.rs | 12 ++- .../src/j2c/ht_block_decode/significance.rs | 97 +++++++++++-------- .../src/j2c/ht_block_decode/tests.rs | 9 +- 4 files changed, 71 insertions(+), 48 deletions(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs index d8a9aa6aa..499f37ffd 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/pipeline.rs @@ -27,6 +27,7 @@ pub(crate) const PHASE_LIMIT_MAGREF: u8 = 2; #[expect( clippy::inline_always, clippy::too_many_arguments, + clippy::too_many_lines, reason = "the monomorphized HT phase pipeline has a stable hot signature and preserves normative pass order" )] #[inline(always)] diff --git a/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs b/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs index 2b8bf8d6f..c5da03769 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/refinement.rs @@ -55,7 +55,7 @@ pub(super) fn apply_magnitude_refinement_phase( } } -/// The MagRef scan over 8x4 groups of the block: `whole_group` refines a +/// The `MagRef` scan over 8x4 groups of the block: `whole_group` refines a /// group that lies entirely inside the block, the rest go sample by sample. #[expect( clippy::inline_always, @@ -184,7 +184,7 @@ mod tests { use super::apply_magnitude_refinement_phase; use crate::j2c::ht_block_decode::significance::sigma_stride; - /// The MagRef scan as first written: eight columns by four rows of + /// The `MagRef` scan as first written: eight columns by four rows of /// branches per group. fn reference( refinement: &[u8], @@ -224,6 +224,10 @@ mod tests { } #[test] + #[expect( + clippy::cast_possible_truncation, + reason = "the test draws its random fields from the low bits of 64-bit generator words" + )] fn magnitude_refinement_matches_the_per_sample_scan() { let mut state = 0x2545_F491_4F6C_DD1D_u64; let mut next = move || { @@ -240,7 +244,7 @@ mod tests { (8, 4, 8), (5, 9, 5), ] { - for case in 0..40 { + for case in 0..40_u32 { let mstr = sigma_stride(width); let sigma_rows = height.div_ceil(4) as usize + 1; let density = case % 4; @@ -257,7 +261,7 @@ mod tests { let refinement: Vec = (0..4096).map(|_| next() as u8).collect(); let len = ((height - 1) * stride + width) as usize; let original: Vec = (0..len).map(|_| (next() as u32) & 0x7FFF_FFF0).collect(); - let p = 5 + (case % 20) as u32; + let p = 5 + case % 20; let mut expected = original.clone(); reference( &refinement, diff --git a/crates/j2k-native/src/j2c/ht_block_decode/significance.rs b/crates/j2k-native/src/j2c/ht_block_decode/significance.rs index d5cd1daa2..4004153be 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/significance.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/significance.rs @@ -7,7 +7,7 @@ use super::readers::{read_u32_pair, ForwardBitReader}; #[cfg(feature = "simd")] use crate::math::{dispatch, Level}; -/// The samples each newly significant sample of a 4x4 group makes a SigProp +/// The samples each newly significant sample of a 4x4 group makes a `SigProp` /// candidate, as bits `4 * column + row`: itself, the sample below it, and /// the three nearest of the next column. The reference for /// [`SIGPROP_COLUMN`], and the order the encoder visits. @@ -17,27 +17,28 @@ pub(super) const SIGPROP_SPREAD_MASKS: [u32; 16] = [ 0x76000, 0xEC000, 0xC8000, ]; -/// SigProp decoding of one four-sample column of a stripe, indexed by -/// `candidates | insignificant << 4 | bits << 8`: the column's candidates, -/// its samples not yet significant, and the next four refinement bits. A +/// `SigProp` decoding of one four-sample column of a stripe, indexed by +/// `bits | insignificant << 4 | candidates << 8`: the next four refinement +/// bits, the column's samples not yet significant, and its candidates. A /// candidate takes one bit, top to bottom; a one makes it significant and /// the insignificant sample below it a candidate. Each entry is -/// `significant | bits_used << 4`. -const SIGPROP_COLUMN: [u8; 4096] = sigprop_column_table(); - -#[expect( - clippy::cast_possible_truncation, - reason = "entries hold four significance bits and a bit count up to four" -)] -const fn sigprop_column_table() -> [u8; 4096] { +/// `used | ones << 5 | spread << 8 | significant << 12`: the bits used (bits +/// 3-4 stay clear, so the entry is also the shift past them), the new +/// samples' count, the candidates they add to the next column (beside and +/// diagonally next to them), and the new samples. `spread` sits where the +/// next column's index takes its candidates, so one lookup feeds the next +/// through a mask. +const SIGPROP_COLUMN: [u16; 4096] = sigprop_column_table(); + +const fn sigprop_column_table() -> [u16; 4096] { let mut table = [0; 4096]; let mut index = 0; while index < 4096 { - let mut candidates = index & 0xF; + let bits = index & 0xF; let insignificant = (index >> 4) & 0xF; - let bits = index >> 8; - let mut used = 0; - let mut significant = 0; + let mut candidates = index >> 8; + let mut used = 0_u16; + let mut significant = 0_u16; let mut row = 0; while row < 4 { if candidates & (1 << row) != 0 { @@ -49,45 +50,53 @@ const fn sigprop_column_table() -> [u8; 4096] { } row += 1; } - table[index] = (significant | (used << 4)) as u8; + let spread = (significant | (significant << 1) | (significant >> 1)) & 0xF; + let ones = (significant & 1) + + ((significant >> 1) & 1) + + ((significant >> 2) & 1) + + (significant >> 3); + table[index] = used | (ones << 5) | (spread << 8) | (significant << 12); index += 1; } table } -/// The significance bits of one 4x4 SigProp group: `candidates` are the +/// The significance bits of one 4x4 `SigProp` group: `candidates` are the /// samples next to a sample significant before this pass, `insignificant` /// the samples that can still become significant (bits `4 * column + row`), /// and `cwd` the group's refinement bits. Newly significant samples make /// their later neighbors candidates; the scan is column by column, top to -/// bottom. Returns the newly significant samples and the bits used. +/// bottom. Returns the newly significant samples, the significance bits +/// used, and the count of new samples (one sign bit each). #[expect(clippy::inline_always, reason = "per-group hot path")] #[inline(always)] pub(super) fn decode_group_significance( candidates: u32, insignificant: u32, cwd: u32, -) -> (u32, u32) { +) -> (u32, u32, u32) { let mut bits = cwd; - let mut used = 0; let mut significant = 0; - // Candidates the previous column's new samples add to this one: the - // samples beside them and diagonally next to them. - let mut spread = 0; + // Bits used in bits 0-4 and new samples from bit 5: each column adds at + // most four of either. + let mut sums = 0; + // The previous column's entry; its `spread` field is in index position. + let mut previous = 0; for column in 0..4 { let shift = 4 * column; let column_insignificant = (insignificant >> shift) & 0xF; - let column_candidates = ((candidates >> shift) & 0xF) | (spread & column_insignificant); - let index = column_candidates | (column_insignificant << 4) | ((bits & 0xF) << 8); + let column_candidates = (candidates >> shift) & 0xF; + let index = (previous & (column_insignificant << 8)) + | (column_candidates << 8) + | (column_insignificant << 4) + | (bits & 0xF); let entry = u32::from(SIGPROP_COLUMN[index as usize]); - let column_significant = entry & 0xF; - let column_used = entry >> 4; - bits >>= column_used; - used += column_used; - significant |= column_significant << shift; - spread = column_significant | (column_significant << 1) | (column_significant >> 1); + bits = bits.wrapping_shr(entry); + sums += entry & 0xE7; + significant |= (entry >> 12) << shift; + previous = entry; } - (significant, used) + (significant, sums & 0x1F, sums >> 5) } pub(crate) fn sigma_stride(width: u32) -> usize { @@ -198,7 +207,7 @@ pub(super) fn apply_significance_propagation_phase( } } -/// The new samples of a SigProp group with their sign bits, in bit order +/// The new samples of a `SigProp` group with their sign bits, in bit order /// (`4 * column + row`), one sample at a time. #[expect(clippy::inline_always, reason = "per-group hot path")] #[inline(always)] @@ -259,7 +268,7 @@ fn write_group_signs( Some(()) } -/// The SigProp scan over 4x4 groups: `whole_group_signs` writes the new +/// The `SigProp` scan over 4x4 groups: `whole_group_signs` writes the new /// samples of a group that lies entirely inside the block, given the group's /// new samples, the bits after its significance bits, and the sample value. #[expect( @@ -335,9 +344,10 @@ fn propagate_stripes( let mut new_sig = 0u32; if mbr != 0 { let mut cwd = sigprop.fetch(); - let (significant, mut cnt) = decode_group_significance(mbr, !cs & col_pattern, cwd); + let (significant, used, ones) = + decode_group_significance(mbr, !cs & col_pattern, cwd); new_sig = significant; - cwd >>= cnt; + cwd >>= used; if new_sig != 0 { let value = 3u32 << (p - 2); @@ -354,10 +364,9 @@ fn propagate_stripes( value, ); } - cnt += new_sig.count_ones(); } - sigprop.advance(cnt); + sigprop.advance(used + ones); } let combined_sig = new_sig | (cs & 0xFFFF); @@ -381,7 +390,7 @@ mod tests { use super::{apply_significance_propagation_phase, sigma_stride}; use crate::j2c::ht_block_decode::readers::{read_u32_pair, ForwardBitReader}; - /// The SigProp phase as first written (candidate scan and per-sample + /// The `SigProp` phase as first written (candidate scan and per-sample /// sign loop), the reference for the column table and vector signs. #[expect( clippy::cast_possible_truncation, @@ -511,6 +520,10 @@ mod tests { } #[test] + #[expect( + clippy::cast_possible_truncation, + reason = "the test draws its random fields from the low bits of 64-bit generator words" + )] fn significance_propagation_matches_the_candidate_scan() { let mut state = 0x9E37_79B9_7F4A_7C15_u64; let mut next = move || { @@ -527,7 +540,7 @@ mod tests { (4, 4, 4), (5, 9, 5), ] { - for case in 0..60 { + for case in 0..60_u32 { let mstr = sigma_stride(width); let sigma_rows = height.div_ceil(4) as usize + 1; let density = case % 4; @@ -554,7 +567,7 @@ mod tests { let refinement: Vec = (0..4096).map(|_| next() as u8).collect(); let len = ((height - 1) * stride + width) as usize; let original: Vec = (0..len).map(|_| (next() as u32) & 0x7FFF_FFF0).collect(); - let p = 5 + (case % 20) as u32; + let p = 5 + case % 20; let stripe_causal = case % 3 == 0; let mut expected = original.clone(); let mut prev = vec![0u16; groups + 8]; diff --git a/crates/j2k-native/src/j2c/ht_block_decode/tests.rs b/crates/j2k-native/src/j2c/ht_block_decode/tests.rs index 4827e0798..b884451eb 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/tests.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/tests.rs @@ -388,7 +388,7 @@ fn sigprop_spread_masks_follow_column_major_scan_order() { } } -/// One SigProp group decoded a candidate at a time, in scan order: the +/// One `SigProp` group decoded a candidate at a time, in scan order: the /// reference for the column-table decode. fn group_significance_by_candidate(candidates: u32, insignificant: u32, cwd: u32) -> (u32, u32) { let (mut candidates, mut cwd) = (candidates, cwd); @@ -409,6 +409,10 @@ fn group_significance_by_candidate(candidates: u32, insignificant: u32, cwd: u32 } #[test] +#[expect( + clippy::cast_possible_truncation, + reason = "the test draws its random fields from the low bits of 64-bit generator words" +)] fn sigprop_group_columns_match_candidate_scan() { let mut state = 0x9E37_79B9_7F4A_7C15_u64; let mut next = || { @@ -429,9 +433,10 @@ fn sigprop_group_columns_match_candidate_scan() { }; let candidates = (((word >> 16) as u32 & (word >> 32) as u32) | dense) & insignificant; let cwd = next() as u32; + let (significant, used) = group_significance_by_candidate(candidates, insignificant, cwd); assert_eq!( decode_group_significance(candidates, insignificant, cwd), - group_significance_by_candidate(candidates, insignificant, cwd), + (significant, used, significant.count_ones()), "candidates={candidates:#06x} insignificant={insignificant:#06x} cwd={cwd:#010x}" ); } From b2a4f03e3b59b3f8b958b78e57d3e14c055610f0 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:02:22 -0400 Subject: [PATCH 31/53] perf(j2k-native): lift consecutive vertical wavefronts in one column sweep The streamed IDWT ran each vertical lifting step of an arrival as its own sweep over a row and its two neighbours: three loads and a store per sample per step, four steps per row pair for 9-7I. Every interior arrival whose row above has the first step's parity runs all of its steps, one row further up each, so those steps now run together: - `lift_arrivals` drives the vertical steps of a batch of arrivals for the whole-level, parallel-strip and fused-last-level streams alike. Arrivals near a level's or strip's edges, where a neighbour is mirrored or outside the strip, keep the one-step-at-a-time path. - Interior wavefronts go to `lift_wavefronts`, which sweeps the columns once for two wavefronts and keeps the rows they share in registers, so each row is loaded and stored about once. Every column sees the same operations in the same order, so the output is bit-identical. - The streams synthesize 16 arrivals ahead of their vertical steps; the ring and halo capacity arguments are in the comments. HT 2 bpp full-image decode (4096x2048 RGB, an 18-core ARM64 machine): -4.5% at 1 thread, -5.0% at 6, -4.9% at 18, -3.0% at the default; HT 512x512 tiles -7.1%. Output is unchanged. --- crates/j2k-native/src/j2c/idwt/banded.rs | 601 ++++++++++++++++-- .../src/j2c/idwt/banded/last_level.rs | 44 +- 2 files changed, 556 insertions(+), 89 deletions(-) diff --git a/crates/j2k-native/src/j2c/idwt/banded.rs b/crates/j2k-native/src/j2c/idwt/banded.rs index f75421148..14bdf06a2 100644 --- a/crates/j2k-native/src/j2c/idwt/banded.rs +++ b/crates/j2k-native/src/j2c/idwt/banded.rs @@ -17,6 +17,7 @@ //! bit-identical. use alloc::vec::Vec; +use core::ops::Range; use super::super::build::Decomposition; use super::super::codestream::WaveletTransform; @@ -514,25 +515,20 @@ fn stream_level( ) { let width = level.width; let height = level.height; - let (steps, count) = vertical_steps(level); + let (_, count) = vertical_steps(level); let (low, high) = split_horizontal_scratch(level, horizontal_scratch); - for arrival in 0..height + count { - if arrival < height { - let row = &mut target[arrival * width..][..width]; - synthesize_level_row(simd, level, arrival, row, low, high); - } - for (index, &(parity, step)) in steps[..count].iter().enumerate() { - let Some(row) = arrival.checked_sub(index + 1) else { - break; - }; - if row >= height || row % 2 != parity { - continue; - } - let above = periodic_symmetric_extension_left(row, 1); - let below = periodic_symmetric_extension_right(row, 1, height); - let (current, above, below) = three_rows(target, width, row, above, below); - apply_vertical_step(simd, step, current, above, below); + let mut rows = RingRows { + rows: target, + width, + stride: width, + slots: height, + }; + for first in (0..height + count).step_by(WAVEFRONT_ARRIVALS) { + let arrivals = first..(first + WAVEFRONT_ARRIVALS).min(height + count); + for arrival in arrivals.start..arrivals.end.min(height) { + synthesize_level_row(simd, level, arrival, rows.row_mut(arrival), low, high); } + lift_arrivals(simd, level, arrivals, 0..height, &mut rows); } } @@ -557,7 +553,7 @@ fn stream_strip( ) { let width = level.width; let height = level.height; - let (steps, count) = vertical_steps(level); + let (_, count) = vertical_steps(level); let end = first_row + output.len() / width; let lo = first_row.saturating_sub(count); let hi = (end + count).min(height); @@ -571,34 +567,21 @@ fn stream_strip( lo, width, }; - for arrival in lo..hi + count { - if arrival < hi { - let row = rows.row_mut(arrival); - synthesize_level_row(simd, level, arrival, row, low, high); - } - for (index, &(parity, step)) in steps[..count].iter().enumerate() { - let Some(row) = arrival.checked_sub(index + 1) else { - break; - }; - if row < lo || row >= hi || row % 2 != parity { - continue; - } - let above = periodic_symmetric_extension_left(row, 1); - let below = periodic_symmetric_extension_right(row, 1, height); - if above < lo || below >= hi { - continue; - } - let (current, above, below) = rows.three(row, above, below); - apply_vertical_step(simd, step, current, above, below); + for first in (lo..hi + count).step_by(WAVEFRONT_ARRIVALS) { + let arrivals = first..(first + WAVEFRONT_ARRIVALS).min(hi + count); + for arrival in arrivals.start..arrivals.end.min(hi) { + synthesize_level_row(simd, level, arrival, rows.row_mut(arrival), low, high); } + lift_arrivals(simd, level, arrivals, lo..hi, &mut rows); } } /// Rows `lo..` of one parallel strip: its own rows `first_row..end` in /// `output`, and its halo rows in `halo`. The rows above and below the strip /// share halo slots: every strip with rows below it spans at least -/// [`MIN_STRIP_ROWS`] rows, so the rows above have taken their last step -/// before the first row below arrives. +/// [`MIN_STRIP_ROWS`] rows, more than [`WAVEFRONT_ARRIVALS`] plus the halo, +/// so the rows above have taken their last step before the batch that +/// synthesizes the first row below. #[cfg(feature = "parallel")] struct StripRows<'a> { output: &'a mut [f32], @@ -631,8 +614,10 @@ impl StripRows<'_> { }; &mut rows[index * self.width..][..self.width] } +} - /// Row `current` mutably, with rows `above` and `below` shared. +#[cfg(feature = "parallel")] +impl LevelRows for StripRows<'_> { fn three( &mut self, current: usize, @@ -662,6 +647,348 @@ impl StripRows<'_> { }; (current_row, neighbor(above), neighbor(below)) } + + fn window(&mut self, first: usize) -> Option<[&mut [f32]; ROWS]> { + let width = self.width; + let rows = first..first + ROWS; + // The halo rows above the strip, its own rows, and the halo rows + // below it. The two halo runs share slots, so at most one of them + // may take part. + let above = rows.start..rows.end.clamp(rows.start, self.first_row.max(rows.start)); + let own = rows.start.max(self.first_row)..rows.end.min(self.end); + let below = rows.start.max(self.end)..rows.end.max(self.end); + let (above, below): (&mut [f32], &mut [f32]) = match (above.is_empty(), below.is_empty()) { + (true, true) => (&mut [], &mut []), + (false, true) => ( + self.halo + .get_mut((above.start - self.lo) * width..(above.end - self.lo) * width)?, + &mut [], + ), + (true, false) => ( + &mut [], + self.halo + .get_mut((below.start - self.end) * width..(below.end - self.end) * width)?, + ), + (false, false) => return None, + }; + let own: &mut [f32] = if own.is_empty() { + &mut [] + } else { + self.output + .get_mut((own.start - self.first_row) * width..(own.end - self.first_row) * width)? + }; + filled_window( + above + .chunks_exact_mut(width) + .chain(own.chunks_exact_mut(width)) + .chain(below.chunks_exact_mut(width)), + ) + } +} + +/// Rows of `width` samples, `stride` apart, where level row `n` occupies +/// slot `n % slots`: a whole level in place when `slots` is its height, or +/// the ring of the last level's rows. +struct RingRows<'a> { + rows: &'a mut [f32], + width: usize, + stride: usize, + slots: usize, +} + +impl RingRows<'_> { + fn row_mut(&mut self, row: usize) -> &mut [f32] { + &mut self.rows[row % self.slots * self.stride..][..self.width] + } +} + +impl LevelRows for RingRows<'_> { + fn three( + &mut self, + current: usize, + above: usize, + below: usize, + ) -> (&mut [f32], &[f32], &[f32]) { + let width = self.width; + let slot = |row: usize| row % self.slots; + let (current, above, below) = three_rows( + self.rows, + self.stride, + slot(current), + slot(above), + slot(below), + ); + (&mut current[..width], &above[..width], &below[..width]) + } + + fn window(&mut self, first: usize) -> Option<[&mut [f32]; ROWS]> { + if ROWS > self.slots { + return None; + } + let (width, stride) = (self.width, self.stride); + // The rows from the first one's slot to the end, then from slot zero. + let (wrapped, from_first) = self.rows.split_at_mut(first % self.slots * stride); + filled_window( + from_first + .chunks_mut(stride) + .chain(wrapped.chunks_mut(stride)) + .filter_map(|row| row.get_mut(..width)), + ) + } +} + +/// The first `ROWS` rows of `rows`, if it has as many. +fn filled_window<'rows, const ROWS: usize>( + mut rows: impl Iterator, +) -> Option<[&'rows mut [f32]; ROWS]> { + let mut filled = 0; + let window = core::array::from_fn(|_| match rows.next() { + Some(row) => { + filled += 1; + row + } + None => &mut [], + }); + (filled == ROWS).then_some(window) +} + +/// Rows of a level under reconstruction, by level row index. +trait LevelRows { + /// Row `current` mutably, with rows `above` and `below`, both different + /// from `current`, shared. + fn three(&mut self, current: usize, above: usize, below: usize) + -> (&mut [f32], &[f32], &[f32]); + + /// The `ROWS` consecutive level rows from `first`, mutably, or `None` + /// when they cannot all be borrowed at once. + fn window(&mut self, first: usize) -> Option<[&mut [f32]; ROWS]>; +} + +/// Arrivals [`stream_level`] and [`stream_strip`] synthesize before they run +/// those arrivals' vertical steps, so one sweep of the columns runs the +/// steps of several wavefronts. +const WAVEFRONT_ARRIVALS: usize = 16; + +/// Wavefronts one sweep of the columns runs, sharing their rows in +/// registers. +const SWEEP_WAVEFRONTS: usize = 2; + +/// Run the vertical lifting steps of `arrivals`, in order, on `rows`, of +/// which only `live` take part (see [`stream_strip`]). Every arrival must +/// already be synthesized. +/// +/// An arrival whose row above has the first step's parity runs every step, +/// one row further up each (a wavefront); the others run none. Away from +/// the edges of `live`, those steps read only unmirrored rows inside it, so +/// consecutive wavefronts run together in [`lift_wavefronts`], which gives +/// every sample the same operations in the same order. Arrivals near the +/// edges run their steps one row at a time, as before. +#[expect( + clippy::inline_always, + reason = "the vertical schedule is specialized with the streaming kernels" +)] +#[inline(always)] +fn lift_arrivals( + simd: S, + level: Level2d<'_>, + arrivals: Range, + live: Range, + rows: &mut impl LevelRows, +) { + let (steps, count) = vertical_steps(level); + let steps = &steps[..count]; + // From `lo + count + 1` the deepest step's upper neighbor lies in + // `live`; before `hi` the first step's lower neighbor does. + let start = (live.start + count + 1).clamp(arrivals.start, arrivals.end); + let end = live.end.clamp(start, arrivals.end); + for arrival in arrivals.start..start { + lift_arrival(simd, steps, arrival, level.height, &live, rows); + } + // `start` is at least one; its row above has its parity. + let mut wavefront = start + usize::from((start + 1) % 2 != steps[0].0); + while wavefront < end { + let wavefronts = if (end - wavefront).div_ceil(2) >= SWEEP_WAVEFRONTS { + SWEEP_WAVEFRONTS + } else { + 1 + }; + let first = wavefront - count - 1; + let lifted = match (level.transform, wavefronts) { + (WaveletTransform::Reversible53, 1) => lift_wavefronts::(simd, rows, first), + (WaveletTransform::Reversible53, _) => { + lift_wavefronts::(simd, rows, first) + } + (WaveletTransform::Irreversible97, 1) => lift_wavefronts::(simd, rows, first), + (WaveletTransform::Irreversible97, _) => { + lift_wavefronts::(simd, rows, first) + } + }; + if !lifted { + for arrival in (wavefront..wavefront + 2 * wavefronts).step_by(2) { + lift_arrival(simd, steps, arrival, level.height, &live, rows); + } + } + wavefront += 2 * wavefronts; + } + for arrival in end..arrivals.end { + lift_arrival(simd, steps, arrival, level.height, &live, rows); + } +} + +/// Run the vertical lifting steps of one arrival a row at a time: step `k` +/// (1-based) on row `arrival - k` when that row has the step's parity and it +/// and both its neighbors (mirrored at the level's edges) lie in `live`. +#[expect( + clippy::inline_always, + reason = "the vertical schedule is specialized with the streaming kernels" +)] +#[inline(always)] +fn lift_arrival( + simd: S, + steps: &[(usize, VerticalStep)], + arrival: usize, + height: usize, + live: &Range, + rows: &mut impl LevelRows, +) { + for (index, &(parity, step)) in steps.iter().enumerate() { + let Some(row) = arrival.checked_sub(index + 1) else { + break; + }; + if !live.contains(&row) || row % 2 != parity { + continue; + } + let above = periodic_symmetric_extension_left(row, 1); + let below = periodic_symmetric_extension_right(row, 1, height); + if above < live.start || below >= live.end { + continue; + } + let (current, above, below) = rows.three(row, above, below); + apply_vertical_step(simd, step, current, above, below); + } +} + +/// Run `(ROWS - STEPS) / 2` consecutive wavefronts of a transform with +/// `STEPS` vertical steps on the `ROWS` level rows from `first`, all in the +/// level's interior: wavefront `j` runs step `k` (0-based) on row +/// `first + STEPS + 2 * j - k` from that row's two neighbors. Each column +/// runs the wavefronts in order with their rows in registers, so every +/// sample sees the operations of [`apply_vertical_step`] in the same order +/// while each row is loaded and stored once. Returns `false`, having changed +/// nothing, when the rows cannot be borrowed together. +#[expect( + clippy::inline_always, + reason = "the lifting kernel is specialized inside each SIMD dispatch target" +)] +#[inline(always)] +fn lift_wavefronts( + simd: S, + rows: &mut impl LevelRows, + first: usize, +) -> bool { + let Some(mut window) = rows.window::(first) else { + return false; + }; + if STEPS == HALO_ROWS_53 { + lift_window::( + simd, + &mut window, + #[inline(always)] + |step, current, above, below| { + if step == 0 { + lift_53_low_simd(current, above, below) + } else { + lift_53_high_simd(current, above, below) + } + }, + #[inline(always)] + |step, current, above, below| { + if step == 0 { + lift_53_low(current, above, below) + } else { + lift_53_high(current, above, below) + } + }, + ); + } else { + let coefficients = [ + dwt::IDWT97_NEG_DELTA_F32, + dwt::IDWT97_NEG_GAMMA_F32, + dwt::IDWT97_NEG_BETA_F32, + dwt::IDWT97_NEG_ALPHA_F32, + ]; + let coefficients_simd = coefficients.map(|value| f32x8::splat(simd, value)); + lift_window::( + simd, + &mut window, + #[inline(always)] + |step, current, above, below| { + lift_97_simd(current, above, below, coefficients_simd[step]) + }, + #[inline(always)] + |step, current, above, below| lift_97(current, above, below, coefficients[step]), + ); + } + true +} + +/// [`lift_wavefronts`] on rows of equal width, `step(k, current, above, +/// below)` being step `k`. Every loop over rows has a constant trip count, +/// so the window and the row pointers stay in registers. +#[expect( + clippy::inline_always, + reason = "the lifting kernel is specialized inside each SIMD dispatch target" +)] +#[expect( + clippy::needless_range_loop, + reason = "constant-index loops over the window unroll into registers; iterator adaptors keep it in memory" +)] +#[inline(always)] +fn lift_window( + simd: S, + rows: &mut [&mut [f32]; ROWS], + step_simd: impl Fn(usize, f32x8, f32x8, f32x8) -> f32x8, + step_scalar: impl Fn(usize, f32, f32, f32) -> f32, +) { + let wavefronts = (ROWS - STEPS) / 2; + let width = rows[0].len(); + let blocks = width / SIMD_WIDTH; + // The scalar tail columns first: columns are independent. + for column in blocks * SIMD_WIDTH..width { + let mut window = [0.0; ROWS]; + for row in 0..ROWS { + window[row] = rows[row][column]; + } + for wavefront in 0..wavefronts { + for step in 0..STEPS { + let row = 2 * wavefront + STEPS - step; + window[row] = step_scalar(step, window[row], window[row - 1], window[row + 1]); + } + } + // The first and last rows are only read. + for row in 1..ROWS - 1 { + rows[row][column] = window[row]; + } + } + // Every row as exactly `blocks` SIMD blocks, so no access needs a check. + let blocks_of = rows + .each_mut() + .map(|row| &mut row.as_chunks_mut::().0[..blocks]); + for block in 0..blocks { + let mut window = [f32x8::splat(simd, 0.0); ROWS]; + for row in 0..ROWS { + window[row] = f32x8::from_slice(simd, &blocks_of[row][block]); + } + for wavefront in 0..wavefronts { + for step in 0..STEPS { + let row = 2 * wavefront + STEPS - step; + window[row] = step_simd(step, window[row], window[row - 1], window[row + 1]); + } + } + for row in 1..ROWS - 1 { + window[row].store(&mut blocks_of[row][block]); + } + } } /// Split one worker's exact horizontal scratch into low- and high-pass bands, @@ -721,26 +1048,12 @@ fn apply_vertical_step( below: &[f32], ) { match step { - VerticalStep::Reversible53Low => lift_rows( - simd, - current, - above, - below, - #[inline(always)] - |s1, s2, s3| s1 - ((s2 + s3 + 2.0) * 0.25).floor(), - #[inline(always)] - |s1, s2, s3| s1 - math::floor_f32(math::mul_add(s2 + s3, 0.25, 0.5)), - ), - VerticalStep::Reversible53High => lift_rows( - simd, - current, - above, - below, - #[inline(always)] - |s1, s2, s3| s1 + ((s2 + s3) * 0.5).floor(), - #[inline(always)] - |s1, s2, s3| s1 + math::floor_half_sum_f32(s2, s3), - ), + VerticalStep::Reversible53Low => { + lift_rows(simd, current, above, below, lift_53_low_simd, lift_53_low); + } + VerticalStep::Reversible53High => { + lift_rows(simd, current, above, below, lift_53_high_simd, lift_53_high); + } VerticalStep::Irreversible97(coefficient) => { let coefficient_simd = f32x8::splat(simd, coefficient); lift_rows( @@ -749,14 +1062,79 @@ fn apply_vertical_step( above, below, #[inline(always)] - |s1, s2, s3| (s2 + s3).mul_add(coefficient_simd, s1), + |s1, s2, s3| lift_97_simd(s1, s2, s3, coefficient_simd), #[inline(always)] - |s1, s2, s3| math::mul_add(s2 + s3, coefficient, s1), + |s1, s2, s3| lift_97(s1, s2, s3, coefficient), ); } } } +/// Equation (F-5) on one sample from its two neighbors: the SIMD form. +#[expect( + clippy::inline_always, + reason = "the lifting step is specialized with its scalar and SIMD operations" +)] +#[inline(always)] +fn lift_53_low_simd(s1: f32x8, s2: f32x8, s3: f32x8) -> f32x8 { + s1 - ((s2 + s3 + 2.0) * 0.25).floor() +} + +/// Equation (F-5): the scalar form of [`lift_53_low_simd`]. +#[expect( + clippy::inline_always, + reason = "the lifting step is specialized with its scalar and SIMD operations" +)] +#[inline(always)] +fn lift_53_low(s1: f32, s2: f32, s3: f32) -> f32 { + s1 - math::floor_f32(math::mul_add(s2 + s3, 0.25, 0.5)) +} + +/// Equation (F-6) on one sample from its two neighbors: the SIMD form. +#[expect( + clippy::inline_always, + reason = "the lifting step is specialized with its scalar and SIMD operations" +)] +#[inline(always)] +fn lift_53_high_simd(s1: f32x8, s2: f32x8, s3: f32x8) -> f32x8 { + s1 + ((s2 + s3) * 0.5).floor() +} + +/// Equation (F-6): the scalar form of [`lift_53_high_simd`]. +#[expect( + clippy::inline_always, + reason = "the lifting step is specialized with its scalar and SIMD operations" +)] +#[inline(always)] +fn lift_53_high(s1: f32, s2: f32, s3: f32) -> f32 { + s1 + math::floor_half_sum_f32(s2, s3) +} + +/// One 9-7I lifting step on one sample from its two neighbors: the SIMD form. +#[expect( + clippy::inline_always, + reason = "the lifting step is specialized with its scalar and SIMD operations" +)] +#[inline(always)] +fn lift_97_simd( + s1: f32x8, + s2: f32x8, + s3: f32x8, + coefficient: f32x8, +) -> f32x8 { + (s2 + s3).mul_add(coefficient, s1) +} + +/// One 9-7I lifting step: the scalar form of [`lift_97_simd`]. +#[expect( + clippy::inline_always, + reason = "the lifting step is specialized with its scalar and SIMD operations" +)] +#[inline(always)] +fn lift_97(s1: f32, s2: f32, s3: f32, coefficient: f32) -> f32 { + math::mul_add(s2 + s3, coefficient, s1) +} + /// Multiply every sample of `samples` by `scale`. #[expect( clippy::inline_always, @@ -1215,3 +1593,102 @@ fn scale_row(simd: S, buffer: &mut [f32], scale: f32) { *value *= scale; } } + +#[cfg(test)] +mod tests { + use alloc::vec::Vec; + + use super::*; + + /// Deterministic xorshift samples: small integers for 5/3, fractions for 9/7. + #[expect( + clippy::cast_precision_loss, + reason = "the drawn value is below 20001 and exact in f32" + )] + fn samples(count: usize, seed: u64, integral: bool) -> Vec { + let mut state = seed | 1; + (0..count) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + let value = (state % 20_001) as f32 - 10_000.0; + if integral { + value + } else { + value / 7.0 + } + }) + .collect() + } + + /// The vertical steps of `arrivals` in batches of `batch` through + /// [`lift_arrivals`] give every sample the bits of one arrival at a + /// time through [`lift_arrival`], whatever the width, height, origin + /// parity, live rows or batch. + #[test] + fn wavefront_batches_match_one_arrival_at_a_time() { + let mut seed = 7_u64; + for transform in [ + WaveletTransform::Reversible53, + WaveletTransform::Irreversible97, + ] { + let integral = transform == WaveletTransform::Reversible53; + for y0 in [0_u32, 1, 6] { + for (width, height) in [(1, 9), (7, 12), (8, 13), (9, 40), (17, 31), (33, 70)] { + let rect = IntRect::from_xywh(3, y0, width, height); + let level = Level2d::new(rect, [&[]; 4], transform, 1.25); + let (steps, count) = vertical_steps(level); + let height = height as usize; + let width = width as usize; + // The whole level, and strip-like windows with halos. + for live in [0..height, 0..height.min(11), 2..height, 3..height - 2] { + for batch in [1, 2, 3, 5, 16, 37] { + seed += 1; + let source = samples(width * height, seed, integral); + let mut expected = source.clone(); + let mut rows = RingRows { + rows: &mut expected, + width, + stride: width, + slots: height, + }; + for arrival in live.start..live.end + count { + dispatch!(Level::new(), simd => lift_arrival( + simd, + &steps[..count], + arrival, + height, + &live, + &mut rows, + )); + } + let mut actual = source; + let mut rows = RingRows { + rows: &mut actual, + width, + stride: width, + slots: height, + }; + for first in (live.start..live.end + count).step_by(batch) { + let arrivals = first..(first + batch).min(live.end + count); + dispatch!(Level::new(), simd => lift_arrivals( + simd, + level, + arrivals.clone(), + live.clone(), + &mut rows, + )); + } + assert_eq!( + actual.iter().map(|value| value.to_bits()).collect::>(), + expected.iter().map(|value| value.to_bits()).collect::>(), + "{transform:?} y0={y0} {width}x{height} live={live:?} batch={batch}" + ); + } + } + } + } + } + } +} diff --git a/crates/j2k-native/src/j2c/idwt/banded/last_level.rs b/crates/j2k-native/src/j2c/idwt/banded/last_level.rs index b39cd018b..fd0f714d3 100644 --- a/crates/j2k-native/src/j2c/idwt/banded/last_level.rs +++ b/crates/j2k-native/src/j2c/idwt/banded/last_level.rs @@ -16,10 +16,9 @@ use core::ops::Range; use super::super::super::codestream::WaveletTransform; use super::super::super::rect::IntRect; use super::{ - apply_vertical_step, horizontal_scratch_elements, packed_band_rects, parallel_strip_layout, - periodic_symmetric_extension_left, periodic_symmetric_extension_right, - split_horizontal_scratch, synthesize_level_row, three_rows, use_banded, vertical_steps, - Level2d, RING_ROWS, + horizontal_scratch_elements, lift_arrivals, packed_band_rects, parallel_strip_layout, + split_horizontal_scratch, synthesize_level_row, use_banded, vertical_steps, Level2d, RingRows, + RING_ROWS, WAVEFRONT_ARRIVALS, }; use crate::error::{bail, DecodingError}; use crate::math::{dispatch, Level, Simd}; @@ -276,7 +275,7 @@ fn stream_rows( let height = levels[0].height; let row_bytes = 3 * width; let end = first_row + output.len() / row_bytes; - let (steps, count) = vertical_steps(levels[0]); + let (_, count) = vertical_steps(levels[0]); let lo = first_row.saturating_sub(count); let hi = (end + count).min(height); let stride = width + RING_ROW_PAD; @@ -303,29 +302,20 @@ fn stream_rows( for first in (lo..hi + count).step_by(batch) { let arrivals = first..(first + batch).min(hi + count); for (level, ring) in levels.iter().zip(rings.iter_mut()) { - for arrival in arrivals.clone() { - if arrival < hi { - let row = &mut ring[slot(arrival) * stride..][..width]; - synthesize_level_row(simd, *level, arrival, row, low, high); - } - for (index, &(parity, step)) in steps[..count].iter().enumerate() { - let Some(row) = arrival.checked_sub(index + 1) else { - break; - }; - if row < lo || row >= hi || row % 2 != parity { - continue; - } - let above = periodic_symmetric_extension_left(row, 1); - let below = periodic_symmetric_extension_right(row, 1, height); - if above < lo || below >= hi { - continue; - } - let (current, above, below) = - three_rows(ring, stride, slot(row), slot(above), slot(below)); - let (current, above, below) = - (&mut current[..width], &above[..width], &below[..width]); - apply_vertical_step(simd, step, current, above, below); + let mut rows = RingRows { + rows: &mut ring[..], + width, + stride, + slots: ring_rows, + }; + // The ring holds every row the batch's steps read besides the + // batch itself, so arrivals can be synthesized ahead. + for first in arrivals.clone().step_by(WAVEFRONT_ARRIVALS) { + let ahead = first..(first + WAVEFRONT_ARRIVALS).min(arrivals.end); + for arrival in ahead.start..ahead.end.min(hi) { + synthesize_level_row(simd, *level, arrival, rows.row_mut(arrival), low, high); } + lift_arrivals(simd, *level, ahead, lo..hi, &mut rows); } } // The batch's last arrival read its rows `count + 1` above it. From 9149ee9bc67160069762665b19c8e23627d8f71e Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:36:34 -0400 Subject: [PATCH 32/53] perf(j2k-native): stream the serial last IDWT level in column strips On one thread the fused last level kept three rings of 21 full-width rows, about 1 MiB for a 4096-wide image, so its horizontal synthesis, vertical lifting and pack ran out of L2. Each batch now runs column strip by column strip: a strip's synthesis and lifting of all three components and the pack of its part of the finished rows, on rings of its own of 1024 columns, which stay in L1. - Lifting is per column, so strips are independent. A strip's horizontal 9-7I synthesis starts the register-window pass at its first block: `lift_row_97_window` now takes a first block and lifts the two blocks before it as far as that block's outputs need, with the same expressions, so the output is bit-identical. The last strip also runs the row's multi-pass tail, factored out of `synthesize_row` as `synthesize_97_tail`, before its window, as `synthesize_row` does. - Only wide even-origin 9-7I levels are split; other levels, and the parallel last level, keep full rows: on the pool the shorter row segments cost more than the strips save. - `scaled_blocks` replaces the block-load closure of the window pass, so it stays inside the dispatch's target features however often it is called. - Tests run 64-column strips, so the fused-last-level fixtures cover several strips, a remainder strip and the row tail. HT 2 bpp full-image decode (4096x2048 RGB, an 18-core ARM64 machine), on top of the previous commit: -3.2% at 1 thread; the pool paths are unchanged. Output is unchanged. --- crates/j2k-native/src/j2c/idwt/banded.rs | 371 ++++++++++++++---- .../src/j2c/idwt/banded/last_level.rs | 159 +++++--- .../j2k-native/src/tests/fused_last_level.rs | 4 +- 3 files changed, 414 insertions(+), 120 deletions(-) diff --git a/crates/j2k-native/src/j2c/idwt/banded.rs b/crates/j2k-native/src/j2c/idwt/banded.rs index 14bdf06a2..107f7ec26 100644 --- a/crates/j2k-native/src/j2c/idwt/banded.rs +++ b/crates/j2k-native/src/j2c/idwt/banded.rs @@ -491,6 +491,53 @@ fn synthesize_level_row( } } +/// [`synthesize_level_row`] restricted to `columns`, one of the level's +/// [`column_strips`], into `out`. +#[expect( + clippy::inline_always, + reason = "specialized with the streaming kernels" +)] +#[inline(always)] +fn synthesize_level_columns( + simd: S, + level: Level2d<'_>, + index: usize, + out: &mut [f32], + columns: Range, + low: &mut [f32], + high: &mut [f32], +) { + if columns.len() == level.width { + synthesize_level_row(simd, level, index, out, low, high); + return; + } + // Strips are only made of even-origin 9-7I levels (`column_strips`). + let row_scale = if index % 2 == level.y0 % 2 { + dwt::DWT97_KAPPA_F32 + } else { + level.high_pass + }; + let source = if index % 2 == level.even.start_row % 2 { + level.even.row(index) + } else { + level.odd.row(index) + }; + match source { + Some((low_source, high_source)) => synthesize_row_columns( + simd, + level, + low_source, + high_source, + out, + columns, + low, + high, + row_scale, + ), + None => out.fill(0.0), + } +} + /// Reconstruct the whole level top to bottom in `target` on one thread. /// /// When row `n` arrives it is synthesized horizontally (and scaled, for 9-7I), @@ -1194,9 +1241,9 @@ const MIN_SIMD_ROW_WIDTH: usize = 2 * SIMD_WIDTH; /// that mirrors its edge sample: that is the symmetric extension (F.3.7) of /// the interleaved row seen from the other parity. #[expect( + clippy::inline_always, clippy::too_many_arguments, - clippy::too_many_lines, - reason = "the row synthesis keeps its geometry, two sources, output, and exact scratch explicit, and its window and step passes together" + reason = "the row synthesis is specialized inside each SIMD dispatch target and keeps its geometry, two sources, output, and exact scratch explicit" )] #[inline(always)] fn synthesize_row( @@ -1209,11 +1256,6 @@ fn synthesize_row( high: &mut [f32], row_scale: f32, ) { - const NEG_ALPHA: f32 = dwt::IDWT97_NEG_ALPHA_F32; - const NEG_BETA: f32 = dwt::IDWT97_NEG_BETA_F32; - const NEG_GAMMA: f32 = dwt::IDWT97_NEG_GAMMA_F32; - const NEG_DELTA: f32 = dwt::IDWT97_NEG_DELTA_F32; - const KAPPA: f32 = dwt::DWT97_KAPPA_F32; let width = level.width; if width < MIN_SIMD_ROW_WIDTH { interleave_row(simd, first, second, row); @@ -1282,51 +1324,16 @@ fn synthesize_row( 0 }; let start = window.saturating_sub(WINDOW_HALO_PAIRS); - let low = &mut low[..low_source.len() - start + 2]; - let high = &mut high[..high_source.len() - start + 2]; - load_ghosted(low, &low_source[start..]); - load_ghosted(high, &high_source[start..]); - let tail = &mut row[2 * start..]; - // Steps 1 and 2. - scale_row(simd, low, KAPPA); - scale_row(simd, high, level.high_pass); - // Steps 3 to 5. - for (update_low, coefficient) in - [(true, NEG_DELTA), (false, NEG_GAMMA), (true, NEG_BETA)] - { - let coefficient_simd = f32x8::splat(simd, coefficient); - let (target, source, reads) = if update_low { - (&mut *low, &mut *high, low_reads) - } else { - (&mut *high, &mut *low, high_reads) - }; - lift_row( - simd, - target, - source, - reads, - #[inline(always)] - |s, a, b| (a + b).mul_add(coefficient_simd, s), - #[inline(always)] - |s, a, b| math::mul_add(a + b, coefficient, s), - ); - } - // Step 6, the interleave, and the vertical scale share their - // final loads and store. The multiply still follows the final - // lifting expression for every sample. - let coefficient_simd = f32x8::splat(simd, NEG_ALPHA); - lift_high_to_row::( + synthesize_97_tail( simd, - high, - low, - high_reads, + [low_source, high_source], + [low, high], + [low_reads, high_reads], low_first, - tail, + &mut row[2 * start..], + start, + level.high_pass, row_scale, - #[inline(always)] - |s, a, b| (a + b).mul_add(coefficient_simd, s), - #[inline(always)] - |s, a, b| math::mul_add(a + b, NEG_ALPHA, s), ); if window > 0 { let row = &mut row[..2 * window]; @@ -1335,6 +1342,7 @@ fn synthesize_row( low_source, high_source, row, + 0, level.high_pass, row_scale, ); @@ -1343,6 +1351,184 @@ fn synthesize_row( } } +/// The multi-pass 9-7I synthesis of a row's pairs from `start` on into +/// `tail`, through ghosted copies of the band rows' remaining samples. +#[expect( + clippy::too_many_arguments, + reason = "the tail keeps its two sources, two scratch bands, reads and scales explicit" +)] +#[inline(always)] +fn synthesize_97_tail( + simd: S, + [low_source, high_source]: [&[f32]; 2], + [low, high]: [&mut [f32]; 2], + [low_reads, high_reads]: [(usize, usize); 2], + low_first: bool, + tail: &mut [f32], + start: usize, + high_pass: f32, + row_scale: f32, +) { + const NEG_ALPHA: f32 = dwt::IDWT97_NEG_ALPHA_F32; + const NEG_BETA: f32 = dwt::IDWT97_NEG_BETA_F32; + const NEG_GAMMA: f32 = dwt::IDWT97_NEG_GAMMA_F32; + const NEG_DELTA: f32 = dwt::IDWT97_NEG_DELTA_F32; + const KAPPA: f32 = dwt::DWT97_KAPPA_F32; + let low = &mut low[..low_source.len() - start + 2]; + let high = &mut high[..high_source.len() - start + 2]; + load_ghosted(low, &low_source[start..]); + load_ghosted(high, &high_source[start..]); + // Steps 1 and 2. + scale_row(simd, low, KAPPA); + scale_row(simd, high, high_pass); + // Steps 3 to 5. + for (update_low, coefficient) in [(true, NEG_DELTA), (false, NEG_GAMMA), (true, NEG_BETA)] { + let coefficient_simd = f32x8::splat(simd, coefficient); + let (target, source, reads) = if update_low { + (&mut *low, &mut *high, low_reads) + } else { + (&mut *high, &mut *low, high_reads) + }; + lift_row( + simd, + target, + source, + reads, + #[inline(always)] + |s, a, b| (a + b).mul_add(coefficient_simd, s), + #[inline(always)] + |s, a, b| math::mul_add(a + b, coefficient, s), + ); + } + // Step 6, the interleave, and the vertical scale share their final loads + // and store. The multiply still follows the final lifting expression for + // every sample. + let coefficient_simd = f32x8::splat(simd, NEG_ALPHA); + lift_high_to_row::( + simd, + high, + low, + high_reads, + low_first, + tail, + row_scale, + #[inline(always)] + |s, a, b| (a + b).mul_add(coefficient_simd, s), + #[inline(always)] + |s, a, b| math::mul_add(a + b, NEG_ALPHA, s), + ); +} + +/// Columns `columns` of [`synthesize_row`] of an even-origin 9-7I row that +/// [`column_strips`] split, into `out`: the window blocks the columns cover +/// and, for the row's last strip, its multi-pass tail. +#[expect( + clippy::too_many_arguments, + reason = "the strip synthesis keeps its geometry, two sources, output, and exact scratch explicit" +)] +#[expect( + clippy::inline_always, + reason = "the lifting kernels are specialized inside each SIMD dispatch target" +)] +#[inline(always)] +fn synthesize_row_columns( + simd: S, + level: Level2d<'_>, + low_source: &[f32], + high_source: &[f32], + out: &mut [f32], + columns: Range, + low: &mut [f32], + high: &mut [f32], + row_scale: f32, +) { + let first_block = columns.start / (2 * SIMD_WIDTH); + if columns.end < level.width { + lift_row_97_window( + simd, + low_source, + high_source, + out, + first_block, + level.high_pass, + row_scale, + ); + return; + } + let window = window_pairs_97(high_source.len()); + let start = window - WINDOW_HALO_PAIRS; + // As in `synthesize_row`, the tail first, then the window over its left + // edge. + synthesize_97_tail( + simd, + [low_source, high_source], + [low, high], + [(0, 1), (1, 2)], + true, + &mut out[2 * start - columns.start..], + start, + level.high_pass, + row_scale, + ); + lift_row_97_window( + simd, + low_source, + high_source, + &mut out[..2 * window - columns.start], + first_block, + level.high_pass, + row_scale, + ); +} + +/// Columns of each strip the last level reconstructs on its own when it +/// streams on one thread ([`last_level`]), so that the strip's rows of all +/// three components stay in the core's L1 cache. Tests use narrow strips so +/// that small images have several. +const STRIP_COLUMNS: usize = if cfg!(test) { 64 } else { 1024 }; + +/// The column ranges a level's rows are synthesized and lifted in: strips +/// of [`STRIP_COLUMNS`] for wide even-origin 9-7I levels, whose last strip +/// takes the remaining columns and the row's multi-pass tail, or else the +/// whole row. +fn column_strips(level: Level2d<'_>) -> (usize, usize) { + let width = level.width; + let strips = width / STRIP_COLUMNS; + let low_first = level.x0.is_multiple_of(2); + let high_len = if low_first { + width / 2 + } else { + width.div_ceil(2) + }; + // The last strip starts before its window ends and before the tail does. + let last_start = strips.saturating_sub(1) * STRIP_COLUMNS; + let window = window_pairs_97(high_len); + if level.transform != WaveletTransform::Irreversible97 + || !low_first + || strips < 2 + || width < MIN_SIMD_ROW_WIDTH + || last_start + 2 * SIMD_WIDTH > 2 * window.saturating_sub(WINDOW_HALO_PAIRS) + { + return (width, 1); + } + (STRIP_COLUMNS, strips) +} + +/// Strip `index` of `strips` strips of `strip_columns` columns each, the +/// last taking every remaining column of a row of `width`. +fn strip_range( + index: usize, + (strip_columns, strips): (usize, usize), + width: usize, +) -> Range { + let start = index * strip_columns; + start..if index + 1 == strips { + width + } else { + start + strip_columns + } +} + /// Pairs before the row's remaining pairs that their multi-pass lifting /// starts from: a wrong left edge spreads one pair per two lifting steps. const WINDOW_HALO_PAIRS: usize = 2; @@ -1354,13 +1540,16 @@ fn window_pairs_97(high_len: usize) -> usize { high_len.saturating_sub(2 * SIMD_WIDTH) / SIMD_WIDTH * SIMD_WIDTH } -/// The horizontal 9-7I synthesis of the first `row.len() / 2` pairs of an -/// even-origin row from its `low` and `high` band rows, in one pass of -/// `SIMD_WIDTH`-pair blocks: both bands are scaled as they are loaded, the -/// delta, gamma, beta and alpha steps run a block apart with their neighbors -/// taken across blocks by `slide`, and each block is interleaved, scaled by -/// `row_scale` and stored. Each sample sees the operations of the multi-pass -/// steps in [`synthesize_row`]. +/// The horizontal 9-7I synthesis of pairs `SIMD_WIDTH * first_block` on, as +/// many as `row` has room for (`row.len() / 2`), of an even-origin row from +/// its `low` and `high` band rows, in one pass of `SIMD_WIDTH`-pair blocks: +/// both bands are scaled as they are loaded, the delta, gamma, beta and alpha +/// steps run a block apart with their neighbors taken across blocks by +/// `slide`, and each block is interleaved, scaled by `row_scale` and stored. +/// A pass from block zero mirrors the row's left edge; one from a later block +/// (at least two) first lifts the two blocks before it as far as their +/// outputs need. Each sample sees the operations of the multi-pass steps in +/// [`synthesize_row`]. #[expect( clippy::inline_always, reason = "the lifting kernels are specialized inside each SIMD dispatch target" @@ -1371,6 +1560,7 @@ fn lift_row_97_window( low: &[f32], high: &[f32], row: &mut [f32], + first_block: usize, high_pass: f32, row_scale: f32, ) { @@ -1383,33 +1573,48 @@ fn lift_row_97_window( dwt::IDWT97_NEG_ALPHA_F32, ] .map(|value| f32x8::splat(simd, value)); - let load = |block: usize| { - let samples = - |band: &[f32]| f32x8::from_slice(simd, &band[block * SIMD_WIDTH..][..SIMD_WIDTH]); - (samples(low) * kappa, samples(high) * high_pass) - }; + let bands = (low, high); + let scales = (kappa, high_pass); // Low sample `k` lies between high samples `k - 1` and `k`; high sample // `k` between low samples `k` and `k + 1`. let update = |previous: f32x8, high, low, c| (previous.slide::<7>(high) + high).mul_add(c, low); let predict = |low: f32x8, next: f32x8, high, c| (low + low.slide::<1>(next)).mul_add(c, high); - // T.800 F.3.7: high sample -1 mirrors high sample 0, which a block - // rotated by one lane holds in its last lane. - let mirror = |high: f32x8| high.slide::<1>(high); - let (low0, high0) = load(0); - let (next_low0, mut next_high0) = load(1); - let low1 = update(mirror(high0), high0, low0, delta); - let mut next_low1 = update(high0, next_high0, next_low0, delta); - let mut high1 = predict(low1, next_low1, high0, gamma); - let mut low2 = update(mirror(high1), high1, low1, beta); - for (block, out) in row + // The pipeline before the first output block: its doubly lifted low + // samples, its once lifted high samples, the next block's once lifted + // low samples and the next block's scaled high samples. + let (mut low2, mut high1, mut next_low1, mut next_high0) = if first_block == 0 { + // T.800 F.3.7: high sample -1 mirrors high sample 0, which a block + // rotated by one lane holds in its last lane. + let mirror = |high: f32x8| high.slide::<1>(high); + let (low0, high0) = scaled_blocks(simd, bands, scales, 0); + let (next_low0, next_high0) = scaled_blocks(simd, bands, scales, 1); + let low1 = update(mirror(high0), high0, low0, delta); + let next_low1 = update(high0, next_high0, next_low0, delta); + let high1 = predict(low1, next_low1, high0, gamma); + let low2 = update(mirror(high1), high1, low1, beta); + (low2, high1, next_low1, next_high0) + } else { + let (_, before_high0) = scaled_blocks(simd, bands, scales, first_block - 2); + let (previous_low0, previous_high0) = scaled_blocks(simd, bands, scales, first_block - 1); + let (low0, high0) = scaled_blocks(simd, bands, scales, first_block); + let (next_low0, next_high0) = scaled_blocks(simd, bands, scales, first_block + 1); + let previous_low1 = update(before_high0, previous_high0, previous_low0, delta); + let low1 = update(previous_high0, high0, low0, delta); + let next_low1 = update(high0, next_high0, next_low0, delta); + let previous_high1 = predict(previous_low1, low1, previous_high0, gamma); + let high1 = predict(low1, next_low1, high0, gamma); + let low2 = update(previous_high1, high1, low1, beta); + (low2, high1, next_low1, next_high0) + }; + for (index, out) in row .as_chunks_mut::<{ 2 * SIMD_WIDTH }>() .0 .iter_mut() .enumerate() { - let (block_low0, block_high0) = load(block + 2); + let (block_low0, block_high0) = scaled_blocks(simd, bands, scales, first_block + index + 2); let block_low1 = update(next_high0, block_high0, block_low0, delta); let next_high1 = predict(next_low1, block_low1, next_high0, gamma); let next_low2 = update(high1, next_high1, next_low1, beta); @@ -1421,6 +1626,28 @@ fn lift_row_97_window( } } +/// Block `block` of the low and high bands, each times its scale. A +/// function rather than a closure in [`lift_row_97_window`]: called several +/// times, a closure is outlined, and an outlined closure does not inherit the +/// dispatch's target features. +#[expect( + clippy::inline_always, + reason = "the block load is specialized inside each SIMD dispatch target" +)] +#[inline(always)] +fn scaled_blocks( + simd: S, + (low, high): (&[f32], &[f32]), + (low_scale, high_scale): (f32x8, f32x8), + block: usize, +) -> (f32x8, f32x8) { + let start = block * SIMD_WIDTH; + ( + f32x8::from_slice(simd, &low[start..][..SIMD_WIDTH]) * low_scale, + f32x8::from_slice(simd, &high[start..][..SIMD_WIDTH]) * high_scale, + ) +} + /// Copy `source` into `buffer[1..]`, leaving room for a ghost at each end. fn load_ghosted(buffer: &mut [f32], source: &[f32]) { debug_assert_eq!(buffer.len(), source.len() + 2); diff --git a/crates/j2k-native/src/j2c/idwt/banded/last_level.rs b/crates/j2k-native/src/j2c/idwt/banded/last_level.rs index fd0f714d3..175669936 100644 --- a/crates/j2k-native/src/j2c/idwt/banded/last_level.rs +++ b/crates/j2k-native/src/j2c/idwt/banded/last_level.rs @@ -16,9 +16,9 @@ use core::ops::Range; use super::super::super::codestream::WaveletTransform; use super::super::super::rect::IntRect; use super::{ - horizontal_scratch_elements, lift_arrivals, packed_band_rects, parallel_strip_layout, - split_horizontal_scratch, synthesize_level_row, use_banded, vertical_steps, Level2d, RingRows, - RING_ROWS, WAVEFRONT_ARRIVALS, + column_strips, horizontal_scratch_elements, lift_arrivals, packed_band_rects, + parallel_strip_layout, split_horizontal_scratch, strip_range, synthesize_level_columns, + use_banded, vertical_steps, Level2d, RingRows, RING_ROWS, STRIP_COLUMNS, WAVEFRONT_ARRIVALS, }; use crate::error::{bail, DecodingError}; use crate::math::{dispatch, Level, Simd}; @@ -106,14 +106,39 @@ const fn ring_rows(batch: usize) -> usize { batch + RING_ROWS - 1 } -/// Scratch one worker needs: three rings and the horizontal bands. -fn worker_scratch_elements(width: usize, batch: usize) -> Result { - (width + RING_ROW_PAD) - .checked_mul(3 * ring_rows(batch)) +/// Scratch one worker needs: three rings for each column strip a level of +/// this width can have when `split`, and the horizontal bands. +fn worker_scratch_elements(width: usize, batch: usize, split: bool) -> Result { + let strips = if split { + (width / STRIP_COLUMNS).max(1) + } else { + 1 + }; + strips + .checked_mul(RING_ROW_PAD) + .and_then(|pads| pads.checked_add(width)) + .and_then(|row| row.checked_mul(3 * ring_rows(batch))) .and_then(|rings| rings.checked_add(horizontal_scratch_elements(width).ok()?)) .ok_or(ValidationError::ImageTooLarge.into()) } +/// The three components' rings of column strip `index` in `rings`, each of +/// `ring_rows` rows `stride` apart: every strip before it is `strips.0` +/// columns wide. +fn strip_rings( + rings: &mut [f32], + index: usize, + strips: (usize, usize), + stride: usize, + ring_rows: usize, +) -> [&mut [f32]; 3] { + let ring_len = ring_rows * stride; + let start = index * 3 * ring_rows * (strips.0 + RING_ROW_PAD); + let (r, rest) = rings[start..].split_at_mut(ring_len); + let (g, rest) = rest.split_at_mut(ring_len); + [r, g, &mut rest[..ring_len]] +} + /// Reconstruct `level` of the three components from their `planes` and the /// first component's `first_ll` band, and pass every row to `emit` with the /// three components' samples, which it may overwrite, and that row of @@ -141,9 +166,13 @@ pub(crate) fn pack_last_level( let Some(layout) = parallel_strip_layout(width, height, parallel)? else { let batch = SERIAL_BATCH_ROWS; let mut scratch = Vec::new(); - try_resize_decode_elements(&mut scratch, worker_scratch_elements(width, batch)?, 0.0)?; + try_resize_decode_elements( + &mut scratch, + worker_scratch_elements(width, batch, true)?, + 0.0, + )?; dispatch!(Level::new(), simd => stream_rows( - simd, &levels, 0, batch, output, &mut scratch, &emit, + simd, &levels, 0, batch, true, output, &mut scratch, &emit, )); return Ok(()); }; @@ -151,7 +180,7 @@ pub(crate) fn pack_last_level( { let strip_bytes = layout.strip_len / width * row_bytes; let workers = layout.concurrent_strips; - let per_worker = worker_scratch_elements(width, PARALLEL_BATCH_ROWS)?; + let per_worker = worker_scratch_elements(width, PARALLEL_BATCH_ROWS, false)?; let complete = dispatch!(Level::new(), simd => run_strips( simd, &levels, output, strip_bytes, workers, per_worker, scratch, &emit, )); @@ -223,6 +252,7 @@ fn run_strips( levels, first_row, PARALLEL_BATCH_ROWS, + false, output, scratch, emit, @@ -238,6 +268,7 @@ fn run_strips( levels, first_row, PARALLEL_BATCH_ROWS, + false, output, scratch, emit, @@ -251,8 +282,11 @@ fn run_strips( /// three components, with the schedule and halo rows of /// [`super::stream_strip`], and emit each row. /// -/// Each component takes `batch` arrivals in turn, so the few rows -/// its lifting steps share stay in cache. At an arrival, lifting step `k` +/// Each component takes `batch` arrivals in turn, one column strip +/// ([`column_strips`]) at a time, so the few rows its lifting steps share +/// stay in cache: lifting is per column, so the strips are independent but +/// for their horizontal synthesis, which reads past their edges. At an +/// arrival, lifting step `k` /// (1-based) updates the row `k` above it, reading that row's two /// neighbors, so after a batch no row more than `count + 1` above its last /// arrival is read again: those rows are emitted, and the ring refills @@ -261,12 +295,17 @@ fn run_strips( clippy::inline_always, reason = "the strip kernel is specialized at the architecture dispatch boundary" )] +#[expect( + clippy::too_many_arguments, + reason = "the row stream keeps its geometry, batch, strip choice, and reused scratch explicit" +)] #[inline(always)] fn stream_rows( simd: S, levels: &[Level2d<'_>; 3], first_row: usize, batch: usize, + split: bool, output: &mut [u8], scratch: &mut [f32], emit: &impl Fn([&mut [f32]; 3], &mut [u8]), @@ -278,49 +317,75 @@ fn stream_rows( let (_, count) = vertical_steps(levels[0]); let lo = first_row.saturating_sub(count); let hi = (end + count).min(height); - let stride = width + RING_ROW_PAD; let ring_rows = ring_rows(batch); - let ring_len = ring_rows * stride; - let (rings, horizontal_scratch) = scratch.split_at_mut(3 * ring_len); + // Column strips pay on one thread. On the pool, where several cores + // stream band rows from memory at once, their shorter row segments cost + // more than the strips save. + let strips = if split { + column_strips(levels[0]) + } else { + (width, 1) + }; + let rings_len = (0..strips.1) + .map(|index| 3 * ring_rows * (strip_range(index, strips, width).len() + RING_ROW_PAD)) + .sum(); + let (rings, horizontal_scratch) = scratch.split_at_mut(rings_len); let (low, high) = split_horizontal_scratch(levels[0], horizontal_scratch); - let (r, rest) = rings.split_at_mut(ring_len); - let (g, b) = rest.split_at_mut(ring_len); - let mut rings = [r, g, b]; - let slot = |row: usize| row % ring_rows; - let mut emitted = first_row; - let mut emit_rows = |rings: &mut [&mut [f32]; 3], rows_end: usize| { - for row in emitted..rows_end { - let start = slot(row) * stride; - let samples = rings.each_mut().map(|ring| &mut ring[start..][..width]); - emit( - samples, - &mut output[(row - first_row) * row_bytes..][..row_bytes], - ); + let mut emit_rows = |rings: &mut [f32], index: usize, rows: Range| { + let columns = strip_range(index, strips, width); + let stride = columns.len() + RING_ROW_PAD; + let mut rings = strip_rings(rings, index, strips, stride, ring_rows); + for row in rows { + let start = row % ring_rows * stride; + let samples = rings + .each_mut() + .map(|ring| &mut ring[start..][..columns.len()]); + let output = &mut output[(row - first_row) * row_bytes..][..row_bytes]; + emit(samples, &mut output[3 * columns.start..3 * columns.end]); } - emitted = emitted.max(rows_end); }; + let mut emitted = first_row; for first in (lo..hi + count).step_by(batch) { let arrivals = first..(first + batch).min(hi + count); - for (level, ring) in levels.iter().zip(rings.iter_mut()) { - let mut rows = RingRows { - rows: &mut ring[..], - width, - stride, - slots: ring_rows, - }; - // The ring holds every row the batch's steps read besides the - // batch itself, so arrivals can be synthesized ahead. - for first in arrivals.clone().step_by(WAVEFRONT_ARRIVALS) { - let ahead = first..(first + WAVEFRONT_ARRIVALS).min(arrivals.end); - for arrival in ahead.start..ahead.end.min(hi) { - synthesize_level_row(simd, *level, arrival, rows.row_mut(arrival), low, high); + // The batch's last arrival read its rows `count + 1` above it. + let unread = (arrivals.end - 1).saturating_sub(count).min(end); + // Each column strip runs the batch of all three components and + // emits its part of the finished rows, while its rings are in cache. + for index in 0..strips.1 { + let columns = strip_range(index, strips, width); + let stride = columns.len() + RING_ROW_PAD; + let component_rings = strip_rings(rings, index, strips, stride, ring_rows); + for (level, ring) in levels.iter().zip(component_rings) { + let mut rows = RingRows { + rows: ring, + width: columns.len(), + stride, + slots: ring_rows, + }; + // The ring holds every row the batch's steps read besides + // the batch itself, so arrivals can be synthesized ahead. + for first in arrivals.clone().step_by(WAVEFRONT_ARRIVALS) { + let ahead = first..(first + WAVEFRONT_ARRIVALS).min(arrivals.end); + for arrival in ahead.start..ahead.end.min(hi) { + let row = rows.row_mut(arrival); + synthesize_level_columns( + simd, + *level, + arrival, + row, + columns.clone(), + low, + high, + ); + } + lift_arrivals(simd, *level, ahead, lo..hi, &mut rows); } - lift_arrivals(simd, *level, ahead, lo..hi, &mut rows); } + emit_rows(rings, index, emitted..unread.max(emitted)); } - // The batch's last arrival read its rows `count + 1` above it. - let unread = (arrivals.end - 1).saturating_sub(count); - emit_rows(&mut rings, unread.min(end)); + emitted = emitted.max(unread); + } + for index in 0..strips.1 { + emit_rows(rings, index, emitted..end); } - emit_rows(&mut rings, end); } diff --git a/crates/j2k-native/src/tests/fused_last_level.rs b/crates/j2k-native/src/tests/fused_last_level.rs index 82115471c..d5e608a6a 100644 --- a/crates/j2k-native/src/tests/fused_last_level.rs +++ b/crates/j2k-native/src/tests/fused_last_level.rs @@ -45,9 +45,11 @@ const fn rgb(size: (u32, u32), levels: u8, fused: bool) -> Case { } } -const CASES: [Case; 9] = [ +const CASES: [Case; 10] = [ // Several parallel strips, odd widths and heights, and the fewest levels. rgb((520, 136), 5, true), + // Column strips whose last one takes an odd remainder and the row tail. + rgb((330, 72), 3, true), // Coarse scratch can supply several parallel pack rings. rgb((64, 2_048), 5, true), rgb((97, 88), 3, true), From 762ac830dcb3ace786a86ac643bc967d9b1fb3d8 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 02:53:33 -0400 Subject: [PATCH 33/53] perf(j2k-native): synthesize 5/3 rows in one register-window pass The 5-3R horizontal synthesis copied both band rows into ghosted scratch, lifted the low samples in one pass and the high samples with the interleave in another. An even-origin row now runs both steps a block apart in one pass of SIMD_WIDTH-pair blocks, with the neighbours taken across blocks by `slide`, as the 9-7I rows already do; the multi-pass steps keep the row's last pairs, from two pairs earlier, and the pass then overwrites their wrong left edge. Every sample sees the same expressions, so the output is bit-identical. The pass can start at any block, so the serial last level's column strips now cover 5/3 levels too. HT lossless full-image decode (4096x2048 RGB, an 18-core ARM64 machine): -4.4% at 1 thread, -6.3% at 6, -2.0% at 18, -2.5% at the default, against the previous commit. Output is unchanged. --- crates/j2k-native/src/j2c/idwt/banded.rs | 267 ++++++++++++++++++----- 1 file changed, 217 insertions(+), 50 deletions(-) diff --git a/crates/j2k-native/src/j2c/idwt/banded.rs b/crates/j2k-native/src/j2c/idwt/banded.rs index 107f7ec26..f80af5fc1 100644 --- a/crates/j2k-native/src/j2c/idwt/banded.rs +++ b/crates/j2k-native/src/j2c/idwt/banded.rs @@ -511,8 +511,10 @@ fn synthesize_level_columns( synthesize_level_row(simd, level, index, out, low, high); return; } - // Strips are only made of even-origin 9-7I levels (`column_strips`). - let row_scale = if index % 2 == level.y0 % 2 { + // Strips are only made of even-origin levels (`column_strips`). + let row_scale = if level.transform == WaveletTransform::Reversible53 { + 1.0 + } else if index % 2 == level.y0 % 2 { dwt::DWT97_KAPPA_F32 } else { level.high_pass @@ -1284,34 +1286,26 @@ fn synthesize_row( match level.transform { WaveletTransform::Reversible53 => { - load_ghosted(low, low_source); - load_ghosted(high, high_source); - // Equation (F-5), then (F-6). - let quarter = f32x8::splat(simd, 0.25); - let half = f32x8::splat(simd, 0.5); - lift_row( - simd, - low, - high, - low_reads, - #[inline(always)] - |s, a, b| s - (a + b).mul_add(quarter, half).floor(), - #[inline(always)] - |s, a, b| s - math::floor_f32(math::mul_add(a + b, 0.25, 0.5)), - ); - lift_high_to_row::( + // As for 9-7I below, with `lift_row_53_window`. + let window = if low_first { + window_pairs_97(high_source.len()) + } else { + 0 + }; + let start = window.saturating_sub(WINDOW_HALO_PAIRS); + synthesize_53_tail( simd, - high, - low, - high_reads, + [low_source, high_source], + [low, high], + [low_reads, high_reads], low_first, - row, - 1.0, - #[inline(always)] - |s, a, b| s + ((a + b) * 0.5).floor(), - #[inline(always)] - |s, a, b| s + math::floor_half_sum_f32(a, b), + &mut row[2 * start..], + start, ); + if window > 0 { + let row = &mut row[..2 * window]; + lift_row_53_window(simd, low_source, high_source, row, 0); + } } WaveletTransform::Irreversible97 => { // An even-origin row lifts its first `window` pairs in one pass @@ -1351,6 +1345,138 @@ fn synthesize_row( } } +/// The multi-pass 5-3R synthesis of a row's pairs from `start` on into +/// `tail`, through ghosted copies of the band rows' remaining samples. +#[inline(always)] +fn synthesize_53_tail( + simd: S, + [low_source, high_source]: [&[f32]; 2], + [low, high]: [&mut [f32]; 2], + [low_reads, high_reads]: [(usize, usize); 2], + low_first: bool, + tail: &mut [f32], + start: usize, +) { + let low = &mut low[..low_source.len() - start + 2]; + let high = &mut high[..high_source.len() - start + 2]; + load_ghosted(low, &low_source[start..]); + load_ghosted(high, &high_source[start..]); + // Equation (F-5), then (F-6). + let quarter = f32x8::splat(simd, 0.25); + let half = f32x8::splat(simd, 0.5); + lift_row( + simd, + low, + high, + low_reads, + #[inline(always)] + |s, a, b| s - (a + b).mul_add(quarter, half).floor(), + #[inline(always)] + |s, a, b| s - math::floor_f32(math::mul_add(a + b, 0.25, 0.5)), + ); + lift_high_to_row::( + simd, + high, + low, + high_reads, + low_first, + tail, + 1.0, + #[inline(always)] + |s, a, b| s + ((a + b) * 0.5).floor(), + #[inline(always)] + |s, a, b| s + math::floor_half_sum_f32(a, b), + ); +} + +/// The horizontal 5-3R synthesis of pairs `SIMD_WIDTH * first_block` on, as +/// many as `row` has room for, of an even-origin row from its `low` and +/// `high` band rows, in one pass of `SIMD_WIDTH`-pair blocks: equation (F-5) +/// a block ahead of (F-6), neighbors taken across blocks by `slide`, each +/// block interleaved and stored. A pass from block zero mirrors the row's +/// left edge; one from a later block first lifts that block's low samples +/// from the block before it. Each sample sees the operations of the +/// multi-pass steps in [`synthesize_53_tail`]. +#[expect( + clippy::inline_always, + reason = "the lifting kernels are specialized inside each SIMD dispatch target" +)] +#[inline(always)] +fn lift_row_53_window( + simd: S, + low: &[f32], + high: &[f32], + row: &mut [f32], + first_block: usize, +) { + let steps = [f32x8::splat(simd, 0.25), f32x8::splat(simd, 0.5)]; + let mut high0 = band_block(simd, high, first_block); + let mut low1 = if first_block == 0 { + // T.800 F.3.7: high sample -1 mirrors high sample 0. + let low0 = band_block(simd, low, 0); + update_53(high0.slide::<1>(high0), high0, low0, steps) + } else { + let previous = band_block(simd, high, first_block - 1); + update_53(previous, high0, band_block(simd, low, first_block), steps) + }; + for (index, out) in row + .as_chunks_mut::<{ 2 * SIMD_WIDTH }>() + .0 + .iter_mut() + .enumerate() + { + let block = first_block + index + 1; + let next_high0 = band_block(simd, high, block); + let next_low1 = update_53(high0, next_high0, band_block(simd, low, block), steps); + let high1 = predict_53(low1, next_low1, high0); + let (first, second) = out.split_at_mut(SIMD_WIDTH); + low1.zip_low(high1).store(first); + low1.zip_high(high1).store(second); + (high0, low1) = (next_high0, next_low1); + } +} + +/// Equation (F-5) on a block of low samples from the high block before them +/// (`previous`) and their own: low sample `k` lies between high samples +/// `k - 1` and `k`. `[quarter, half]` are splats of 0.25 and 0.5. +#[expect( + clippy::inline_always, + reason = "the lifting kernels are specialized inside each SIMD dispatch target" +)] +#[inline(always)] +fn update_53( + previous: f32x8, + high: f32x8, + low: f32x8, + [quarter, half]: [f32x8; 2], +) -> f32x8 { + low - (previous.slide::<7>(high) + high) + .mul_add(quarter, half) + .floor() +} + +/// Equation (F-6) on a block of high samples from the lifted low block with +/// them and the next: high sample `k` lies between low samples `k` and +/// `k + 1`. +#[expect( + clippy::inline_always, + reason = "the lifting kernels are specialized inside each SIMD dispatch target" +)] +#[inline(always)] +fn predict_53(low: f32x8, next: f32x8, high: f32x8) -> f32x8 { + high + ((low + low.slide::<1>(next)) * 0.5).floor() +} + +/// Block `block` of `SIMD_WIDTH` samples of `band`. +#[expect( + clippy::inline_always, + reason = "the block load is specialized inside each SIMD dispatch target" +)] +#[inline(always)] +fn band_block(simd: S, band: &[f32], block: usize) -> f32x8 { + f32x8::from_slice(simd, &band[block * SIMD_WIDTH..][..SIMD_WIDTH]) +} + /// The multi-pass 9-7I synthesis of a row's pairs from `start` on into /// `tail`, through ghosted copies of the band rows' remaining samples. #[expect( @@ -1419,7 +1545,7 @@ fn synthesize_97_tail( ); } -/// Columns `columns` of [`synthesize_row`] of an even-origin 9-7I row that +/// Columns `columns` of [`synthesize_row`] of an even-origin row that /// [`column_strips`] split, into `out`: the window blocks the columns cover /// and, for the row's last strip, its multi-pass tail. #[expect( @@ -1443,44 +1569,86 @@ fn synthesize_row_columns( row_scale: f32, ) { let first_block = columns.start / (2 * SIMD_WIDTH); + let window = window_pairs_97(high_source.len()); + let reversible = level.transform == WaveletTransform::Reversible53; if columns.end < level.width { - lift_row_97_window( + lift_row_window( simd, - low_source, - high_source, + level, + [low_source, high_source], out, first_block, - level.high_pass, row_scale, ); return; } - let window = window_pairs_97(high_source.len()); let start = window - WINDOW_HALO_PAIRS; // As in `synthesize_row`, the tail first, then the window over its left // edge. - synthesize_97_tail( + let tail = &mut out[2 * start - columns.start..]; + if reversible { + synthesize_53_tail( + simd, + [low_source, high_source], + [low, high], + [(0, 1), (1, 2)], + true, + tail, + start, + ); + } else { + synthesize_97_tail( + simd, + [low_source, high_source], + [low, high], + [(0, 1), (1, 2)], + true, + tail, + start, + level.high_pass, + row_scale, + ); + } + lift_row_window( simd, + level, [low_source, high_source], - [low, high], - [(0, 1), (1, 2)], - true, - &mut out[2 * start - columns.start..], - start, - level.high_pass, - row_scale, - ); - lift_row_97_window( - simd, - low_source, - high_source, &mut out[..2 * window - columns.start], first_block, - level.high_pass, row_scale, ); } +/// The register-window pass of `level`'s transform over `out` from +/// `first_block` ([`lift_row_53_window`], [`lift_row_97_window`]). +#[expect( + clippy::inline_always, + reason = "the lifting kernels are specialized inside each SIMD dispatch target" +)] +#[inline(always)] +fn lift_row_window( + simd: S, + level: Level2d<'_>, + [low_source, high_source]: [&[f32]; 2], + out: &mut [f32], + first_block: usize, + row_scale: f32, +) { + if level.transform == WaveletTransform::Reversible53 { + lift_row_53_window(simd, low_source, high_source, out, first_block); + } else { + lift_row_97_window( + simd, + low_source, + high_source, + out, + first_block, + level.high_pass, + row_scale, + ); + } +} + /// Columns of each strip the last level reconstructs on its own when it /// streams on one thread ([`last_level`]), so that the strip's rows of all /// three components stay in the core's L1 cache. Tests use narrow strips so @@ -1488,7 +1656,7 @@ fn synthesize_row_columns( const STRIP_COLUMNS: usize = if cfg!(test) { 64 } else { 1024 }; /// The column ranges a level's rows are synthesized and lifted in: strips -/// of [`STRIP_COLUMNS`] for wide even-origin 9-7I levels, whose last strip +/// of [`STRIP_COLUMNS`] for wide even-origin levels, whose last strip /// takes the remaining columns and the row's multi-pass tail, or else the /// whole row. fn column_strips(level: Level2d<'_>) -> (usize, usize) { @@ -1503,8 +1671,7 @@ fn column_strips(level: Level2d<'_>) -> (usize, usize) { // The last strip starts before its window ends and before the tail does. let last_start = strips.saturating_sub(1) * STRIP_COLUMNS; let window = window_pairs_97(high_len); - if level.transform != WaveletTransform::Irreversible97 - || !low_first + if !low_first || strips < 2 || width < MIN_SIMD_ROW_WIDTH || last_start + 2 * SIMD_WIDTH > 2 * window.saturating_sub(WINDOW_HALO_PAIRS) From 7bdd713176b5e953080f4601ef460a9b5c7753be Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:45:23 -0400 Subject: [PATCH 34/53] perf(j2k-native): run the two finest pool IDWT levels in one pass On the pool every IDWT level was its own pass of row strips, so the finer level waited at a barrier for the coarser level's slowest strip, and the levels' short phases left workers idle. The two finest levels `idwt::apply` reconstructs now run as one pass, `filter_2d_banded_pair`: each strip of the finer level first streams the rows of the coarser level it reads, with the halo rows of a strip of that level, into its worker's scratch, then streams its own rows from them. Its workers do not wait for the rest of the coarser level, and the coarser level is no longer written to the ping-pong buffer. - Those coarser rows go through the same expressions as in a pass over the whole level, so the output is bit-identical. The cost is the coarser level's halo rows, recomputed per strip of the finer level. - `RowSource` can skip band rows at its front, so a level can read its LL band rows from scratch (`Level2d::with_ll_rows`). - One thread, regions, the decoder hook and the integer path keep the level-by-level passes. HT 2 bpp full-image decode (4096x2048 RGB, an 18-core ARM64 machine): -1.5% at 6 threads, -4.7% at 18, -6.9% at the default; HT lossless -1.3%, -4.0%, -3.2%; one thread, regions and concurrent region decodes unchanged. Output is unchanged. --- crates/j2k-native/src/j2c/idwt/banded.rs | 288 +++++++++++++++++- crates/j2k-native/src/j2c/idwt/orchestrate.rs | 63 +++- 2 files changed, 343 insertions(+), 8 deletions(-) diff --git a/crates/j2k-native/src/j2c/idwt/banded.rs b/crates/j2k-native/src/j2c/idwt/banded.rs index f80af5fc1..9dd178e67 100644 --- a/crates/j2k-native/src/j2c/idwt/banded.rs +++ b/crates/j2k-native/src/j2c/idwt/banded.rs @@ -28,6 +28,8 @@ use super::horizontal::filter_row; use super::interleave::interleave_row; use super::model::{IDWTInput, IDWTTempOutput}; use super::try_grow_level_buffer; +#[cfg(feature = "parallel")] +use crate::error::{bail, DecodingError}; use crate::math::{self, dispatch, f32x8, Level, Simd, SIMD_WIDTH}; use crate::{checked_decode_usize_product2, Result, ValidationError}; use j2k_codec_math::dwt; @@ -179,6 +181,9 @@ struct RowSource<'a> { second: &'a [f32], first_width: usize, second_width: usize, + /// Band rows missing from the front of `first` and `second`. + first_skip: usize, + second_skip: usize, start_row: usize, rows: usize, } @@ -193,13 +198,12 @@ impl<'a> RowSource<'a> { if index >= self.rows { return None; } + let band_row = |band: &'a [f32], width: usize, skip: usize| { + band.get(index.checked_sub(skip)? * width..)?.get(..width) + }; Some(( - self.first - .get(index * self.first_width..)? - .get(..self.first_width)?, - self.second - .get(index * self.second_width..)? - .get(..self.second_width)?, + band_row(self.first, self.first_width, self.first_skip)?, + band_row(self.second, self.second_width, self.second_skip)?, )) } } @@ -248,6 +252,8 @@ impl<'a> Level2d<'a> { second: second_even, first_width, second_width, + first_skip: 0, + second_skip: 0, start_row: usize::from(v0 % 2 != 0), rows: num_v_low, }, @@ -256,6 +262,8 @@ impl<'a> Level2d<'a> { second: second_odd, first_width, second_width, + first_skip: 0, + second_skip: 0, start_row: usize::from(v0 % 2 == 0), rows: num_v_high, }, @@ -267,6 +275,19 @@ impl<'a> Level2d<'a> { high_pass, } } + + /// This level with its LL band rows `first_row..` taken from `rows`. + #[cfg(feature = "parallel")] + fn with_ll_rows(mut self, rows: &'a [f32], first_row: usize) -> Self { + if self.x0.is_multiple_of(2) { + self.even.first = rows; + self.even.first_skip = first_row; + } else { + self.even.second = rows; + self.even.second_skip = first_row; + } + self + } } pub(super) fn filter_2d_banded( @@ -409,6 +430,181 @@ fn run_strips( }); } +/// Whether [`filter_2d_banded_pair`] can reconstruct `outer`, whose LL band +/// `inner` reconstructs, together with `inner` on the pool. +#[cfg(feature = "parallel")] +pub(super) fn pair_runs_on_pool(inner: IntRect, outer: IntRect, parallel: bool) -> bool { + let size = |rect: IntRect| (rect.width() as usize, rect.height() as usize); + let ((inner_width, inner_height), (width, height)) = (size(inner), size(outer)); + size(packed_band_rects(outer)[0]) == (inner_width, inner_height) + && use_banded(inner_width, inner_height) + && use_banded(width, height) + && matches!(parallel_strip_layout(width, height, parallel), Ok(Some(_))) +} + +/// Reconstruct `inner` and then `outer`, whose LL band `inner` reconstructs, +/// into `target` in one pass on the pool, without waiting for all of +/// `inner` before `outer` starts: each strip of `outer` first streams the +/// rows of `inner` it reads into its worker's scratch, as a strip of +/// [`stream_strip`] with its own halo, and then streams its own rows. Those +/// `inner` rows go through the same expressions as in a pass over the whole +/// level, so the output is bit-identical to two level passes. +#[cfg(feature = "parallel")] +pub(super) fn filter_2d_banded_pair( + input: IDWTInput<'_>, + [inner, outer]: [&Decomposition; 2], + target: &mut [f32], + band_scratch: &mut Vec, + transform: WaveletTransform, + storage: &DecompositionStorage<'_>, +) -> Result { + let band = |decomposition: &Decomposition, index: usize| { + &storage.coefficients[storage.sub_bands[decomposition.sub_bands[index]] + .coefficients + .clone()] + }; + let high_pass = super::OPENJPEG_NORMALIZED_HIGH_PASS_F32; + let inner_level = Level2d::new( + inner.rect, + [ + input.coefficients, + band(inner, 0), + band(inner, 1), + band(inner, 2), + ], + transform, + high_pass, + ); + let outer_level = Level2d::new( + outer.rect, + [&[], band(outer, 0), band(outer, 1), band(outer, 2)], + transform, + high_pass, + ); + filter_levels_pair([inner_level, outer_level], target, band_scratch)?; + Ok(IDWTTempOutput { rect: outer.rect }) +} + +/// [`filter_2d_banded_pair`] of `outer`, whose LL band `inner` reconstructs. +#[cfg(feature = "parallel")] +fn filter_levels_pair( + [inner_level, outer_level]: [Level2d<'_>; 2], + target: &mut [f32], + band_scratch: &mut Vec, +) -> Result<()> { + let (width, height) = (outer_level.width, outer_level.height); + let required_len = checked_decode_usize_product2(width, height)?; + let target = target + .get_mut(..required_len) + .ok_or(DecodingError::OutputBufferTooSmall)?; + let Some(layout) = parallel_strip_layout(width, height, true)? else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + let rows_len = pair_ll_rows(layout.strip_len / width, vertical_steps(outer_level).1) + .checked_mul(inner_level.width) + .ok_or(ValidationError::ImageTooLarge)?; + let per_worker = worker_scratch_elements(width)? + .max(worker_scratch_elements(inner_level.width)?) + .checked_add(rows_len) + .ok_or(ValidationError::ImageTooLarge)?; + let scratch_len = per_worker + .checked_mul(layout.concurrent_strips) + .ok_or(ValidationError::ImageTooLarge)?; + try_grow_level_buffer(band_scratch, scratch_len)?; + dispatch!(Level::new(), simd => run_pair_strips( + simd, + [inner_level, outer_level], + target, + layout.strip_len, + &mut band_scratch[..scratch_len], + [per_worker, rows_len], + )); + Ok(()) +} + +/// Rows of the LL band a strip of `strip_rows` rows reads, at most: its +/// arrivals start and end `count` rows beyond it, two output rows per band +/// row, and one band row of slack at each end. +#[cfg(feature = "parallel")] +const fn pair_ll_rows(strip_rows: usize, count: usize) -> usize { + strip_rows.div_ceil(2) + count + 3 +} + +#[cfg(feature = "parallel")] +#[inline(always)] +fn run_pair_strips( + simd: S, + levels: [Level2d<'_>; 2], + target: &mut [f32], + strip_len: usize, + scratch: &mut [f32], + [per_worker, rows_len]: [usize; 2], +) { + use rayon::prelude::*; + use std::sync::Mutex; + + let strips = Mutex::new(target.chunks_mut(strip_len).enumerate()); + let next = || strips.lock().ok().and_then(|mut strips| strips.next()); + let width = levels[1].width; + scratch + .par_chunks_mut(per_worker) + .with_max_len(1) + .for_each(|worker_scratch| { + // As in `run_strips`. + #[cfg(feature = "simd")] + simd.vectorize( + #[inline(always)] + || { + while let Some((strip, output)) = next() { + let first_row = strip * strip_len / width; + pair_strip(simd, levels, first_row, output, worker_scratch, rows_len); + } + }, + ); + #[cfg(not(feature = "simd"))] + while let Some((strip, output)) = next() { + let first_row = strip * strip_len / width; + pair_strip(simd, levels, first_row, output, worker_scratch, rows_len); + } + }); +} + +/// Reconstruct rows `first_row..` of `outer` into `output`, after the rows +/// of `inner` they read. +#[cfg(feature = "parallel")] +#[expect( + clippy::inline_always, + reason = "the strip kernel is specialized at the architecture dispatch boundary" +)] +#[inline(always)] +fn pair_strip( + simd: S, + [inner, outer]: [Level2d<'_>; 2], + first_row: usize, + output: &mut [f32], + worker_scratch: &mut [f32], + rows_len: usize, +) { + let (_, count) = vertical_steps(outer); + let end = first_row + output.len() / outer.width; + // The arrivals of `stream_strip` and the LL band rows they read. + let lo = first_row.saturating_sub(count); + let hi = (end + count).min(outer.height); + let first_ll = (lo / 2).saturating_sub(1); + let end_ll = (hi.div_ceil(2) + 1).min(inner.height); + // `pair_ll_rows` bounds `end_ll - first_ll` for every strip. + let (rows, strip_scratch) = worker_scratch.split_at_mut(rows_len); + let rows = &mut rows[..(end_ll - first_ll) * inner.width]; + stream_strip(simd, inner, first_ll, rows, strip_scratch); + stream_strip( + simd, + outer.with_ll_rows(rows, first_ll), + first_row, + output, + strip_scratch, + ); +} + /// One vertical lifting step: the rows it updates have parity `parity`. #[derive(Clone, Copy)] enum VerticalStep { @@ -2085,4 +2281,84 @@ mod tests { } } } + + /// Two levels in one pool pass give the bits of the inner level's pass + /// and then the outer level's, whatever the transform, origin parity + /// and size. + #[cfg(feature = "parallel")] + #[test] + fn level_pair_matches_two_level_passes() { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .expect("pool"); + let mut seed = 11_u64; + for transform in [ + WaveletTransform::Reversible53, + WaveletTransform::Irreversible97, + ] { + let integral = transform == WaveletTransform::Reversible53; + for (x0, y0, width, height) in [(0, 0, 256, 300), (3, 5, 301, 263), (2, 1, 290, 517)] { + let outer_rect = IntRect::from_xywh(x0, y0, width, height); + let inner_rect = packed_band_rects(outer_rect)[0]; + let mut bands = |rect: IntRect| { + packed_band_rects(rect).map(|band| { + seed += 1; + samples((band.width() * band.height()) as usize, seed, integral) + }) + }; + let [ll, hl, lh, hh] = bands(inner_rect); + let [_, outer_hl, outer_lh, outer_hh] = bands(outer_rect); + let size = |rect: IntRect| (rect.width() * rect.height()) as usize; + let mut scratch = Vec::new(); + let mut inner = alloc::vec![0.0; size(inner_rect)]; + filter_2d_banded_from_bands( + inner_rect, + [&ll, &hl, &lh, &hh], + &mut inner, + &mut scratch, + transform, + 1.25, + false, + ) + .expect("inner level"); + let mut expected = alloc::vec![0.0; size(outer_rect)]; + filter_2d_banded_from_bands( + outer_rect, + [&inner, &outer_hl, &outer_lh, &outer_hh], + &mut expected, + &mut scratch, + transform, + 1.25, + false, + ) + .expect("outer level"); + let levels = [ + Level2d::new(inner_rect, [&ll, &hl, &lh, &hh], transform, 1.25), + Level2d::new( + outer_rect, + [&[], &outer_hl, &outer_lh, &outer_hh], + transform, + 1.25, + ), + ]; + assert!(pool.install(|| pair_runs_on_pool(inner_rect, outer_rect, true))); + let mut actual = alloc::vec![f32::NAN; size(outer_rect)]; + let mut scratch = Vec::new(); + pool.install(|| filter_levels_pair(levels, &mut actual, &mut scratch)) + .expect("level pair"); + assert_eq!( + actual + .iter() + .map(|value| value.to_bits()) + .collect::>(), + expected + .iter() + .map(|value| value.to_bits()) + .collect::>(), + "{transform:?} {outer_rect:?}" + ); + } + } + } } diff --git a/crates/j2k-native/src/j2c/idwt/orchestrate.rs b/crates/j2k-native/src/j2c/idwt/orchestrate.rs index f29813fa4..8127901f0 100644 --- a/crates/j2k-native/src/j2c/idwt/orchestrate.rs +++ b/crates/j2k-native/src/j2c/idwt/orchestrate.rs @@ -2,6 +2,8 @@ use super::super::codestream::WaveletTransform; use super::super::decode::{DecompositionStorage, TileDecodeContext}; +#[cfg(feature = "parallel")] +use super::banded::{filter_2d_banded_pair, pair_runs_on_pool}; use super::direct::{apply_level, apply_level_into}; use super::interleave_i64::apply_level_i64; use super::model::{ @@ -107,9 +109,24 @@ pub(crate) fn apply( } } + // The two finest levels may run as one pass on the pool, which writes + // only the finer level. + #[cfg(feature = "parallel")] + let pair = parallel + && backend.is_none() + && decompositions.len() >= 2 + && pair_runs_on_pool( + decompositions[decompositions.len() - 2].rect, + decompositions[decompositions.len() - 1].rect, + parallel, + ); + #[cfg(not(feature = "parallel"))] + let pair = false; + let writes = decompositions.len() - usize::from(pair); + // Determine which buffer we should use first, such that the `coefficients` // array will always hold the final values. - let mut use_scratch = decompositions.len().is_multiple_of(2); + let mut use_scratch = writes.is_multiple_of(2); let mut current_source = InputSource::SubBand; let mut current_rect = ll_sub_band.rect; // Level buffers only grow, so the previous level's samples are the first @@ -119,7 +136,49 @@ pub(crate) fn apply( rect: ll_sub_band.rect, }; - for decomposition in decompositions { + for (index, decomposition) in decompositions.iter().enumerate() { + #[cfg(feature = "parallel")] + if pair && index + 2 == decompositions.len() { + let input = match current_source { + InputSource::SubBand => IDWTInput::from_sub_band(ll_sub_band, storage), + InputSource::Scratch => { + IDWTInput::from_output(&scratch_buf[..current_len], current_rect) + } + InputSource::Output => unreachable!(), + }; + let outer = &decompositions[index + 1]; + let outer_len = checked_decode_usize_product2( + outer.rect.width() as usize, + outer.rect.height() as usize, + )?; + let target = if let Some(prefix) = prefix.as_deref_mut() { + prefix + } else { + super::try_grow_level_buffer(&mut output.coefficients, outer_len)?; + &mut output.coefficients[..] + }; + temp_output = filter_2d_banded_pair( + input, + [decomposition, outer], + target, + band_scratch, + transform, + storage, + )?; + let inner_len = checked_decode_usize_product2( + decomposition.rect.width() as usize, + decomposition.rect.height() as usize, + )?; + tile_ctx.debug_counters.idwt_output_samples = tile_ctx + .debug_counters + .idwt_output_samples + .saturating_add(inner_len) + .saturating_add(outer_len); + current_len = outer_len; + break; + } + #[cfg(not(feature = "parallel"))] + let _ = index; temp_output = match (current_source, use_scratch) { (InputSource::SubBand, true) => apply_level( IDWTInput::from_sub_band(ll_sub_band, storage), From 5a6a51e7205f07fdcefc6f2fca99ce73b427f898 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 17:31:18 -0400 Subject: [PATCH 35/53] fix(j2k-native): charge the paired IDWT levels' band scratch On the pool, `idwt::apply` reconstructs the two finest levels of a component in one pass (`filter_levels_pair`), and each worker first streams the rows of the coarser level its strip reads into its band scratch. That scratch is the wider level's worker scratch plus those LL rows per concurrent strip, but the build plan and the planes floor charged one level's worker scratch only, so the LL rows went uncharged (at 4096 x 4096 with 4 workers, 8.6 MB). `apply` also still reserved an intermediate buffer for the second-finest level, which the paired pass reads from its sub-bands or the third-finest level and never writes. - One layout, `PairScratch`, sizes the paired band scratch for both the pass and `planned_pair_band_scratch_elements`, and one planning function, `idwt::planned_workspace_elements`, gives the plan and the planes floor the component's IDWT workspace: level by level, the finest level's output, the second-finest level's buffer and the finest level's band scratch; when the levels pair, the output, the third-finest level's buffer and the pair's band scratch. A serial decode or a code-block decoder hook keeps the level passes, so a component whose levels pair is planned for the larger of the two. - `apply` reserves the intermediate buffer for the level written before the last, which is the third-finest level below a paired pass. Where the LL rows are the larger term (small strips on many workers), the charge grows to what the IDWT holds; elsewhere it is unchanged and the IDWT holds less within it: 1024 x 768 gray on 4 threads holds 3.9 MB instead of 4.5 MB against a 4.1 MB charge, on 8 threads exactly its 4.2 MB charge. The pass itself is unchanged and output is bit-identical. On an 18-core ARM64 machine, 7 interleaved rounds at 1, 6, 18 and the default thread count: every full-image, scaled and 4096 x 2048 mosaic decode measured is within noise (median ratios 0.971 to 1.013, the same spread as between the 18-thread and default runs), with identical output. The mosaics, which pair their levels, peak at 1.5 MiB (1.3%) less heap. Tests: the planned paired band scratch equals what the pass grows to on pools of 2 to 18 threads; real paired decodes hold no more than the plan charges, keep only the level below the pair in the intermediate buffer and match the serial decode; and a paired tile's build is admitted exactly at the budget, refused before reserving anything one byte over, with the admitted decode's IDWT within its charge. --- .../src/j2c/build/allocation/plan.rs | 38 ++-- .../src/j2c/build/allocation/tests.rs | 193 +++++++++++++++++- crates/j2k-native/src/j2c/idwt.rs | 11 +- crates/j2k-native/src/j2c/idwt/banded.rs | 164 +++++++++++++-- crates/j2k-native/src/j2c/idwt/orchestrate.rs | 81 ++++++-- 5 files changed, 429 insertions(+), 58 deletions(-) diff --git a/crates/j2k-native/src/j2c/build/allocation/plan.rs b/crates/j2k-native/src/j2c/build/allocation/plan.rs index 9a6839099..663394d94 100644 --- a/crates/j2k-native/src/j2c/build/allocation/plan.rs +++ b/crates/j2k-native/src/j2c/build/allocation/plan.rs @@ -6,7 +6,8 @@ use super::super::{PrecinctSelection, PrecinctUse}; use super::{BuildWorkspace, DecompositionAllocationPlan}; use crate::error::{DecodingError, Result, ValidationError}; use crate::j2c::build::SubBandType; -use crate::j2c::idwt::idwt_buffer_size; +use crate::j2c::codestream::WaveletTransform; +use crate::j2c::idwt::planned_workspace_elements; use crate::j2c::rect::IntRect; use crate::j2c::roi::RoiPlan; use crate::j2c::tile::{ResolutionTile, Tile}; @@ -74,6 +75,7 @@ pub(super) fn build_allocation_plan( ); let mut idwt_tracker = IdwtWorkspaceTracker::new( workspace.active_decomposition_count(total_decomposition_count), + component_tile.component_info.wavelet_transform(), ); let mut observed_decomposition_count = 0usize; for resolution_tile in resolution_tiles { @@ -453,16 +455,23 @@ fn checked_rect_elements(rect: IntRect) -> Result { #[derive(Clone, Copy)] pub(super) struct IdwtWorkspaceTracker { active_decomposition_count: Option, + transform: WaveletTransform, observed_count: usize, + third_highest_active_rect: Option, second_highest_active_rect: Option, highest_active_rect: Option, } impl IdwtWorkspaceTracker { - pub(super) fn new(active_decomposition_count: Option) -> Self { + pub(super) fn new( + active_decomposition_count: Option, + transform: WaveletTransform, + ) -> Self { Self { active_decomposition_count, + transform, observed_count: 0, + third_highest_active_rect: None, second_highest_active_rect: None, highest_active_rect: None, } @@ -473,6 +482,7 @@ impl IdwtWorkspaceTracker { .active_decomposition_count .is_some_and(|count| self.observed_count < count) { + self.third_highest_active_rect = self.second_highest_active_rect; self.second_highest_active_rect = self.highest_active_rect; self.highest_active_rect = Some(rect); } @@ -484,22 +494,14 @@ impl IdwtWorkspaceTracker { return Ok(None); } let workspace = match self.highest_active_rect { - Some(highest_rect) => { - let output = idwt_buffer_size(highest_rect)?.1; - let scratch = self - .second_highest_active_rect - .map(|rect| idwt_buffer_size(rect).map(|(_, maximum)| maximum)) - .transpose()? - .unwrap_or(0); - let band_scratch = crate::j2c::idwt::planned_band_scratch_elements( - highest_rect.width() as usize, - highest_rect.height() as usize, - )?; - output - .checked_add(scratch) - .and_then(|elements| elements.checked_add(band_scratch)) - .ok_or(ValidationError::ImageTooLarge)? - } + Some(highest_rect) => planned_workspace_elements( + highest_rect, + [ + self.second_highest_active_rect, + self.third_highest_active_rect, + ], + self.transform, + )?, None => checked_rect_elements(ll_rect)?, }; Ok(Some(workspace)) diff --git a/crates/j2k-native/src/j2c/build/allocation/tests.rs b/crates/j2k-native/src/j2c/build/allocation/tests.rs index 4ec4f3731..b5285a0a2 100644 --- a/crates/j2k-native/src/j2c/build/allocation/tests.rs +++ b/crates/j2k-native/src/j2c/build/allocation/tests.rs @@ -8,6 +8,7 @@ use super::{ }; use crate::error::{DecodeError, ValidationError}; use crate::j2c::build::{Decomposition, SubBand, SubBandType}; +use crate::j2c::codestream::WaveletTransform; use crate::j2c::decode::{DecodeAllocationBudget, DecompositionStorage}; use crate::j2c::rect::IntRect; use crate::j2c::tag_tree::TagTree; @@ -188,25 +189,211 @@ fn skipped_resolution_idwt_workspace_uses_padded_active_high_water() { IntRect::from_xywh(0, 0, 16, 9), ]; - let mut skipped_one = IdwtWorkspaceTracker::new(Some(2)); + let mut skipped_one = IdwtWorkspaceTracker::new(Some(2), WaveletTransform::Irreversible97); for rect in rects { skipped_one.observe(rect); } assert_eq!(skipped_one.finish(ll).unwrap(), Some(54 + 20)); - let mut full = IdwtWorkspaceTracker::new(Some(3)); + let mut full = IdwtWorkspaceTracker::new(Some(3), WaveletTransform::Irreversible97); for rect in rects { full.observe(rect); } assert_eq!(full.finish(ll).unwrap(), Some(170 + 54)); - let mut ll_only = IdwtWorkspaceTracker::new(Some(0)); + let mut ll_only = IdwtWorkspaceTracker::new(Some(0), WaveletTransform::Irreversible97); for rect in rects { ll_only.observe(rect); } assert_eq!(ll_only.finish(ll).unwrap(), Some(4)); } +/// A one-component image of noise, `levels` decompositions deep. +#[cfg(feature = "parallel")] +fn gray_noise_codestream(width: u32, height: u32, levels: u8, reversible: bool) -> Vec { + let mut state = 0x9e37_79b9_u32; + let pixels: Vec = (0..width * height) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state.to_le_bytes()[3] + }) + .collect(); + let options = crate::EncodeOptions { + reversible, + num_decomposition_levels: levels, + ..crate::EncodeOptions::default() + }; + crate::encode(&pixels, width, height, 1, 8, false, &options).expect("encode fixture") +} + +/// Images whose two finest IDWT levels `idwt::apply` reconstructs as one +/// pass on a pool of 2, 4 or 8 threads, with and without a level below the +/// pair, for both transforms. +#[cfg(feature = "parallel")] +const PAIRED_IDWT_IMAGES: [(u32, u32, u8, bool); 4] = [ + (512, 512, 2, false), + (600, 700, 3, true), + (1024, 768, 4, false), + (777, 1100, 3, false), +]; + +/// The paired pass writes only the finer level, and each worker streams +/// the coarser level's rows it reads into its band scratch: what the IDWT +/// holds is the finest level's output, that band scratch, and a ping-pong +/// buffer for the level below the pair alone. The plan must charge all of +/// it, and the output is the serial level-by-level decode's. +#[cfg(feature = "parallel")] +#[test] +fn paired_idwt_levels_hold_no_more_than_the_plan_charges() { + use crate::j2c::build::CoefficientLayout; + use crate::j2c::idwt::idwt_buffer_size; + use crate::{CpuDecodeParallelism, DecodeSettings, DecoderContext, Image}; + + for (width, height, levels, reversible) in PAIRED_IDWT_IMAGES { + let bytes = gray_noise_codestream(width, height, levels, reversible); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse fixture"); + let mut serial_context = DecoderContext::default(); + serial_context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + let serial = image + .decode_with_context(&mut serial_context) + .expect("serial decode") + .data; + for threads in [2, 4, 8] { + let label = format!("{width}x{height}, {levels} levels, {threads} threads"); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("pool"); + pool.install(|| { + let tiles = crate::j2c::tile::parse( + &mut crate::reader::BitReader::new(image.codestream), + &image.header, + 0, + ) + .expect("parse tile"); + let (_, charged) = super::plan::build_allocation_plan( + &tiles[0], + 0, + false, + false, + BuildWorkspace::DecodePixels { + skipped_resolution_levels: 0, + coefficient_layout: CoefficientLayout::Component, + }, + None, + None, + ) + .expect("plan"); + let level_rects: Vec = tiles[0] + .component_tiles() + .next() + .expect("component tile") + .resolution_tiles() + .map(|resolution| resolution.rect) + .collect(); + let mut context = DecoderContext::default(); + let paired = image + .decode_with_context(&mut context) + .expect("paired decode") + .data; + assert_eq!(paired, serial, "{label}"); + let tile_ctx = &context.tile_decode_context; + let held = tile_ctx.idwt_capacity_bytes().expect("IDWT bytes"); + assert!( + held <= charged * size_of::(), + "{label}: the IDWT holds {held} bytes, the plan charged {}", + charged * size_of::() + ); + // Only the levels below the pair go through the ping-pong + // buffer, and the last of them is the third-finest level. + let below_pair = usize::from(levels) + .checked_sub(2) + .filter(|&index| index > 0) + .map_or(0, |index| idwt_buffer_size(level_rects[index]).unwrap().1); + assert_eq!( + tile_ctx.idwt_scratch_buffer.capacity(), + below_pair, + "{label}: ping-pong buffer" + ); + }); + } + } +} + +/// The build of a tile whose two finest IDWT levels pair admits it exactly +/// at the budget: from a retained baseline of the budget less its charge, +/// with one byte more refused before any storage is reserved. The IDWT part +/// of that charge is what the admitted decode's IDWT then holds at most. +#[cfg(feature = "parallel")] +#[test] +fn paired_idwt_tile_fits_exactly_the_budget_it_is_charged() { + use crate::j2c::build::CoefficientLayout; + use crate::{DecodeSettings, DecoderContext, Image}; + + let bytes = gray_noise_codestream(1024, 768, 4, false); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("parse fixture"); + let tiles = crate::j2c::tile::parse( + &mut crate::reader::BitReader::new(image.codestream), + &image.header, + 0, + ) + .expect("parse tile"); + let pixels = BuildWorkspace::DecodePixels { + skipped_resolution_levels: 0, + coefficient_layout: CoefficientLayout::Component, + }; + let build = |baseline: usize, workspace| { + let mut storage = DecompositionStorage::default(); + let result = + crate::j2c::build::build(&tiles[0], &mut storage, baseline, false, workspace, None); + (result, storage) + }; + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .expect("pool"); + pool.install(|| { + let (planned, storage) = build(0, pixels); + planned.expect("plan from an empty baseline"); + let charge = storage.structural_workspace_bytes; + + let (fits, storage) = build(DEFAULT_MAX_DECODE_BYTES - charge, pixels); + fits.expect("a tile that fits exactly"); + assert_eq!(storage.structural_workspace_bytes, DEFAULT_MAX_DECODE_BYTES); + + let (refused, storage) = build(DEFAULT_MAX_DECODE_BYTES - charge + 1, pixels); + assert_eq!( + refused, + Err(DecodeError::Validation(ValidationError::ImageTooLarge)) + ); + assert_eq!( + ( + storage.coefficients.capacity(), + storage.code_blocks.capacity() + ), + (0, 0), + "refused before allocating" + ); + + let (_, without_idwt) = build(0, BuildWorkspace::CoefficientsOnly); + let idwt_charge = charge - without_idwt.structural_workspace_bytes; + let mut context = DecoderContext::default(); + image + .decode_with_context(&mut context) + .expect("paired decode"); + let held = context + .tile_decode_context + .idwt_capacity_bytes() + .expect("IDWT bytes"); + assert!( + held <= idwt_charge, + "the IDWT holds {held} bytes, the plan charged {idwt_charge}" + ); + }); +} + #[test] fn impossible_metadata_count_rejects_at_the_logical_preflight() { let mut plan = coefficient_plan(0, 0); diff --git a/crates/j2k-native/src/j2c/idwt.rs b/crates/j2k-native/src/j2c/idwt.rs index 0d0647c4d..5e57feace 100644 --- a/crates/j2k-native/src/j2c/idwt.rs +++ b/crates/j2k-native/src/j2c/idwt.rs @@ -16,18 +16,17 @@ mod orchestrate; mod roi; mod vertical; -pub(crate) use banded::{ - banded_scratch_elements_for_execution, pack_last_level, planned_band_scratch_elements, - LastLevel, -}; +pub(crate) use banded::{banded_scratch_elements_for_execution, pack_last_level, LastLevel}; pub(in crate::j2c) use banded::{lift_rows, scale_samples}; pub(crate) use direct::apply_codestream_single_decomposition_idwt_job; pub(crate) use direct::apply_single_decomposition_idwt_job; #[cfg(test)] pub(crate) use horizontal::test_irreversible_filter_97i; pub(crate) use lines::{LineReconstruction, SubBandRows}; -pub(crate) use model::{idwt_buffer_size, IDWTOutput, IdwtPlacement}; -pub(crate) use orchestrate::apply; +#[cfg(all(test, feature = "parallel"))] +pub(crate) use model::idwt_buffer_size; +pub(crate) use model::{IDWTOutput, IdwtPlacement}; +pub(crate) use orchestrate::{apply, planned_workspace_elements}; pub(crate) use roi::{last_level_fits_channels, roi_idwt_peak_elements, ComponentLines, LinePlan}; // Decoded high-pass sub-bands include JPEG 2000's orientation gain. OpenJPEG's diff --git a/crates/j2k-native/src/j2c/idwt/banded.rs b/crates/j2k-native/src/j2c/idwt/banded.rs index 9dd178e67..afa8993e8 100644 --- a/crates/j2k-native/src/j2c/idwt/banded.rs +++ b/crates/j2k-native/src/j2c/idwt/banded.rs @@ -103,6 +103,35 @@ pub(crate) fn planned_band_scratch_elements(width: usize, height: usize) -> Resu banded_scratch_elements_for_execution(width, height, true) } +/// Band scratch the build plan reserves for a component whose two finest +/// active levels `inner` and `outer`, whose LL band `inner` reconstructs, +/// `idwt::apply` runs as one pass on the pool: what [`filter_levels_pair`] +/// grows its scratch to. `None` when the levels do not pair. Planned for +/// parallel execution, as [`planned_band_scratch_elements`] is. +pub(crate) fn planned_pair_band_scratch_elements( + inner: IntRect, + outer: IntRect, + transform: WaveletTransform, +) -> Result> { + #[cfg(feature = "parallel")] + { + if !pair_runs_on_pool(inner, outer, true) { + return Ok(None); + } + let (width, height) = (outer.width() as usize, outer.height() as usize); + let Some(layout) = parallel_strip_layout(width, height, true)? else { + return Ok(None); + }; + let steps = lifting_steps(transform); + PairScratch::new(inner.width() as usize, width, layout, steps).map(|pair| Some(pair.len)) + } + #[cfg(not(feature = "parallel"))] + { + let _ = (inner, outer, transform); + Ok(None) + } +} + /// Workers that may each hold ring scratch at once. #[cfg(feature = "parallel")] fn band_workers() -> usize { @@ -500,28 +529,61 @@ fn filter_levels_pair( let Some(layout) = parallel_strip_layout(width, height, true)? else { bail!(DecodingError::CodeBlockDecodeFailure); }; - let rows_len = pair_ll_rows(layout.strip_len / width, vertical_steps(outer_level).1) - .checked_mul(inner_level.width) - .ok_or(ValidationError::ImageTooLarge)?; - let per_worker = worker_scratch_elements(width)? - .max(worker_scratch_elements(inner_level.width)?) - .checked_add(rows_len) - .ok_or(ValidationError::ImageTooLarge)?; - let scratch_len = per_worker - .checked_mul(layout.concurrent_strips) - .ok_or(ValidationError::ImageTooLarge)?; - try_grow_level_buffer(band_scratch, scratch_len)?; + let steps = lifting_steps(outer_level.transform); + let scratch = PairScratch::new(inner_level.width, width, layout, steps)?; + try_grow_level_buffer(band_scratch, scratch.len)?; dispatch!(Level::new(), simd => run_pair_strips( simd, [inner_level, outer_level], target, layout.strip_len, - &mut band_scratch[..scratch_len], - [per_worker, rows_len], + &mut band_scratch[..scratch.len], + [scratch.per_worker, scratch.rows_len], )); Ok(()) } +/// The band scratch of [`filter_levels_pair`] for an outer level `width` +/// wide on `layout`, whose LL band is a level `inner_width` wide, with +/// `steps` vertical lifting steps: every concurrent strip's worker holds the +/// LL rows its strip reads and the wider level's worker scratch. +#[cfg(feature = "parallel")] +#[derive(Clone, Copy)] +struct PairScratch { + /// One worker's elements. + per_worker: usize, + /// The LL rows at the front of each worker's elements. + rows_len: usize, + /// Every concurrent worker's elements. + len: usize, +} + +#[cfg(feature = "parallel")] +impl PairScratch { + fn new( + inner_width: usize, + width: usize, + layout: ParallelStripLayout, + steps: usize, + ) -> Result { + let rows_len = pair_ll_rows(layout.strip_len / width, steps) + .checked_mul(inner_width) + .ok_or(ValidationError::ImageTooLarge)?; + let per_worker = worker_scratch_elements(width)? + .max(worker_scratch_elements(inner_width)?) + .checked_add(rows_len) + .ok_or(ValidationError::ImageTooLarge)?; + let len = per_worker + .checked_mul(layout.concurrent_strips) + .ok_or(ValidationError::ImageTooLarge)?; + Ok(Self { + per_worker, + rows_len, + len, + }) + } +} + /// Rows of the LL band a strip of `strip_rows` rows reads, at most: its /// arrivals start and end `count` rows beyond it, two output rows per band /// row, and one band row of slack at each end. @@ -649,6 +711,16 @@ fn vertical_steps(level: Level2d<'_>) -> ([(usize, VerticalStep); MAX_HALO_ROWS] } } +/// How many vertical lifting steps [`vertical_steps`] gives a level of +/// `transform`. +#[cfg(feature = "parallel")] +const fn lifting_steps(transform: WaveletTransform) -> usize { + match transform { + WaveletTransform::Reversible53 => HALO_ROWS_53, + WaveletTransform::Irreversible97 => HALO_ROWS_97, + } +} + /// Synthesize level row `index` horizontally into `row`, and for 9-7I apply /// its whole-row scaling (steps 1 and 2). #[expect( @@ -1543,6 +1615,10 @@ fn synthesize_row( /// The multi-pass 5-3R synthesis of a row's pairs from `start` on into /// `tail`, through ghosted copies of the band rows' remaining samples. +#[expect( + clippy::inline_always, + reason = "the 5/3 tail is specialized with the row synthesis inside each SIMD dispatch target" +)] #[inline(always)] fn synthesize_53_tail( simd: S, @@ -2287,6 +2363,10 @@ mod tests { /// and size. #[cfg(feature = "parallel")] #[test] + #[expect( + clippy::similar_names, + reason = "paired LL, HL, LH, and HH band names follow JPEG 2000 specification notation" + )] fn level_pair_matches_two_level_passes() { let pool = rayon::ThreadPoolBuilder::new() .num_threads(4) @@ -2361,4 +2441,62 @@ mod tests { } } } + + /// The band scratch the build plan charges for two paired levels is the + /// scratch their pass grows to, whatever the pool, transform and size. + #[cfg(feature = "parallel")] + #[test] + #[expect( + clippy::similar_names, + reason = "paired LL, HL, LH, and HH band names follow JPEG 2000 specification notation" + )] + fn level_pair_band_scratch_is_what_the_plan_charges() { + for threads in [2, 3, 4, 8, 18] { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("pool"); + for transform in [ + WaveletTransform::Reversible53, + WaveletTransform::Irreversible97, + ] { + for (x0, y0, width, height) in [ + (0, 0, 256, 300), + (3, 5, 301, 263), + (2, 1, 290, 517), + (0, 0, 640, 1300), + (1, 0, 1031, 777), + ] { + let outer_rect = IntRect::from_xywh(x0, y0, width, height); + let inner_rect = packed_band_rects(outer_rect)[0]; + let zeros = |rect: IntRect| { + packed_band_rects(rect) + .map(|band| alloc::vec![0.0; (band.width() * band.height()) as usize]) + }; + let [ll, hl, lh, hh] = zeros(inner_rect); + let [_, outer_hl, outer_lh, outer_hh] = zeros(outer_rect); + let levels = [ + Level2d::new(inner_rect, [&ll, &hl, &lh, &hh], transform, 1.25), + Level2d::new( + outer_rect, + [&[], &outer_hl, &outer_lh, &outer_hh], + transform, + 1.25, + ), + ]; + let label = format!("{threads} threads, {transform:?} {outer_rect:?}"); + let mut target = + alloc::vec![0.0; (outer_rect.width() * outer_rect.height()) as usize]; + let mut scratch = Vec::new(); + let charged = pool.install(|| { + assert!(pair_runs_on_pool(inner_rect, outer_rect, true), "{label}"); + filter_levels_pair(levels, &mut target, &mut scratch).expect("level pair"); + planned_pair_band_scratch_elements(inner_rect, outer_rect, transform) + .expect("planned scratch") + }); + assert_eq!(Some(scratch.len()), charged, "{label}"); + } + } + } + } } diff --git a/crates/j2k-native/src/j2c/idwt/orchestrate.rs b/crates/j2k-native/src/j2c/idwt/orchestrate.rs index 8127901f0..7b542d96c 100644 --- a/crates/j2k-native/src/j2c/idwt/orchestrate.rs +++ b/crates/j2k-native/src/j2c/idwt/orchestrate.rs @@ -4,13 +4,15 @@ use super::super::codestream::WaveletTransform; use super::super::decode::{DecompositionStorage, TileDecodeContext}; #[cfg(feature = "parallel")] use super::banded::{filter_2d_banded_pair, pair_runs_on_pool}; +use super::banded::{planned_band_scratch_elements, planned_pair_band_scratch_elements}; use super::direct::{apply_level, apply_level_into}; use super::interleave_i64::apply_level_i64; use super::model::{ idwt_buffer_size, IDWTInput, IDWTInputI64, IDWTTempOutput, IdwtPlacement, InputSource, }; use super::roi::apply_roi; -use crate::error::{bail, DecodingError}; +use crate::error::{bail, DecodingError, ValidationError}; +use crate::j2c::rect::IntRect; use crate::j2c::Header; use crate::{ checked_decode_usize_product2, try_reserve_decode_elements, HtCodeBlockDecoder, Result, @@ -92,23 +94,6 @@ pub(crate) fn apply( return Ok(()); } - // The coefficient array will always be the one that holds the coefficients - // from the highest decomposition. Therefore, reserve as much. - let (s_min, s_max) = idwt_buffer_size(decompositions.last().unwrap().rect)?; - if prefix.is_none() && output.coefficients.len() < s_min { - try_reserve_decode_elements(&mut output.coefficients, s_max)?; - } - - if decompositions.len() > 1 { - // Due to the above, the intermediate buffer will never need more than - // the second-highest decomposition. - let (s_min, s_max) = idwt_buffer_size(decompositions[decompositions.len() - 2].rect)?; - - if scratch_buf.len() < s_min { - try_reserve_decode_elements(scratch_buf, s_max)?; - } - } - // The two finest levels may run as one pass on the pool, which writes // only the finer level. #[cfg(feature = "parallel")] @@ -124,6 +109,24 @@ pub(crate) fn apply( let pair = false; let writes = decompositions.len() - usize::from(pair); + // The coefficient array will always be the one that holds the coefficients + // from the highest decomposition. Therefore, reserve as much. + let (s_min, s_max) = idwt_buffer_size(decompositions.last().unwrap().rect)?; + if prefix.is_none() && output.coefficients.len() < s_min { + try_reserve_decode_elements(&mut output.coefficients, s_max)?; + } + + if writes > 1 { + // Due to the above, the intermediate buffer will never need more than + // the level written before the last: the second-highest + // decomposition, or the third-highest below a paired pass. + let (s_min, s_max) = idwt_buffer_size(decompositions[writes - 2].rect)?; + + if scratch_buf.len() < s_min { + try_reserve_decode_elements(scratch_buf, s_max)?; + } + } + // Determine which buffer we should use first, such that the `coefficients` // array will always hold the final values. let mut use_scratch = writes.is_multiple_of(2); @@ -270,6 +273,48 @@ pub(crate) fn apply( Ok(()) } +/// The `f32` IDWT workspace [`apply`] holds at most for a component whose +/// finest active level is `highest`, above `second` and `third` when it has +/// them, planned for the pool: the finest level's output, the band scratch, +/// and the intermediate buffer of the level written before the last. Level +/// by level, that is `second` with the finest level's band scratch. When +/// the two finest levels pair (see [`planned_pair_band_scratch_elements`]), +/// their one pass writes only the finest level, so the buffer holds `third` +/// and the band scratch is the pair's. A serial decode or a code-block +/// decoder hook keeps the level passes, so a component whose levels pair is +/// planned for the larger of both. +pub(crate) fn planned_workspace_elements( + highest: IntRect, + [second, third]: [Option; 2], + transform: WaveletTransform, +) -> Result { + let padded = |rect: Option| { + rect.map_or(Ok(0), |rect| { + idwt_buffer_size(rect).map(|(_, maximum)| maximum) + }) + }; + let total = |elements: [usize; 3]| { + elements + .into_iter() + .try_fold(0_usize, usize::checked_add) + .ok_or(ValidationError::ImageTooLarge) + }; + let output = padded(Some(highest))?; + let band_scratch = + planned_band_scratch_elements(highest.width() as usize, highest.height() as usize)?; + let levels = total([output, padded(second)?, band_scratch])?; + let pair_band_scratch = match second { + Some(inner) => planned_pair_band_scratch_elements(inner, highest, transform)?, + None => None, + }; + match pair_band_scratch { + Some(pair_band_scratch) => { + Ok(levels.max(total([output, padded(third)?, pair_band_scratch])?)) + } + None => Ok(levels), + } +} + fn apply_i64( storage: &DecompositionStorage<'_>, tile_ctx: &mut TileDecodeContext, From 475302d02b21501cdf9bd1667e485ffb2b3a3223 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 17:31:18 -0400 Subject: [PATCH 36/53] docs(changelog): note the budget charge of the paired IDWT levels --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f531b7d33..4be7e8a71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,14 @@ and stale roadmap entries have been removed from the public documentation set. regions included, left it unwritten and still returned `Ok`. 8-bit region decodes, and buffers with any bytes after the last row, were not affected. The other rows are unchanged. +- Decodes on the worker pool that reconstruct the two finest wavelet levels + of a component in one pass now charge the decode budget for what that pass + holds: each worker's rows of the coarser level, which were not charged, + and no intermediate buffer for the coarser level, which the pass never + writes and which is no longer reserved. Where those rows are the larger + term the charge grows to match; elsewhere it is unchanged and the IDWT + holds less within it (a 1024 x 768 gray image on 4 threads: 3.9 MB in + place of 4.5 MB, against a 4.1 MB charge). Output is unchanged. - Parallel CPU region decodes on the row-streamed IDWT decode Tier-1 band by band on the worker pool while the IDWT reads the bands, instead of decoding every code-block of a component first. Each sub-band holds three bands at From 1c2236d6f501f2194c82bd7ce2ac499c9fa7b934 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:34:31 -0400 Subject: [PATCH 37/53] perf(j2k-native): round and pack the x86-64 inverse ICT with SIMD An x86-64 build targets baseline x86-64, which has no SSE4.1 rounding or FMA instructions, so outside a runtime SIMD dispatch every `round_ties_even` is a call to `rintf` and every `mul_add` a call to `fmaf`. The centered rounding after the inverse ICT was such a scalar loop: one library call per sample. - `round_centered_and_shift` runs at the CPU's SIMD level on x86-64 (eight samples per AVX2 instruction). - The last-level RGB8 pack fuses the inverse ICT, the rounding and the interleave at the SIMD level on x86-64, as ARM64's scalar loop already does through auto-vectorization. The fused pack is kept out of line so the per-row emit closure still inlines into the dispatched row streamer (an outlined closure loses the dispatch's target features). Output is byte-identical. Measured on x86-64 (Zen 3, AVX2), median of per-round ratios over 5 interleaved rounds against the base build: HT 2 bpp 4096x2048 full decode -31.1% at 1 thread and -15.5% at 16, HT 512x512 tiles -25.4%, 18000x12000 HT overview (reduce 2) -18.7% at 1 thread, HT 1024x1024 region -12.1%, classic 2 bpp -12.2% at 1 thread. Lossless rows unchanged (within the A/A spread). --- crates/j2k-native/src/color/mod.rs | 2 + crates/j2k-native/src/color/packing.rs | 80 +++++++--- crates/j2k-native/src/j2c/mct.rs | 197 ++++++++++++++++++++++++- 3 files changed, 255 insertions(+), 24 deletions(-) diff --git a/crates/j2k-native/src/color/mod.rs b/crates/j2k-native/src/color/mod.rs index ba37e4e28..7e494a759 100644 --- a/crates/j2k-native/src/color/mod.rs +++ b/crates/j2k-native/src/color/mod.rs @@ -23,6 +23,8 @@ pub use output_planes::{ DecodedComponentsParts, DecodedNativeComponents, DecodedNativeComponentsParts, NativeComponentPlane, NativeComponentPlaneParts, }; +#[cfg(all(feature = "simd", target_arch = "x86_64"))] +pub(crate) use packing::store_rgb8_x8; pub(crate) use packing::{ interleave_and_convert_after_mct, interleave_and_convert_region, validate_interleaved_output_buffer, diff --git a/crates/j2k-native/src/color/packing.rs b/crates/j2k-native/src/color/packing.rs index 682d076a5..f30414294 100644 --- a/crates/j2k-native/src/color/packing.rs +++ b/crates/j2k-native/src/color/packing.rs @@ -331,31 +331,19 @@ fn round_to_u8(sample: f32) -> u8 { #[cfg(all(feature = "simd", target_arch = "x86_64"))] #[inline(always)] fn pack_rgb8_simd(simd: S, planes: [&[f32]; 3], output: &mut [u8]) { - use fearless_simd::{f32x8, u32x8, u8x16, Bytes, Select, SimdBase, SimdFloat}; + use fearless_simd::{f32x8, SimdBase}; - let zero = f32x8::splat(simd, 0.0); - let max = f32x8::splat(simd, 255.0); - let half = f32x8::splat(simd, 0.5); - let one = f32x8::splat(simd, 1.0); - let indices = u8x16::from_slice(simd, &[0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0, 0, 0, 0]); let vector_len = planes[0].len() / 8 * 8; for start in (0..vector_len).step_by(8) { - let channels = planes.map(|plane| { - let value = f32x8::from_slice(simd, &plane[start..start + 8]); - // Match Rust's saturating float-to-u8 cast, including NaN -> 0. - let value = value.simd_ge(zero).select(value, zero).min(max); - let integral = value.floor(); - // Adding 0.5 before rounding would incorrectly round the float - // immediately below 0.5 up to 1. Compare the fractional part. - let rounded = integral + (value - integral).simd_ge(half).select(one, zero); - simd.cvt_u32_f32x8(rounded) - }); - let packed: u32x8 = channels[0] | (channels[1] << 8) | (channels[2] << 16); - let (first, second) = simd.split_u32x8(packed); - for (offset, words) in [(0, first), (12, second)] { - let bytes = words.bitcast::>().swizzle_dyn(indices).to_array(); - output[start * 3 + offset..start * 3 + offset + 12].copy_from_slice(&bytes[..12]); - } + store_rgb8_x8( + simd, + [ + f32x8::from_slice(simd, &planes[0][start..start + 8]), + f32x8::from_slice(simd, &planes[1][start..start + 8]), + f32x8::from_slice(simd, &planes[2][start..start + 8]), + ], + &mut output[start * 3..start * 3 + 24], + ); } for index in vector_len..planes[0].len() { for channel in 0..3 { @@ -371,6 +359,54 @@ fn pack_rgb8_simd(simd: S, planes: [&[f32]; 3], output: } } +/// Round and saturate eight pixels' red, green and blue samples as the +/// scalar 8-bit cast does, and store them interleaved in the first 24 bytes +/// of `output`. +#[cfg(all(feature = "simd", target_arch = "x86_64"))] +#[inline(always)] +pub(crate) fn store_rgb8_x8( + simd: S, + [red, green, blue]: [fearless_simd::f32x8; 3], + output: &mut [u8], +) { + use fearless_simd::{u32x8, u8x16, Bytes, SimdBase}; + + let indices = u8x16::from_slice(simd, &[0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0, 0, 0, 0]); + let packed: u32x8 = quantize_u8_x8(simd, red) + | (quantize_u8_x8(simd, green) << 8) + | (quantize_u8_x8(simd, blue) << 16); + let (first, second) = simd.split_u32x8(packed); + for (offset, words) in [(0, first), (12, second)] { + let bytes = words.bitcast::>().swizzle_dyn(indices).to_array(); + output[offset..offset + 12].copy_from_slice(&bytes[..12]); + } +} + +/// [`round_to_u8`] of eight samples, one byte value per lane. +#[cfg(all(feature = "simd", target_arch = "x86_64"))] +#[inline(always)] +fn quantize_u8_x8( + simd: S, + value: fearless_simd::f32x8, +) -> fearless_simd::u32x8 { + use fearless_simd::{f32x8, Select, SimdBase, SimdFloat}; + + let zero = f32x8::splat(simd, 0.0); + // Match Rust's saturating float-to-u8 cast, including NaN -> 0. + let value = value + .simd_ge(zero) + .select(value, zero) + .min(f32x8::splat(simd, 255.0)); + let integral = value.floor(); + // Adding 0.5 before rounding would incorrectly round the float + // immediately below 0.5 up to 1. Compare the fractional part. + let rounded = integral + + (value - integral) + .simd_ge(f32x8::splat(simd, 0.5)) + .select(f32x8::splat(simd, 1.0), zero); + simd.cvt_u32_f32x8(rounded) +} + pub(crate) fn interleave_and_convert_region( image: &mut DecodedImage<'_, '_>, image_width: usize, diff --git a/crates/j2k-native/src/j2c/mct.rs b/crates/j2k-native/src/j2c/mct.rs index ad6e30278..206ef517e 100644 --- a/crates/j2k-native/src/j2c/mct.rs +++ b/crates/j2k-native/src/j2c/mct.rs @@ -194,8 +194,8 @@ impl DeferredMct { /// costs less than the copy. With a last level, the planes hold its /// high bands and the next component's LL band, with `first_ll` kept /// separately. Each reconstructed row is transformed and packed. On - /// hosted ARM64 with SIMD, 9/7 rows take the fused ICT pack instead of - /// `pack`, so `pack` must be the standard RGB8 pack there. + /// hosted ARM64 and x86-64 with SIMD, 9/7 rows take the fused ICT pack + /// instead of `pack`, so `pack` must be the standard RGB8 pack there. pub(crate) fn pack_rgb8( self, [r, g, b]: [&mut [f32]; 3], @@ -215,6 +215,11 @@ impl DeferredMct { output, parallel, |[r, g, b], output| { + #[cfg(all(feature = "simd", target_arch = "x86_64"))] + if self.transform == WaveletTransform::Irreversible97 { + self.pack_ict_rgb8_x86([r, g, b], output); + return; + } if cfg!(all( feature = "std", feature = "simd", @@ -304,6 +309,82 @@ impl DeferredMct { } } + /// [`Self::pack_ict_rgb8`] at the CPU's SIMD level. Outside a dispatch, + /// an x86-64 build has no FMA or rounding instructions, so the scalar + /// loop would call a library routine for every sample. Kept out of line + /// so the per-row closure of [`Self::pack_rgb8`] stays small enough to + /// inline into the dispatched row streamer; an outlined closure loses + /// the dispatch's target features. + #[cfg(all(feature = "simd", target_arch = "x86_64"))] + #[inline(never)] + fn pack_ict_rgb8_x86(self, planes: [&[f32]; 3], output: &mut [u8]) { + if self.rounding.is_some() { + dispatch!(Level::new(), simd => self.pack_ict_rgb8_simd::(simd, planes, output)); + } else { + dispatch!(Level::new(), simd => self.pack_ict_rgb8_simd::(simd, planes, output)); + } + } + + /// Eight pixels at a time, the expressions of [`apply_inner_impl`] and + /// [`round_centered_and_shift`], then the RGB8 pack; the scalar + /// [`Self::pack_ict_rgb8`] takes the remainder. + #[cfg(all(feature = "simd", target_arch = "x86_64"))] + #[inline(always)] + fn pack_ict_rgb8_simd( + self, + simd: S, + [luma, cb, cr]: [&[f32]; 3], + output: &mut [u8], + ) { + use fearless_simd::{f32x8, SimdBase, SimdFloat}; + + let len = (output.len() / 3) + .min(luma.len()) + .min(cb.len()) + .min(cr.len()); + let vector_len = len / 8 * 8; + let red_from_chroma = f32x8::splat(simd, mct::ICT_INV_R_CR); + let green_from_red_chroma = f32x8::splat(simd, mct::ICT_INV_G_CR); + let green_from_blue_chroma = f32x8::splat(simd, mct::ICT_INV_G_CB); + let blue_from_chroma = f32x8::splat(simd, mct::ICT_INV_B_CB); + let red_level = f32x8::splat(simd, self.addends[0]); + let green_level = f32x8::splat(simd, self.addends[1]); + let blue_level = f32x8::splat(simd, self.addends[2]); + let shift = self.rounding.unwrap_or([0.0; 3]); + let red_shift = f32x8::splat(simd, shift[0]); + let green_shift = f32x8::splat(simd, shift[1]); + let blue_shift = f32x8::splat(simd, shift[2]); + for start in (0..vector_len).step_by(8) { + let y_0 = f32x8::from_slice(simd, &luma[start..start + 8]); + let y_1 = f32x8::from_slice(simd, &cb[start..start + 8]); + let y_2 = f32x8::from_slice(simd, &cr[start..start + 8]); + let mut red = y_2.mul_add_precise(red_from_chroma, y_0) + red_level; + let mut green = y_2.mul_add_precise( + green_from_red_chroma, + y_1.mul_add_precise(green_from_blue_chroma, y_0), + ) + green_level; + let mut blue = y_1.mul_add_precise(blue_from_chroma, y_0) + blue_level; + if CENTERED { + red = red.round_ties_even() + red_shift; + green = green.round_ties_even() + green_shift; + blue = blue.round_ties_even() + blue_shift; + } + crate::color::store_rgb8_x8( + simd, + [red, green, blue], + &mut output[3 * start..3 * start + 24], + ); + } + self.pack_ict_rgb8::( + [ + &luma[vector_len..len], + &cb[vector_len..len], + &cr[vector_len..len], + ], + &mut output[3 * vector_len..3 * len], + ); + } + fn transform_chunk(self, planes: (&mut [f32], &mut [f32], &mut [f32])) { transform_planes(self.transform, planes, self.addends, self.rounding, false); } @@ -345,11 +426,31 @@ fn transform_planes( } fn round_centered_and_shift(samples: &mut [f32], addend: f32) { + // Outside a dispatch an x86-64 build rounds each sample with a library + // call; the SIMD level rounds eight at a time with the same result. + #[cfg(all(feature = "simd", target_arch = "x86_64"))] + dispatch!(Level::new(), simd => round_centered_and_shift_simd(simd, samples, addend)); + #[cfg(not(all(feature = "simd", target_arch = "x86_64")))] for sample in samples { *sample = round_ties_even_then_add(*sample, addend); } } +#[cfg(all(feature = "simd", target_arch = "x86_64"))] +#[inline(always)] +fn round_centered_and_shift_simd(simd: S, samples: &mut [f32], addend: f32) { + use fearless_simd::{f32x8, SimdBase, SimdFloat}; + + let shift = f32x8::splat(simd, addend); + let (chunks, rest) = samples.as_chunks_mut::<8>(); + for chunk in chunks { + (f32x8::from_slice(simd, chunk).round_ties_even() + shift).store_slice(chunk); + } + for sample in rest { + *sample = round_ties_even_then_add(*sample, addend); + } +} + fn apply_inverse_i64( transform: WaveletTransform, s0: &mut super::ComponentData, @@ -731,6 +832,78 @@ mod tests { }; centered.pack_ict_rgb8::([&[y0; 9], &[y1; 9], &[y2; 9]], &mut packed); assert_eq!(packed, [128; 27]); + + // Nine pixels: one SIMD block and a scalar remainder. + #[cfg(all(feature = "simd", target_arch = "x86_64"))] + { + deferred.pack_ict_rgb8_x86([&[y0; 9], &[y1; 9], &[y2; 9]], &mut packed); + assert_eq!(packed, [1; 27]); + centered.pack_ict_rgb8_x86([&[y0; 9], &[y1; 9], &[y2; 9]], &mut packed); + assert_eq!(packed, [128; 27]); + } + } + + #[test] + #[cfg(all(feature = "simd", target_arch = "x86_64"))] + #[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss, + reason = "the reference pack rounds and saturates to 8 bits like the RGB8 output" + )] + fn x86_fused_ict_pack_matches_transform_then_pack() { + // Ties, saturation on both sides, NaN and infinities, in SIMD blocks + // and a ragged remainder. + let mut values = vec![ + f32::NAN, + f32::INFINITY, + f32::NEG_INFINITY, + -0.0, + 0.5, + -0.5, + 1.5, + 2.5, + 254.5, + 255.5, + -127.5, + 127.5, + f32::from_bits(0x3eff_ffff), + f32::from_bits(0xc117_fffc), + ]; + let mut state = 0x2545_f491_u32; + while values.len() < 3 * 61 { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + values.push((state >> 8) as f32 / 16_384.0 - 512.0); + } + let (luma, chroma) = values.split_at(61); + let (cb, cr) = chroma.split_at(61); + for (addends, rounding) in [ + ([0.0; 3], Some([128.0; 3])), + ([128.0; 3], None), + ([0.25, -3.0, 127.5], None), + ] { + let deferred = DeferredMct { + transform: WaveletTransform::Irreversible97, + addends, + rounding, + last_level: None, + }; + let mut planes = [luma.to_vec(), cb.to_vec(), cr.to_vec()]; + let [r, g, b] = &mut planes; + deferred.transform_chunk((r.as_mut_slice(), g.as_mut_slice(), b.as_mut_slice())); + let expected: Vec = (0..luma.len()) + .flat_map(|index| { + planes + .each_ref() + .map(|plane| crate::math::round_f32(plane[index]) as u8) + }) + .collect(); + for len in [0, 7, 8, 9, 16, 61] { + let mut packed = vec![0xa5_u8; 3 * len]; + deferred.pack_ict_rgb8_x86([&luma[..len], &cb[..len], &cr[..len]], &mut packed); + assert_eq!(packed, expected[..3 * len], "{rounding:?} {len}"); + } + } } #[test] @@ -741,6 +914,26 @@ mod tests { samples.map(f32::to_bits), [119.0, 118.0, 120.0].map(f32::to_bits) ); + // A full SIMD block and a remainder round the same way. + let mut samples = [ + f32::from_bits(0xc117_fffc), + -9.5, + -8.5, + 0.5, + 1.5, + 2.5, + -0.5, + -1.5, + 254.5, + -128.5, + f32::from_bits(0x3eff_ffff), + ]; + round_centered_and_shift(&mut samples, 128.0); + assert_eq!( + samples.map(f32::to_bits), + [119.0, 118.0, 120.0, 128.0, 130.0, 130.0, 128.0, 126.0, 382.0, 0.0, 128.0] + .map(f32::to_bits) + ); let deferred = DeferredMct { transform: WaveletTransform::Irreversible97, From d9943b3fde686625e51f7de15d53613542eb62d5 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:34:31 -0400 Subject: [PATCH 38/53] perf(j2k-native): round non-MCT 9/7 planes at the SIMD level on x86-64 The rounding of 9/7 planes that are not inverse-transformed by the MCT (`SampleShift` round-then-add and the float sign shift) was a scalar `round_ties_even` loop, a `rintf` call per sample on x86-64. On x86-64 it now goes through the dispatched `round_centered_and_shift`; ARM64 keeps its single-pass scalar copy loop. Output is byte-identical. Measured on x86-64 (Zen 3, AVX2) on an HT 2 bpp 4096x2048 RGB image encoded without the colour transform, median of per-round ratios over 9 interleaved rounds against the build without this commit: -38.3% at 1 thread (-58.5% against the base build), -10.1% at 8 threads. --- crates/j2k-native/src/j2c/decode/store.rs | 23 +++++++++++++---------- crates/j2k-native/src/j2c/mct.rs | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/crates/j2k-native/src/j2c/decode/store.rs b/crates/j2k-native/src/j2c/decode/store.rs index 4334fd722..c20e146bf 100644 --- a/crates/j2k-native/src/j2c/decode/store.rs +++ b/crates/j2k-native/src/j2c/decode/store.rs @@ -13,7 +13,7 @@ use crate::j2c::build::CoefficientLayout; use crate::j2c::idwt::{IdwtPlacement, LastLevel}; use crate::j2c::rect::IntRect; use crate::j2c::roi::{output_grid_offset, output_region_rect}; -use crate::math::{round_ties_even_then_add, SIMD_WIDTH}; +use crate::math::SIMD_WIDTH; mod upsample; @@ -698,11 +698,7 @@ impl SampleShift { *sample += addend; } } - Self::RoundThenAdd(addend) => { - for sample in samples { - *sample = round_ties_even_then_add(*sample, addend); - } - } + Self::RoundThenAdd(addend) => mct::round_centered_and_shift(samples, addend), } } @@ -714,9 +710,18 @@ impl SampleShift { *output = input + addend; } } + // On x86-64 a scalar rounding loop calls a library routine per + // sample: copy, then round at the SIMD level. + #[cfg(all(feature = "simd", target_arch = "x86_64"))] + Self::RoundThenAdd(addend) => { + let len = output.len().min(input.len()); + output[..len].copy_from_slice(&input[..len]); + mct::round_centered_and_shift(&mut output[..len], addend); + } + #[cfg(not(all(feature = "simd", target_arch = "x86_64")))] Self::RoundThenAdd(addend) => { for (output, &input) in output.iter_mut().zip(input) { - *output = round_ties_even_then_add(input, addend); + *output = crate::math::round_ties_even_then_add(input, addend); } } } @@ -730,9 +735,7 @@ fn apply_float_sign_shift( round_irreversible_output: bool, ) { if round_irreversible_output && transform == WaveletTransform::Irreversible97 { - for sample in samples { - *sample = round_ties_even_then_add(*sample, sign_shift); - } + mct::round_centered_and_shift(samples, sign_shift); } else if sign_shift != 0.0 { for sample in samples { *sample += sign_shift; diff --git a/crates/j2k-native/src/j2c/mct.rs b/crates/j2k-native/src/j2c/mct.rs index 206ef517e..390e6267a 100644 --- a/crates/j2k-native/src/j2c/mct.rs +++ b/crates/j2k-native/src/j2c/mct.rs @@ -425,7 +425,7 @@ fn transform_planes( } } -fn round_centered_and_shift(samples: &mut [f32], addend: f32) { +pub(crate) fn round_centered_and_shift(samples: &mut [f32], addend: f32) { // Outside a dispatch an x86-64 build rounds each sample with a library // call; the SIMD level rounds eight at a time with the same result. #[cfg(all(feature = "simd", target_arch = "x86_64"))] From c2f8f07889b08282e54a943ba75a487d0d8631c8 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:34:31 -0400 Subject: [PATCH 39/53] perf(j2k-native): convert HT 9/7 magnitudes with the signed SIMD conversion The magnitude is masked to 31 bits, so the signed int-to-float conversion is exact and gives the same value. On x86-64 AVX2 it is one instruction; the unsigned conversion is a five-instruction emulation. Neon converts either way natively. Output is byte-identical. Measured on x86-64 (Zen 3, AVX2), median of per-round ratios over 5 interleaved rounds against the previous commit: 18000x12000 HT overviews (reduce 2/3/4) -1.3% to -2.3% at 1 and 16 threads, HT 1024x1024 regions -0.9% to -2.4% in-process; full-image HT decodes and 512x512 tiles unchanged (A/A spread 0.4%). --- crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs index 8305812be..fc1d1b631 100644 --- a/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs +++ b/crates/j2k-native/src/j2c/ht_block_decode/magnitude/simd.rs @@ -258,7 +258,10 @@ fn final_samples( ) -> f32x8 { let magnitude = samples & 0x7FFF_FFFF; let values = if IRREVERSIBLE_MIDPOINT { - let magnitude = lanes.simd.cvt_f32_u32x8(magnitude) * output.midpoint_scale; + // The magnitude is masked to 31 bits, so the signed conversion is exact and gives the + // same value; it is one instruction on x86-64, where the unsigned one is emulated. + let magnitude = + lanes.simd.cvt_f32_i32x8(magnitude.bitcast::>()) * output.midpoint_scale; let signed = magnitude.bitcast::>() | (samples & 0x8000_0000); signed.bitcast::>() } else { From 764f756c478bf48d81eb25f704c639191fb7efae Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:34:31 -0400 Subject: [PATCH 40/53] perf(j2k-native): run the packed classic Tier-1 passes inside the x86-64 SIMD dispatch On baseline x86-64 every MQ decision's `leading_zeros` is a BSR plus a zero fixup on the per-symbol dependency chain, and the renormalizing shifts go through CL. The packed-column passes now run inside the runtime dispatch on x86-64, with the passes inlined into it, so they compile with LZCNT and BMI2 where present. ARM64 is unchanged (CLZ is baseline there). Output is byte-identical. Measured on x86-64 (Zen 3, AVX2), median of per-round ratios over 5 interleaved rounds against the build without this commit: classic lossless 4096x2048 full decode -9.9% at 1 thread, -10.5% / -13.5% / -12.4% at 4 / 8 / 16; classic 2 bpp -9.2% at 1 thread, -7.0% / -6.1% / -8.6%; classic 1024x1024 regions -7.0% to -8.5% at 1 thread; 18000x12000 classic overviews (reduce 2/3/4) -9.0% to -11.1%. --- .../src/j2c/bitplane/flag_passes.rs | 24 +++++++++++++++ .../j2k-native/src/j2c/bitplane/schedule.rs | 30 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/crates/j2k-native/src/j2c/bitplane/flag_passes.rs b/crates/j2k-native/src/j2c/bitplane/flag_passes.rs index bfd52683c..5dbfb5664 100644 --- a/crates/j2k-native/src/j2c/bitplane/flag_passes.rs +++ b/crates/j2k-native/src/j2c/bitplane/flag_passes.rs @@ -139,6 +139,14 @@ impl<'c, 'd> Scan<'c, 'd> { } } +#[cfg_attr( + target_arch = "x86_64", + expect( + clippy::inline_always, + reason = "inlines into the x86-64 Tier-1 dispatch in schedule.rs" + ), + inline(always) +)] pub(super) fn significance_propagation_pass_flags( ctx: &mut BitPlaneDecodeContext, decoder: &mut ArithmeticDecoder<'_>, @@ -169,6 +177,14 @@ pub(super) fn significance_propagation_pass_flags( scan.finish(decoder); } +#[cfg_attr( + target_arch = "x86_64", + expect( + clippy::inline_always, + reason = "inlines into the x86-64 Tier-1 dispatch in schedule.rs" + ), + inline(always) +)] pub(super) fn magnitude_refinement_pass_flags( ctx: &mut BitPlaneDecodeContext, decoder: &mut ArithmeticDecoder<'_>, @@ -207,6 +223,14 @@ pub(super) fn magnitude_refinement_pass_flags( scan.finish(decoder); } +#[cfg_attr( + target_arch = "x86_64", + expect( + clippy::inline_always, + reason = "inlines into the x86-64 Tier-1 dispatch in schedule.rs" + ), + inline(always) +)] pub(super) fn cleanup_pass_flags( ctx: &mut BitPlaneDecodeContext, decoder: &mut ArithmeticDecoder<'_>, diff --git a/crates/j2k-native/src/j2c/bitplane/schedule.rs b/crates/j2k-native/src/j2c/bitplane/schedule.rs index 68da6ea88..726edf8d9 100644 --- a/crates/j2k-native/src/j2c/bitplane/schedule.rs +++ b/crates/j2k-native/src/j2c/bitplane/schedule.rs @@ -17,6 +17,8 @@ use super::observer::{J2kDecodeObserver, NoJ2kDecodeStats}; use super::state::{ extend_preallocated, push_preallocated, BitPlaneDecodeBuffers, BitPlaneDecodeContext, }; +#[cfg(target_arch = "x86_64")] +use crate::math::{dispatch, Level}; use crate::J2kCodeBlockSegment; pub(super) fn decode_inner( @@ -169,6 +171,26 @@ pub(super) fn handle_arithmetic_coding_passes( run_arithmetic_coding_passes::(start, end, ctx, decoder, observer) } +/// On x86-64 the passes run inside the runtime dispatch, so the MQ decode +/// compiles with LZCNT and BMI2 where present: on baseline x86-64 every +/// decision's `leading_zeros` is a BSR plus a zero fixup on the per-symbol +/// dependency chain, and the renormalizing shifts go through CL. ARM64 has +/// CLZ in its baseline. +#[cfg(target_arch = "x86_64")] +pub(super) fn handle_packed_arithmetic_coding_passes( + start: u8, + end: u8, + ctx: &mut BitPlaneDecodeContext, + decoder: &mut ArithmeticDecoder<'_>, + observer: &mut impl J2kDecodeObserver, +) -> Option<()> { + dispatch!(Level::new(), simd => { + let _ = simd; + run_arithmetic_coding_passes::(start, end, ctx, decoder, observer) + }) +} + +#[cfg(not(target_arch = "x86_64"))] pub(super) fn handle_packed_arithmetic_coding_passes( start: u8, end: u8, @@ -181,6 +203,14 @@ pub(super) fn handle_packed_arithmetic_coding_passes( /// Arithmetic passes over the packed column words (`PACKED_COLUMNS`) or the /// per-coefficient arrays; decisions and contexts are identical. +#[cfg_attr( + target_arch = "x86_64", + expect( + clippy::inline_always, + reason = "the passes must inline into the caller's x86-64 dispatch target" + ), + inline(always) +)] fn run_arithmetic_coding_passes( start: u8, end: u8, From 83cbe28d0d79419b802814e4031de1e0d71492e5 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:34:31 -0400 Subject: [PATCH 41/53] perf(j2k-native): store whole-number RGB8 pixels without re-rounding in the x86-64 fused ICT pack After the centered rounding the fused pack's samples are whole numbers, yet the RGB8 store re-ran the generic quantization (floor, fraction compare, select) and the unsigned float-to-int conversion, which AVX2 emulates, and copied each 12-byte half through the stack. The fused pack now only clamps like the saturating u8 cast (NaN to zero) and converts with the native signed conversion (exact on 0..=255), and stores each half with one overlapping 16-byte store when the row has room past the eight pixels; the next pixels or the scalar remainder overwrite the spare bytes. Bytes past the planes' pixels are never written. Output is byte-identical. Measured on x86-64 (Zen 3, AVX2), median of per-round ratios over 5 interleaved rounds against the build without this commit: HT 2 bpp 4096x2048 full decode -7.7% at 1 thread and -5.3% at 4, HT 512x512 tiles -4.8%, classic 2 bpp -1.3% to -1.7%. --- crates/j2k-native/src/j2c/mct.rs | 67 +++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/crates/j2k-native/src/j2c/mct.rs b/crates/j2k-native/src/j2c/mct.rs index 390e6267a..960e93f72 100644 --- a/crates/j2k-native/src/j2c/mct.rs +++ b/crates/j2k-native/src/j2c/mct.rs @@ -368,12 +368,14 @@ impl DeferredMct { red = red.round_ties_even() + red_shift; green = green.round_ties_even() + green_shift; blue = blue.round_ties_even() + blue_shift; + store_whole_rgb8_x8(simd, [red, green, blue], &mut output[3 * start..3 * len]); + } else { + crate::color::store_rgb8_x8( + simd, + [red, green, blue], + &mut output[3 * start..3 * start + 24], + ); } - crate::color::store_rgb8_x8( - simd, - [red, green, blue], - &mut output[3 * start..3 * start + 24], - ); } self.pack_ict_rgb8::( [ @@ -390,6 +392,53 @@ impl DeferredMct { } } +/// The RGB8 pack of eight pixels whose samples are whole numbers (the +/// centered rounding already ran), into the first 24 bytes of `output`. +/// Matches the saturating `as u8` cast (NaN to zero) like +/// [`crate::color::store_rgb8_x8`], but skips its rounding and uses the +/// signed conversion, which is exact on `0..=255` and native on x86-64 where +/// the unsigned one is emulated. With four spare bytes after the pixels, each +/// half is stored with one overlapping 16-byte store; the next pixels (or the +/// scalar remainder) overwrite the spare bytes. +#[cfg(all(feature = "simd", target_arch = "x86_64"))] +#[inline(always)] +fn store_whole_rgb8_x8( + simd: S, + [red, green, blue]: [fearless_simd::f32x8; 3], + output: &mut [u8], +) { + use fearless_simd::{u8x16, Bytes, SimdBase}; + + let packed = + whole_u8_x8(simd, red) | (whole_u8_x8(simd, green) << 8) | (whole_u8_x8(simd, blue) << 16); + let indices = u8x16::from_slice(simd, &[0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0, 0, 0, 0]); + let (first, second) = simd.split_u32x8(packed); + let first = first.bitcast::>().swizzle_dyn(indices); + let second = second.bitcast::>().swizzle_dyn(indices); + if output.len() >= 28 { + first.store_slice(&mut output[..16]); + second.store_slice(&mut output[12..28]); + } else { + output[..12].copy_from_slice(&first.to_array()[..12]); + output[12..24].copy_from_slice(&second.to_array()[..12]); + } +} + +/// One byte value per lane of eight whole-number samples: the saturating +/// `as u8` cast (NaN to zero) through the native signed conversion. +#[cfg(all(feature = "simd", target_arch = "x86_64"))] +#[inline(always)] +fn whole_u8_x8(simd: S, value: fearless_simd::f32x8) -> fearless_simd::u32x8 { + use fearless_simd::{f32x8, Bytes, Select, SimdBase}; + + let zero = f32x8::splat(simd, 0.0); + let clamped = value + .simd_ge(zero) + .select(value, zero) + .min(f32x8::splat(simd, 255.0)); + simd.cvt_i32_f32x8(clamped).bitcast() +} + /// The CPU inverse transform, then (for centered 9/7 output) the rounding /// and level shift, chunk by chunk so each chunk makes one trip to memory. /// Chunks are multiples of the SIMD width, so every sample sees the same @@ -902,6 +951,14 @@ mod tests { let mut packed = vec![0xa5_u8; 3 * len]; deferred.pack_ict_rgb8_x86([&luma[..len], &cb[..len], &cr[..len]], &mut packed); assert_eq!(packed, expected[..3 * len], "{rounding:?} {len}"); + // Output room past the planes' pixels stays untouched. + let mut packed = vec![0xa5_u8; 3 * len + 5]; + deferred.pack_ict_rgb8_x86([&luma[..len], &cb[..len], &cr[..len]], &mut packed); + assert_eq!(packed[..3 * len], expected[..3 * len], "{rounding:?} {len}"); + assert!( + packed[3 * len..].iter().all(|&byte| byte == 0xa5), + "{rounding:?} {len}" + ); } } } From 0ad869a50c861bc7f2ac5fcbbb53ec658b19d5ed Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:34:31 -0400 Subject: [PATCH 42/53] perf(j2k-native): dequantize classic rows inside the x86-64 SIMD dispatch Baseline x86-64 has no per-lane variable shift and no unsigned int-to-float conversion, so the classic midpoint dequantization loop stayed scalar on x86-64 while ARM64 vectorizes it. The row now runs inside the runtime dispatch on x86-64, where AVX2 vectorizes it. ARM64 is unchanged. Output is byte-identical. Measured on x86-64 (Zen 3, AVX2), median of per-round ratios over 5 interleaved rounds against the build without this commit: classic 2 bpp 4096x2048 full decode -4.4% at 1 thread, -4.3% at 4 and -7.2% at 16; classic 1024x1024 regions -1.5% to -1.8% at 1 thread. --- .../src/j2c/bitplane/reconstruction.rs | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/crates/j2k-native/src/j2c/bitplane/reconstruction.rs b/crates/j2k-native/src/j2c/bitplane/reconstruction.rs index 0cb0f0258..fd2898e1d 100644 --- a/crates/j2k-native/src/j2c/bitplane/reconstruction.rs +++ b/crates/j2k-native/src/j2c/bitplane/reconstruction.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::state::Coefficient; +#[cfg(target_arch = "x86_64")] +use crate::math::{dispatch, Level}; use j2k_codec_math::classic::irreversible_midpoint_bit; /// Reconstruct an irreversible coefficient at the centre of its final @@ -89,17 +91,35 @@ impl MidpointReconstructor { /// midpoints fit in `u32`, and the per-coefficient branches become selects /// that vectorize. Halving is exact, so `fixed * (step / 2)` rounds like /// `(fixed / 2) * step`, and negation is a sign-bit flip. - #[expect( - clippy::cast_possible_truncation, - clippy::cast_precision_loss, - reason = "magnitudes are below 2^31 on the u32 path, which enters the f32 domain" - )] + /// + /// On x86-64 the row runs inside the runtime SIMD dispatch: baseline + /// x86-64 has no per-lane variable shift and no unsigned conversion, so + /// the loop would stay scalar there; with AVX2 it vectorizes as it does + /// on ARM64. pub(crate) fn dequantize_row( self, coefficients: &[Coefficient], output: &mut [f32], step: f32, ) { + #[cfg(target_arch = "x86_64")] + dispatch!(Level::new(), simd => { + let _ = simd; + self.dequantize_row_inlined(coefficients, output, step); + }); + #[cfg(not(target_arch = "x86_64"))] + self.dequantize_row_inlined(coefficients, output, step); + } + + #[expect( + clippy::cast_possible_truncation, + clippy::cast_precision_loss, + clippy::inline_always, + reason = "magnitudes are below 2^31 on the u32 path, which enters the f32 domain; \ + inlined into the x86-64 dispatch target" + )] + #[inline(always)] + fn dequantize_row_inlined(self, coefficients: &[Coefficient], output: &mut [f32], step: f32) { let Some((lowest, final_sigprop)) = self .fast .filter(|_| self.roi_shift == 0 && self.decoded_bitplanes <= 31) From 925719ce81ebb4ea971aad8b7c029757431dac9e Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 04:34:31 -0400 Subject: [PATCH 43/53] perf(j2k-native): run the per-coefficient classic Tier-1 passes inside the x86-64 SIMD dispatch Code blocks coded with the bypass, restart, causal or termination styles take the per-coefficient passes, which still compiled for baseline x86-64 (BSR-based `leading_zeros` on every MQ decision). They now run inside the runtime dispatch on x86-64 like the packed passes. ARM64 is unchanged. Output is byte-identical. Measured on x86-64 (Zen 3, AVX2) on a classic 2 bpp 4096x2048 RGB image encoded with Cmodes=BYPASS, median of per-round ratios over 5 interleaved rounds against the build without this commit: -7.3% at 1 thread, -6.0% at 4 and -3.5% at 16. Default code blocks (packed passes) are unchanged. --- .../j2k-native/src/j2c/bitplane/arithmetic.rs | 24 +++++++++++++++++++ .../j2k-native/src/j2c/bitplane/schedule.rs | 18 ++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/crates/j2k-native/src/j2c/bitplane/arithmetic.rs b/crates/j2k-native/src/j2c/bitplane/arithmetic.rs index 11600f7f4..696e3f4b2 100644 --- a/crates/j2k-native/src/j2c/bitplane/arithmetic.rs +++ b/crates/j2k-native/src/j2c/bitplane/arithmetic.rs @@ -32,6 +32,14 @@ fn decode_sign_bit_arithmetic( ctx.set_sign_index(idx, u8::from(sign_bit != 0)); } +#[cfg_attr( + target_arch = "x86_64", + expect( + clippy::inline_always, + reason = "inlines into the x86-64 Tier-1 dispatch in schedule.rs" + ), + inline(always) +)] pub(super) fn cleanup_pass_arithmetic( ctx: &mut BitPlaneDecodeContext, shared_decoder: &mut ArithmeticDecoder<'_>, @@ -102,6 +110,14 @@ pub(super) fn cleanup_pass_arithmetic( *shared_decoder = local_decoder; } +#[cfg_attr( + target_arch = "x86_64", + expect( + clippy::inline_always, + reason = "inlines into the x86-64 Tier-1 dispatch in schedule.rs" + ), + inline(always) +)] pub(super) fn significance_propagation_pass_arithmetic( ctx: &mut BitPlaneDecodeContext, shared_decoder: &mut ArithmeticDecoder<'_>, @@ -151,6 +167,14 @@ pub(super) fn significance_propagation_pass_arithmetic( *shared_decoder = local_decoder; } +#[cfg_attr( + target_arch = "x86_64", + expect( + clippy::inline_always, + reason = "inlines into the x86-64 Tier-1 dispatch in schedule.rs" + ), + inline(always) +)] pub(super) fn magnitude_refinement_pass_arithmetic( ctx: &mut BitPlaneDecodeContext, shared_decoder: &mut ArithmeticDecoder<'_>, diff --git a/crates/j2k-native/src/j2c/bitplane/schedule.rs b/crates/j2k-native/src/j2c/bitplane/schedule.rs index 726edf8d9..934e10961 100644 --- a/crates/j2k-native/src/j2c/bitplane/schedule.rs +++ b/crates/j2k-native/src/j2c/bitplane/schedule.rs @@ -161,6 +161,24 @@ fn decode_inner_with_observer( Some(()) } +/// The per-coefficient passes (bypass, restart, causal and termination +/// code-block styles) run inside the x86-64 dispatch for the same reason as +/// [`handle_packed_arithmetic_coding_passes`]. +#[cfg(target_arch = "x86_64")] +pub(super) fn handle_arithmetic_coding_passes( + start: u8, + end: u8, + ctx: &mut BitPlaneDecodeContext, + decoder: &mut ArithmeticDecoder<'_>, + observer: &mut impl J2kDecodeObserver, +) -> Option<()> { + dispatch!(Level::new(), simd => { + let _ = simd; + run_arithmetic_coding_passes::(start, end, ctx, decoder, observer) + }) +} + +#[cfg(not(target_arch = "x86_64"))] pub(super) fn handle_arithmetic_coding_passes( start: u8, end: u8, From 7dcd18101d3db5d446226c2fc9504a6bfb754ffe Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 06:10:32 -0400 Subject: [PATCH 44/53] perf(j2k-native): keep the MQ pass spills off page boundaries on ARM64 On AArch64 the classic Tier-1 pass loops in run_arithmetic_coding_passes keep more values live than there are registers and reload spilled pairs with `ldp`: 16-byte loads at 8-byte-aligned stack offsets. The significance pass reloads one such pair (the zero-coding table and the context array pointers) once per symbol. Wherever the stack depth put one of these pairs across a 16 KiB page boundary, every reload was a page-split load and the passes ran 44-45% slower. That happened at three 16-byte positions per 16 KiB of stack depth: on the caller's thread (one-thread decodes) the caller's stack and environment chose them, on pool threads the build did. A 4 KiB-aligned local now makes the function realign its stack pointer to 4 KiB, so where the spills sit within a page no longer depends on the caller; here they are single loads and stores just below the anchor, and none can straddle a page. The frame grows by up to 12 KiB of stack per call, so builds without `std` keep the old frame. x86-64 is unchanged: its spills are single aligned registers, and the same stack-offset sweep finds no slow offset there. Output is byte-identical. Measured on an 18-core ARM64 machine at one thread, sweeping the caller's stack offset in 16-byte steps: the parent decoded the classic 2 bpp 4096x2048 mosaic in 246.6 ms at its worst offset against a 171.5 ms median (+44%), and a 512x512 classic tile +40% at its worst offset of a full 16 KiB period; with this change the medians are unchanged (171.5 and 4.57 ms) and no offset repeats as slow. Interleaved A/B over 7 rounds against two builds of the parent at 1, 6, 18 and default threads: classic mosaics -0.3% to +1.8% (A/A -1.6% to +1.2%), the 5661x8287 9/7 workload's full-ROI decode -1.2% to +1.8% (A/A -1.2% to +1.8%), heap unchanged, max RSS +0.3 MB at 18 threads (worker stack pages); no classic row of the decode benchmark set slower after 7- and 15-round reruns. --- .../j2k-native/src/j2c/bitplane/schedule.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/j2k-native/src/j2c/bitplane/schedule.rs b/crates/j2k-native/src/j2c/bitplane/schedule.rs index 934e10961..c9799b8ff 100644 --- a/crates/j2k-native/src/j2c/bitplane/schedule.rs +++ b/crates/j2k-native/src/j2c/bitplane/schedule.rs @@ -219,6 +219,12 @@ pub(super) fn handle_packed_arithmetic_coding_passes( run_arithmetic_coding_passes::(start, end, ctx, decoder, observer) } +/// A local that makes its function realign the stack pointer to 4 KiB; see +/// [`run_arithmetic_coding_passes`]. +#[cfg(all(target_arch = "aarch64", feature = "std"))] +#[repr(align(4096))] +struct PageAlignedFrame(u8); + /// Arithmetic passes over the packed column words (`PACKED_COLUMNS`) or the /// per-coefficient arrays; decisions and contexts are identical. #[cfg_attr( @@ -236,6 +242,21 @@ fn run_arithmetic_coding_passes( decoder: &mut ArithmeticDecoder<'_>, observer: &mut impl J2kDecodeObserver, ) -> Option<()> { + // On AArch64 the pass loops reload spilled registers in pairs (`ldp`, 16 + // bytes, 8-byte aligned), some once per symbol. Wherever the stack put + // such a pair across a page boundary (16 KiB pages on Apple silicon), + // every reload was a page-split load and the passes ran ~45% slower, at + // stack depths set by the caller or, on pool threads, by the build. A + // 4 KiB-aligned local realigns this frame to 4 KiB, which fixes where + // the spills sit within a page instead of leaving it to the caller's + // stack depth; they sit below the anchor, inside one 4 KiB block. The + // frame grows by up to 12 KiB of stack, so bare-metal builds keep the + // old frame. x86-64 spills single aligned registers, which cannot split. + #[cfg(all(target_arch = "aarch64", feature = "std"))] + let frame_anchor = PageAlignedFrame(0); + #[cfg(all(target_arch = "aarch64", feature = "std"))] + core::hint::black_box(&frame_anchor.0); + for coding_pass in start..end { let current_bitplane = coding_pass.div_ceil(3); ctx.current_bit_position = ctx.bitplanes - 1 - current_bitplane; From f36be74948539269a5efb98efe407bf0c0fcca09 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 31 Oct 2026 06:10:32 -0400 Subject: [PATCH 45/53] docs(changelog): note the x86-64 SIMD dispatch coverage --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4be7e8a71..7d93da067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,19 @@ and stale roadmap entries have been removed from the public documentation set. term the charge grows to match; elsewhere it is unchanged and the IDWT holds less within it (a 1024 x 768 gray image on 4 threads: 3.9 MB in place of 4.5 MB, against a 4.1 MB charge). Output is unchanged. +- x86-64 RGB8 decodes of 9/7 images round and pack the inverse ICT output with + the runtime SIMD level (AVX2 where present) instead of a scalar library + call per sample: the centered rounding of the inverse MCT runs eight + samples at a time, and the last-level RGB8 pack fuses the ICT, rounding + and interleave as ARM64 already does. Output is unchanged. +- x86-64 decodes keep more of their hot loops at the runtime SIMD level + instead of baseline x86-64 code: the banded 9/7 IDWT's block load no + longer falls back to out-of-line SIMD kernels, 8-bit RGB region rows + use the SIMD RGB8 pack, non-MCT 9/7 rounding and the HT 9/7 magnitude + conversion use native instructions, the fused RGB8 pack skips a second + rounding, and classic Tier-1 (MQ decoding) and dequantization run inside + the dispatch so they use LZCNT, BMI2 and AVX2 where present. ARM64 code + is unchanged and output is unchanged. - Parallel CPU region decodes on the row-streamed IDWT decode Tier-1 band by band on the worker pool while the IDWT reads the bands, instead of decoding every code-block of a component first. Each sub-band holds three bands at From edf819ce8eb777ea7a26965065fb2ac59f7711c3 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 21:07:51 -0400 Subject: [PATCH 46/53] fix(j2k): report decode resource limits as their own error class ImageTooLarge and AllocationTooLarge reached callers as the generic J2kError::NativeDecode, so a decode that only exceeded the working-memory cap looked like any other backend failure. j2k_native::DecodeError::classify now reports DecodeErrorClass::ResourceLimit { requested, cap } for them, and the facade maps that class to the new J2kError::ResourceLimit { requested, cap, source } on every decode route: J2kDecoder, the tile batch functions and the owned batch API, as it already maps truncated input and unsupported features. requested and cap are known for AllocationTooLarge and None for ImageTooLarge. Host allocator failures keep their classification. The source chain keeps the native error. The native decode error conversion is now #[cold] and out of line. Inlined into its 53 call sites, the larger match made HTJ2K lossless 4096x2048 decodes at 18 threads 1.0-1.4% slower in interleaved A/B runs (code layout; a variant that kept J2kError at 96 bytes was 2.7% slower). Out of line it measures 1.002 against base (A/A 1.000), and 0.995 / 0.997 on the HTJ2K 2bpp and classic lossless rows. Public additions (both enums are #[non_exhaustive]): - j2k_native::DecodeErrorClass::ResourceLimit { requested, cap } - j2k::J2kError::ResourceLimit { requested, cap, source } J2kError grows from 96 to 112 bytes. Decoded bytes are unchanged. --- crates/j2k-native/src/error.rs | 64 ++++++++++ crates/j2k/src/error.rs | 85 ++++++++++++-- crates/j2k/src/parse/boxes.rs | 10 +- crates/j2k/src/parse/boxes/tests.rs | 11 ++ crates/j2k/tests/resource_limits.rs | 150 ++++++++++++++++++++++++ xtask/api/stable-api-1.0.public-api.txt | 7 ++ 6 files changed, 314 insertions(+), 13 deletions(-) create mode 100644 crates/j2k/tests/resource_limits.rs diff --git a/crates/j2k-native/src/error.rs b/crates/j2k-native/src/error.rs index d01d5656f..adf74d0b5 100644 --- a/crates/j2k-native/src/error.rs +++ b/crates/j2k-native/src/error.rs @@ -64,6 +64,17 @@ pub enum DecodeErrorClass { /// Stable user-facing feature label. what: &'static str, }, + /// The decode would exceed a resource limit: its live working memory + /// would pass the decode cap, or the image is too large to size. + /// + /// The input may be valid; a smaller request (region, reduced + /// resolution) or a larger limit can succeed where this one failed. + ResourceLimit { + /// Requested live bytes at the rejected boundary, when known. + requested: Option, + /// The cap the request exceeded, when known. + cap: Option, + }, /// All other native decoder failures. Backend, } @@ -156,6 +167,14 @@ impl DecodeError { offset: 0, segment: "JPEG 2000 entropy data", }, + Self::AllocationTooLarge { requested, cap, .. } => DecodeErrorClass::ResourceLimit { + requested: Some(requested), + cap: Some(cap), + }, + Self::Validation(ValidationError::ImageTooLarge) => DecodeErrorClass::ResourceLimit { + requested: None, + cap: None, + }, _ => DecodeErrorClass::Backend, } } @@ -713,6 +732,51 @@ mod classification_tests { } } + #[test] + fn resource_limits_classify_apart_from_invalid_input_and_allocator_failures() { + let cases = [ + ( + DecodeError::AllocationTooLarge { + what: "native decoder context retained components", + requested: 513, + cap: 512, + }, + DecodeErrorClass::ResourceLimit { + requested: Some(513), + cap: Some(512), + }, + ), + ( + DecodeError::Validation(ValidationError::ImageTooLarge), + DecodeErrorClass::ResourceLimit { + requested: None, + cap: None, + }, + ), + // The allocator failing a cap-valid request is not a limit the + // caller can plan around, and malformed geometry is invalid input. + ( + DecodeError::HostAllocationFailed { + what: "decode coefficients", + bytes: 511, + }, + DecodeErrorClass::Backend, + ), + ( + DecodeError::Validation(ValidationError::TooManyTiles), + DecodeErrorClass::Backend, + ), + ( + DecodeError::Validation(ValidationError::InvalidDimensions), + DecodeErrorClass::Backend, + ), + ]; + + for (error, expected) in cases { + assert_eq!(error.classify(), expected, "{error}"); + } + } + #[test] fn direct_plan_classification_and_display_share_the_same_label() { let reason = DirectPlanUnsupportedReason::ColorThreeComponentRgbCodestream; diff --git a/crates/j2k/src/error.rs b/crates/j2k/src/error.rs index bf12605f2..6b97e6157 100644 --- a/crates/j2k/src/error.rs +++ b/crates/j2k/src/error.rs @@ -125,6 +125,22 @@ pub enum J2kError { source: NativeBackendError, }, + /// Native decode would exceed a resource limit: its live working memory + /// would pass the decode cap, or the image is too large to size. + /// + /// Unlike invalid or unsupported input, the same codestream may decode + /// through a smaller request, such as a region or a reduced resolution. + #[error("decode resource limit exceeded: {source}")] + ResourceLimit { + /// Requested live bytes at the rejected boundary, when known. + requested: Option, + /// The cap the request exceeded, when known. + cap: Option, + /// Structured native decoder failure. + #[source] + source: NativeBackendError, + }, + /// Native decoder failure while validating generated encoder output. /// /// The structured source is retained for diagnostics, but this phase is @@ -296,10 +312,14 @@ impl J2kError { /// /// This hidden SPI exists for sibling accelerator adapters. Applications /// should receive [`J2kError`] through the public codec APIs instead. + #[cold] + #[inline(never)] pub(crate) fn from_native_decode_error(error: NativeDecodeError) -> Self { Self::from_native_decode_error_with_context(error, "native JPEG 2000 backend failed") } + #[cold] + #[inline(never)] pub(crate) fn from_native_decode_error_with_context( error: NativeDecodeError, context: &'static str, @@ -312,6 +332,11 @@ impl J2kError { Self::Input(InputError::TruncatedAt { offset, segment }) } NativeDecodeErrorClass::Unsupported { what } => Self::Unsupported(Unsupported { what }), + NativeDecodeErrorClass::ResourceLimit { requested, cap } => Self::ResourceLimit { + requested, + cap, + source: NativeBackendError::decode(error), + }, _ => Self::NativeDecode { context, source: NativeBackendError::decode(error), @@ -518,17 +543,10 @@ mod tests { #[test] fn native_decode_resource_errors_preserve_context_and_source() { - let sources = [ - NativeDecodeError::AllocationTooLarge { - what: "validation fixture", - requested: 9, - cap: 8, - }, - NativeDecodeError::HostAllocationFailed { - what: "validation fixture", - bytes: 7, - }, - ]; + let sources = [NativeDecodeError::HostAllocationFailed { + what: "validation fixture", + bytes: 7, + }]; for source in sources { let error = J2kError::from_native_decode_error_with_context( @@ -547,6 +565,51 @@ mod tests { } } + #[test] + fn native_resource_limits_are_their_own_class_with_known_bytes() { + let cases = [ + ( + NativeDecodeError::AllocationTooLarge { + what: "cap fixture", + requested: 9, + cap: 8, + }, + Some(9), + Some(8), + ), + ( + NativeDecodeError::Validation(j2k_native::ValidationError::ImageTooLarge), + None, + None, + ), + ]; + + for (source, expected_requested, expected_cap) in cases { + let error = + J2kError::from_native_decode_error_with_context(source, "decode fixture failed"); + let J2kError::ResourceLimit { + requested, + cap, + source: stored, + } = &error + else { + panic!("expected a resource limit, got {error:?}"); + }; + assert_eq!((*requested, *cap), (expected_requested, expected_cap)); + assert_eq!(stored, &NativeBackendError::decode(source)); + assert_eq!( + error.to_string(), + format!("decode resource limit exceeded: {source}") + ); + assert!(!error.is_truncated()); + assert!(!error.is_unsupported()); + assert!(!error.is_not_implemented()); + assert!(!error.is_buffer_error()); + assert!(error.is_heap_free_batch_decode_error()); + assert_native_source_chain(&error, &source); + } + } + #[test] fn generated_output_validation_errors_do_not_masquerade_as_caller_failures() { for source in [ diff --git a/crates/j2k/src/parse/boxes.rs b/crates/j2k/src/parse/boxes.rs index 44fea29de..696c85709 100644 --- a/crates/j2k/src/parse/boxes.rs +++ b/crates/j2k/src/parse/boxes.rs @@ -485,8 +485,14 @@ fn map_native_jp2_error(error: NativeDecodeError) -> J2kError { offset: 0, what: "unsupported JP2/JPH box metadata", }, - source @ (NativeDecodeError::AllocationTooLarge { .. } - | NativeDecodeError::HostAllocationFailed { .. }) => J2kError::NativeDecode { + // A cap failure classifies as a resource limit. + source @ NativeDecodeError::AllocationTooLarge { .. } => { + J2kError::from_native_decode_error_with_context( + source, + "JP2/JPH metadata inspection failed", + ) + } + source @ NativeDecodeError::HostAllocationFailed { .. } => J2kError::NativeDecode { context: "JP2/JPH metadata inspection failed", source: crate::NativeBackendError::decode(source), }, diff --git a/crates/j2k/src/parse/boxes/tests.rs b/crates/j2k/src/parse/boxes/tests.rs index 0a6b72752..d25676827 100644 --- a/crates/j2k/src/parse/boxes/tests.rs +++ b/crates/j2k/src/parse/boxes/tests.rs @@ -264,6 +264,17 @@ fn native_container_errors_keep_specific_facade_categories() { requested: 9, cap: 8 }), + J2kError::ResourceLimit { + requested: Some(9), + cap: Some(8), + .. + } + )); + assert!(matches!( + map_native_jp2_error(NativeDecodeError::HostAllocationFailed { + what: "metadata", + bytes: 9, + }), J2kError::NativeDecode { .. } )); } diff --git a/crates/j2k/tests/resource_limits.rs b/crates/j2k/tests/resource_limits.rs new file mode 100644 index 000000000..3b82a1ebd --- /dev/null +++ b/crates/j2k/tests/resource_limits.rs @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Decodes that exceed a resource limit fail with `J2kError::ResourceLimit` +//! on every route, apart from invalid input and unsupported features. + +use std::sync::Arc; + +use j2k::{ + decode_tiles_region_into, prepare_batch, BatchDecodeOptions, BatchItemError, CodecError, + CpuBatchDecoder, DecodeRequest, EncodedImage, J2kDecoder, J2kError, PixelFormat, Rect, + TileBatchError, TileBatchOptions, TileRegionDecodeJob, +}; +use j2k_native::{encode, EncodeOptions}; + +/// A small gray codestream whose SIZ claims one `dimension`-square tile. +/// Headers parse; a decode is rejected by its resource limits before it +/// allocates for, or reads the packets of, the claimed tile. +fn large_siz_codestream(dimension: u32) -> Vec { + let pixels = (0..=u8::MAX).collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let mut codestream = encode(&pixels, 16, 16, 1, 8, false, &options).expect("encode"); + let siz = codestream + .windows(2) + .position(|marker| marker == [0xff, 0x51]) + .expect("SIZ marker"); + for field in [6, 10, 22, 26] { + codestream[siz + field..siz + field + 4].copy_from_slice(&dimension.to_be_bytes()); + } + codestream +} + +/// The resource limit `error` reports, after checking it is no other class. +fn resource_limit(error: &J2kError) -> (Option, Option) { + let J2kError::ResourceLimit { requested, cap, .. } = error else { + panic!("expected a resource limit, got {error:?}"); + }; + assert!(!error.is_truncated()); + assert!(!error.is_unsupported()); + assert!(!error.is_not_implemented()); + assert!(!error.is_buffer_error()); + if let (Some(requested), Some(cap)) = (requested, cap) { + assert!(requested > cap, "requested {requested} within cap {cap}"); + } + (*requested, *cap) +} + +const TINY_ROI: Rect = Rect { + x: 0, + y: 0, + w: 4, + h: 4, +}; + +#[test] +fn decoder_component_planes_over_the_cap_report_requested_and_cap_bytes() { + let codestream = large_siz_codestream(65_537); + let mut decoder = J2kDecoder::new(&codestream).expect("large SIZ parses"); + + let error = decoder + .decode_native_components() + .expect_err("65537x65537 planes exceed the decode cap"); + + let (requested, cap) = resource_limit(&error); + assert!(requested.is_some() && cap.is_some(), "{error:?}"); +} + +#[test] +fn decode_into_over_the_cap_reports_requested_and_cap_bytes() { + // 11600x11600 gray needs 538 MB of f32 planes, over the 512 MiB decode + // cap. The decode is rejected before it writes the caller's buffer. + let dimension = 11_600_u32; + let codestream = large_siz_codestream(dimension); + let mut decoder = J2kDecoder::new(&codestream).expect("large SIZ parses"); + let stride = dimension as usize; + let mut out = vec![0_u8; stride * stride]; + + let error = decoder + .decode_into(&mut out, stride, PixelFormat::Gray8) + .expect_err("the planes exceed the decode cap"); + + let (requested, cap) = resource_limit(&error); + assert!(requested.is_some() && cap.is_some(), "{error:?}"); +} + +#[test] +fn decoder_region_over_the_metadata_limit_is_a_resource_limit() { + let codestream = large_siz_codestream(1 << 24); + let mut decoder = J2kDecoder::new(&codestream).expect("large SIZ parses"); + let mut pool = j2k::J2kScratchPool::new(); + let stride = TINY_ROI.w as usize; + let mut out = vec![0_u8; stride * TINY_ROI.h as usize]; + + let error = decoder + .decode_region_into(&mut pool, &mut out, stride, PixelFormat::Gray8, TINY_ROI) + .expect_err("a 2^24-square tile's code-block metadata exceeds the cap"); + + resource_limit(&error); +} + +#[test] +fn tile_batch_errors_keep_the_resource_limit_with_the_tile_index() { + let codestream = large_siz_codestream(1 << 24); + let stride = TINY_ROI.w as usize; + let mut out = vec![0_u8; stride * TINY_ROI.h as usize]; + let mut jobs = [TileRegionDecodeJob { + input: &codestream, + out: &mut out, + stride, + roi: TINY_ROI, + }]; + + let error = + decode_tiles_region_into(&mut jobs, PixelFormat::Gray8, TileBatchOptions::new(None)) + .expect_err("the tile exceeds the decode cap"); + + let TileBatchError::Tile(tile) = &error else { + panic!("expected a tile failure, got {error:?}"); + }; + assert_eq!(tile.index, 0); + resource_limit(&tile.source); +} + +#[test] +fn owned_batch_item_errors_keep_the_resource_limit() { + let codestream = Arc::<[u8]>::from(large_siz_codestream(1 << 24)); + let input = EncodedImage::new(codestream, DecodeRequest::Region { roi: TINY_ROI }); + let options = BatchDecodeOptions::default(); + let prepared = prepare_batch(vec![input], options).expect("prepare the batch"); + let errors = if prepared.errors().is_empty() { + let mut decoder = CpuBatchDecoder::new(options); + decoder + .decode_prepared(&prepared) + .expect("item failures stay per item") + .errors() + .to_vec() + } else { + prepared.errors().to_vec() + }; + + assert_eq!(errors.len(), 1, "{errors:?}"); + assert_eq!(errors[0].index, 0); + let BatchItemError::Codec { source, .. } = &errors[0].source else { + panic!("expected a codec failure, got {:?}", errors[0].source); + }; + resource_limit(source); +} diff --git a/xtask/api/stable-api-1.0.public-api.txt b/xtask/api/stable-api-1.0.public-api.txt index b84af23e7..49def898c 100644 --- a/xtask/api/stable-api-1.0.public-api.txt +++ b/xtask/api/stable-api-1.0.public-api.txt @@ -562,6 +562,10 @@ pub j2k::J2kError::NotImplemented(j2k_core::error::NotImplemented) pub j2k::J2kError::RateTargetUnreachable pub j2k::J2kError::RateTargetUnreachable::best: alloc::string::String pub j2k::J2kError::RateTargetUnreachable::target: alloc::string::String +pub j2k::J2kError::ResourceLimit +pub j2k::J2kError::ResourceLimit::cap: core::option::Option +pub j2k::J2kError::ResourceLimit::requested: core::option::Option +pub j2k::J2kError::ResourceLimit::source: j2k::NativeBackendError pub j2k::J2kError::Unsupported(j2k_core::error::Unsupported) pub j2k::J2kFileBoxMetadata::channel_definitions: &'a [j2k::J2kChannelDefinition] pub j2k::J2kFileBoxMetadata::component_mappings: &'a [j2k::J2kComponentMapping] @@ -4023,6 +4027,9 @@ pub j2k_native::DecodeErrorClass::InputTooShort::need: usize pub j2k_native::DecodeErrorClass::InputTruncatedAt pub j2k_native::DecodeErrorClass::InputTruncatedAt::offset: usize pub j2k_native::DecodeErrorClass::InputTruncatedAt::segment: &'static str +pub j2k_native::DecodeErrorClass::ResourceLimit +pub j2k_native::DecodeErrorClass::ResourceLimit::cap: core::option::Option +pub j2k_native::DecodeErrorClass::ResourceLimit::requested: core::option::Option pub j2k_native::DecodeErrorClass::Unsupported pub j2k_native::DecodeErrorClass::Unsupported::what: &'static str pub j2k_native::DecodeSettings::resolve_palette_indices: bool From c21fbe0217af47c030f059e1f91b9b772b1dc7f1 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 21:07:51 -0400 Subject: [PATCH 47/53] docs(changelog): note the resource-limit error class --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d93da067..647c51c75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,18 @@ and stale roadmap entries have been removed from the public documentation set. term the charge grows to match; elsewhere it is unchanged and the IDWT holds less within it (a 1024 x 768 gray image on 4 threads: 3.9 MB in place of 4.5 MB, against a 4.1 MB charge). Output is unchanged. +- Decodes that would exceed a resource limit now fail with + `J2kError::ResourceLimit { requested, cap, source }` instead of the generic + `J2kError::NativeDecode`, on every decode route: `J2kDecoder`, the tile + batch functions (inside `TileBatchError::Tile`) and the owned batch API + (inside `BatchItemError::Codec`). `requested` and `cap` are the live bytes + and the cap when the native decoder knows them (`AllocationTooLarge`), and + `None` for `ImageTooLarge`. Callers can tell these apart from invalid input + and unsupported features, and retry with a smaller request such as a region + or a reduced resolution. The display text and the source chain keep the + native error. `j2k_native::DecodeError::classify` reports the same class as + `DecodeErrorClass::ResourceLimit`. Both enums are `#[non_exhaustive]`; + host allocator failures keep their previous classification. - x86-64 RGB8 decodes of 9/7 images round and pack the inverse ICT output with the runtime SIMD level (AVX2 where present) instead of a scalar library call per sample: the centered rounding of the inverse MCT runs eight From e104e76aa0620e5a840b2b566ca33f4df9cae72c Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 20:37:38 -0400 Subject: [PATCH 48/53] fix(j2k): fail batch admission claims that cannot fit beside the shared plan BatchAllocationBudget::claim waited forever when a claim fit the execution cap but not the cap minus the budget's permanent baseline (the shared direct plan), because no claim can ever release those bytes. Such a claim now fails at once with AllocationTooLarge (cap = cap - baseline); claims that can fit still wait for other owners as before. --- crates/j2k/src/batch/admission.rs | 16 +++++++++++++ crates/j2k/src/batch/admission/tests.rs | 32 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/crates/j2k/src/batch/admission.rs b/crates/j2k/src/batch/admission.rs index 3f3df5da0..f875d296f 100644 --- a/crates/j2k/src/batch/admission.rs +++ b/crates/j2k/src/batch/admission.rs @@ -10,6 +10,10 @@ use super::allocation::batch_execution_cap_bytes; pub(super) struct BatchAllocationBudget { cap: usize, + /// Bytes charged for the budget's whole lifetime (a shared plan). No + /// claim ever releases them, so a claim above `cap - baseline` can never + /// be admitted. + baseline: usize, state: Mutex, changed: Condvar, #[cfg(test)] @@ -33,6 +37,7 @@ impl BatchAllocationBudget { } Ok(Arc::new(Self { cap, + baseline, state: Mutex::new(AdmissionState { live: baseline, waiters: 0, @@ -54,6 +59,17 @@ impl BatchAllocationBudget { cap: self.cap, }); } + // Waiting only helps while other claims hold the missing bytes; the + // permanent baseline never leaves, so such a claim fails now instead + // of waiting forever. + let admissible = self.cap - self.baseline; + if bytes > admissible { + return Err(BatchInfrastructureError::AllocationTooLarge { + what: "one J2K batch execution claim beside the shared plan", + requested: bytes, + cap: admissible, + }); + } let mut state = self .state .lock() diff --git a/crates/j2k/src/batch/admission/tests.rs b/crates/j2k/src/batch/admission/tests.rs index f4ee4065a..d12df4b6c 100644 --- a/crates/j2k/src/batch/admission/tests.rs +++ b/crates/j2k/src/batch/admission/tests.rs @@ -34,6 +34,38 @@ fn shared_baseline_is_subtracted_from_each_initial_worker_claim() { assert_eq!(budget.lock_state().live, shared); } +#[test] +fn claim_that_cannot_fit_beside_the_baseline_fails_without_waiting() { + let baseline = super::super::allocation::GENERIC_WORKER_CLAIM_BYTES / 8; + let budget = BatchAllocationBudget::with_baseline(baseline).expect("shared-plan budget"); + let admissible = batch_execution_cap_bytes() - baseline; + let claiming = Arc::clone(&budget); + let (sender, receiver) = std::sync::mpsc::channel(); + let claimant = std::thread::spawn(move || { + // Fits the cap, but never beside the permanent baseline. + let result = claiming.claim(admissible + 1).map(drop); + sender.send(result).expect("report the claim result"); + }); + let result = receiver + .recv_timeout(std::time::Duration::from_secs(30)) + .unwrap_or_else(|_| { + panic!( + "claim blocked; registered waiters: {}", + budget.lock_state().waiters + ) + }); + claimant.join().expect("claimant joined"); + assert!(matches!( + result, + Err(BatchInfrastructureError::AllocationTooLarge { requested, cap, .. }) + if requested == admissible + 1 && cap == admissible + )); + assert_eq!(budget.lock_state().waiters, 0, "the claim never waited"); + let largest = budget.claim(admissible).expect("largest admissible claim"); + drop(largest); + assert_eq!(budget.lock_state().live, baseline); +} + #[test] fn waiting_claim_resumes_after_an_owner_reconciles() { let budget = BatchAllocationBudget::with_baseline(0).expect("budget"); From f431a4b4e70ef2ee934d05a8050cb1f26a055db9 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 20:51:56 -0400 Subject: [PATCH 49/53] perf(j2k-native): decode prepared classic batch plans in parallel Batch workers enabled parallel referenced execution for HTJ2K plans only; classic plans never set the flag and the classic executor ignored it, so a prepared classic batch decoded every code block and wavelet level on one thread (4.7-8.3x slower than single-image decode). The HT row dispatch moves to a shared generic module (same types and sizes), and classic sub-bands now decode their rows of code blocks on the Rayon pool when every payload is one contiguous fragment, falling back to the serial path otherwise; IDWT steps follow the same flag. Extra classic workspaces are accounted and reserved like HT's, but bounded by the most code-block rows any sub-band holds, so small tiles keep few of them. On an 18-core ARM64 machine, decode only, byte-identical: batches of 2048 x 2048 RGB 191 -> 41 ms, 3584 x 3584 RGB 550 -> 104 ms, 4096 x 4096 gray 425 -> 51 ms, 4096 x 4096 RGB at half scale 191 -> 40 ms; 8 images in flight 1.7-1.9x faster; 64 small tiles unchanged. Peak heap +0.3% to +1.7%. --- crates/j2k-native/src/direct_cpu.rs | 13 +- .../j2k-native/src/direct_cpu/allocation.rs | 83 ++++++ .../src/direct_cpu/allocation/referenced.rs | 32 ++- .../allocation/referenced/budget.rs | 15 +- .../allocation/referenced/storage.rs | 10 +- .../direct_cpu/allocation/referenced/view.rs | 80 +++++- .../j2k-native/src/direct_cpu/referenced.rs | 3 + .../src/direct_cpu/referenced/component.rs | 197 +++---------- .../src/direct_cpu/referenced/rows.rs | 186 ++++++++++++ .../src/direct_cpu/referenced_classic.rs | 264 +++++++++++++++++- .../direct_cpu/referenced_classic/tests.rs | 84 ++++++ crates/j2k/src/batch/worker/owned.rs | 2 + 12 files changed, 794 insertions(+), 175 deletions(-) create mode 100644 crates/j2k-native/src/direct_cpu/referenced/rows.rs create mode 100644 crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs diff --git a/crates/j2k-native/src/direct_cpu.rs b/crates/j2k-native/src/direct_cpu.rs index 5883fe71a..18d287cb0 100644 --- a/crates/j2k-native/src/direct_cpu.rs +++ b/crates/j2k-native/src/direct_cpu.rs @@ -57,6 +57,9 @@ pub struct J2kDirectCpuScratch { /// Extra HT workspaces for parallel referenced execution, one per /// additional Rayon worker. ht_task_workspaces: Vec, + /// Extra classic workspaces for parallel referenced classic execution, + /// one per additional Rayon worker. + classic_task_workspaces: Vec, /// Whether referenced execution may spread work over the Rayon pool. parallel: bool, staged_state: Option, @@ -87,6 +90,7 @@ impl J2kDirectCpuScratch { classic_workspace: J2kCodeBlockDecodeWorkspace::empty(), ht_workspace: HtCodeBlockDecodeWorkspace::empty(), ht_task_workspaces: Vec::new(), + classic_task_workspaces: Vec::new(), parallel: false, staged_state: None, } @@ -126,8 +130,13 @@ impl J2kDirectCpuScratch { #[doc(hidden)] #[must_use] pub fn retained_classic_workspace_bytes(&self) -> usize { - self.classic_workspace - .allocated_bytes() + self.classic_task_workspaces + .iter() + .try_fold( + self.classic_workspace.allocated_bytes().ok(), + |bytes, workspace| Some(bytes?.checked_add(workspace.allocated_bytes().ok()?)), + ) + .flatten() .unwrap_or(usize::MAX) } diff --git a/crates/j2k-native/src/direct_cpu/allocation.rs b/crates/j2k-native/src/direct_cpu/allocation.rs index 60fcb8f64..141c71a19 100644 --- a/crates/j2k-native/src/direct_cpu/allocation.rs +++ b/crates/j2k-native/src/direct_cpu/allocation.rs @@ -186,6 +186,8 @@ fn normalize_retained_scratch( scratch.ht_workspace = crate::HtCodeBlockDecodeWorkspace::default(); scratch.ht_task_workspaces = Vec::new(); } + // Only referenced classic execution decodes classic blocks in parallel. + scratch.classic_task_workspaces = Vec::new(); Ok(()) } @@ -448,6 +450,87 @@ fn parallel_ht_task_workspace_count( 0 } +/// Largest code block and most code-block rows of any sub-band a parallel +/// referenced classic execution decodes. One dispatch decodes at most one +/// row per Rayon worker, so it never needs more workspaces than rows. +#[derive(Debug, Clone, Copy)] +struct ClassicTaskShape { + dimensions: (u32, u32), + #[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "rows bound parallel dispatch only") + )] + rows: usize, +} + +/// Extra classic workspaces beside the main one: one per additional Rayon +/// worker that a dispatch can occupy, or none when serial. +fn classic_task_workspace_count(parallel: bool, classic_tasks: Option) -> usize { + #[cfg(feature = "parallel")] + if let (true, Some(shape)) = (parallel, classic_tasks) { + return rayon::current_num_threads() + .min(shape.rows) + .saturating_sub(1); + } + let _ = (parallel, classic_tasks); + 0 +} + +/// Charge the extra classic workspaces of parallel referenced classic +/// execution. `classic_tasks` is `None` for every other route. +fn include_classic_task_workspaces( + budget: &mut DirectAllocationBudget, + classic_tasks: Option, + scratch: &J2kDirectCpuScratch, +) -> Result<()> { + let target_count = classic_task_workspace_count(scratch.parallel, classic_tasks); + budget.include_capacity::( + scratch.classic_task_workspaces.capacity().max(target_count), + )?; + let target_bytes = classic_tasks.map_or(Ok(0), |shape| { + bitplane::classic_decode_workspace_bytes(shape.dimensions.0, shape.dimensions.1) + })?; + for index in 0..scratch.classic_task_workspaces.len().max(target_count) { + let retained = scratch + .classic_task_workspaces + .get(index) + .map_or(Ok(0), crate::J2kCodeBlockDecodeWorkspace::allocated_bytes)?; + let target = if index < target_count { + target_bytes + } else { + 0 + }; + budget.include_bytes(retained.max(target))?; + } + Ok(()) +} + +/// Keep exactly the extra classic workspaces parallel referenced classic +/// execution uses, each prepared for the plan's largest code block. +fn reserve_classic_task_workspaces( + classic_tasks: Option, + scratch: &mut J2kDirectCpuScratch, +) -> Result<()> { + let count = classic_task_workspace_count(scratch.parallel, classic_tasks); + scratch.classic_task_workspaces.truncate(count); + try_reserve_decode_elements(&mut scratch.classic_task_workspaces, count)?; + while scratch.classic_task_workspaces.len() < count { + scratch + .classic_task_workspaces + .push(crate::J2kCodeBlockDecodeWorkspace::default()); + } + if let Some(ClassicTaskShape { + dimensions: (width, height), + .. + }) = classic_tasks + { + for workspace in &mut scratch.classic_task_workspaces { + workspace.prepare(width, height)?; + } + } + Ok(()) +} + fn reserve_scratch( components: &[J2kDirectGrayscalePlan], compressed_payload_bytes: usize, diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs index dfbb3142b..9743bbc2d 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs @@ -9,14 +9,14 @@ mod view; pub(in super::super) use view::{max_referenced_classic_dimensions, max_referenced_ht_dimensions}; use super::{ - DecodeError, DirectWorkspaceBudget, J2kDirectCpuScratch, J2kReferencedClassicPlan, - J2kReferencedHtj2kPlan, Result, ValidationError, + ClassicTaskShape, DecodeError, DirectWorkspaceBudget, J2kDirectCpuScratch, + J2kReferencedClassicPlan, J2kReferencedHtj2kPlan, Result, ValidationError, }; use budget::validate_referenced_aggregate_plan; use storage::{normalize_referenced_scratch, reserve_referenced_scratch}; use view::{ - max_referenced_classic_payload_bytes, max_referenced_dimensions, max_referenced_payload_bytes, - ReferencedPlanView, + max_referenced_classic_payload_bytes, max_referenced_classic_rows, max_referenced_dimensions, + max_referenced_payload_bytes, ReferencedPlanView, }; pub(in super::super) fn prepare_referenced_direct_scratch( @@ -32,6 +32,7 @@ pub(in super::super) fn prepare_referenced_direct_scratch( max_referenced_payload_bytes(plan)?, max_referenced_dimensions(view, false), max_referenced_dimensions(view, true), + None, scratch, ) } @@ -41,13 +42,20 @@ pub(in super::super) fn prepare_referenced_classic_scratch( scratch: &mut J2kDirectCpuScratch, ) -> Result { let view = ReferencedPlanView::Classic(plan); + let parallel = scratch.parallel; + let classic_dimensions = max_referenced_classic_dimensions(plan); + let classic_tasks = classic_dimensions.map(|dimensions| ClassicTaskShape { + dimensions, + rows: max_referenced_classic_rows(plan), + }); prepare_referenced_component_scratch( view, - false, + parallel, plan.retained_allocation_bytes()?, max_referenced_classic_payload_bytes(plan), - max_referenced_classic_dimensions(plan), + classic_dimensions, None, + classic_tasks, scratch, ) } @@ -63,6 +71,7 @@ pub(in super::super) fn prepare_referenced_htj2k_staged_scratch( 0, None, None, + None, scratch, ) } @@ -78,10 +87,15 @@ pub(in super::super) fn prepare_referenced_classic_staged_scratch( 0, None, None, + None, scratch, ) } +#[expect( + clippy::too_many_arguments, + reason = "normalization, accounting, and reservation share every retained owner shape explicitly" +)] fn prepare_referenced_component_scratch( plan: ReferencedPlanView<'_>, parallel: bool, @@ -89,6 +103,7 @@ fn prepare_referenced_component_scratch( compressed_payload_bytes: usize, retained_classic_workspace_dimensions: Option<(u32, u32)>, retained_ht_workspace_dimensions: Option<(u32, u32)>, + classic_tasks: Option, scratch: &mut J2kDirectCpuScratch, ) -> Result { normalize_referenced_scratch( @@ -96,6 +111,7 @@ fn prepare_referenced_component_scratch( compressed_payload_bytes, retained_classic_workspace_dimensions.is_some(), retained_ht_workspace_dimensions.is_some(), + classic_tasks.is_some(), scratch, )?; if let Err(error) = validate_referenced_aggregate_plan( @@ -105,6 +121,7 @@ fn prepare_referenced_component_scratch( compressed_payload_bytes, retained_classic_workspace_dimensions, retained_ht_workspace_dimensions, + classic_tasks, scratch, ) { if !matches!( @@ -121,6 +138,7 @@ fn prepare_referenced_component_scratch( compressed_payload_bytes, retained_classic_workspace_dimensions, retained_ht_workspace_dimensions, + classic_tasks, scratch, )?; } @@ -130,6 +148,7 @@ fn prepare_referenced_component_scratch( compressed_payload_bytes, retained_classic_workspace_dimensions, retained_ht_workspace_dimensions, + classic_tasks, scratch, ) { scratch.clear(); @@ -142,6 +161,7 @@ fn prepare_referenced_component_scratch( compressed_payload_bytes, retained_classic_workspace_dimensions, retained_ht_workspace_dimensions, + classic_tasks, scratch, ) { Ok(workspace_budget) => Ok(workspace_budget), diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced/budget.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced/budget.rs index 29f4badc0..7d909efe1 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced/budget.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced/budget.rs @@ -1,14 +1,19 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::super::{ - bitplane, ht_block_decode, DirectAllocationBudget, DirectComponentBandScratch, - DirectComponentPlane, DirectCpuBand, DirectWorkspaceBudget, J2kDirectCpuScratch, Result, + bitplane, ht_block_decode, ClassicTaskShape, DirectAllocationBudget, + DirectComponentBandScratch, DirectComponentPlane, DirectCpuBand, DirectWorkspaceBudget, + J2kDirectCpuScratch, Result, }; use super::view::{ referenced_band_target, referenced_component_band_count, referenced_component_plane_len, referenced_temporary_workspace_bytes, validate_referenced_shape, ReferencedPlanView, }; +#[expect( + clippy::too_many_arguments, + reason = "referenced accounting keeps every retained owner shape explicit, as reservation does" +)] pub(super) fn validate_referenced_aggregate_plan( plan: ReferencedPlanView<'_>, parallel: bool, @@ -16,6 +21,7 @@ pub(super) fn validate_referenced_aggregate_plan( compressed_payload_bytes: usize, retained_classic_workspace_dimensions: Option<(u32, u32)>, retained_ht_workspace_dimensions: Option<(u32, u32)>, + classic_tasks: Option, scratch: &J2kDirectCpuScratch, ) -> Result { validate_referenced_shape(plan)?; @@ -28,6 +34,7 @@ pub(super) fn validate_referenced_aggregate_plan( compressed_payload_bytes, retained_classic_workspace_dimensions, retained_ht_workspace_dimensions, + classic_tasks, scratch, )?; let base_bytes = budget.bytes; @@ -44,6 +51,7 @@ fn include_referenced_scratch_allocations( compressed_payload_bytes: usize, retained_classic_workspace_dimensions: Option<(u32, u32)>, retained_ht_workspace_dimensions: Option<(u32, u32)>, + classic_tasks: Option, scratch: &J2kDirectCpuScratch, ) -> Result<()> { let component_count = plan.component_count(); @@ -93,5 +101,6 @@ fn include_referenced_scratch_allocations( ht_block_decode::ht_decode_workspace_bytes(width, height) })?; budget.include_bytes(retained_ht_workspace_bytes.max(target_ht_workspace_bytes))?; - super::super::include_ht_task_workspaces(budget, retained_ht_workspace_dimensions, scratch) + super::super::include_ht_task_workspaces(budget, retained_ht_workspace_dimensions, scratch)?; + super::super::include_classic_task_workspaces(budget, classic_tasks, scratch) } diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced/storage.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced/storage.rs index 4078c02a6..4dbba696c 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced/storage.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced/storage.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::super::{ - fill_without_allocation, normalize_outer_owner, try_reserve_decode_elements, + fill_without_allocation, normalize_outer_owner, try_reserve_decode_elements, ClassicTaskShape, DirectComponentBandScratch, DirectComponentPlane, DirectCpuBand, J2kDirectCpuScratch, Result, Vec, }; @@ -15,6 +15,7 @@ pub(super) fn normalize_referenced_scratch( compressed_payload_bytes: usize, retain_classic_workspace: bool, retain_ht_workspace: bool, + retain_classic_task_workspaces: bool, scratch: &mut J2kDirectCpuScratch, ) -> Result<()> { validate_referenced_shape(plan)?; @@ -68,6 +69,9 @@ pub(super) fn normalize_referenced_scratch( scratch.ht_workspace = crate::HtCodeBlockDecodeWorkspace::default(); scratch.ht_task_workspaces = Vec::new(); } + if !retain_classic_task_workspaces { + scratch.classic_task_workspaces = Vec::new(); + } scratch.staged_state = None; Ok(()) } @@ -77,6 +81,7 @@ pub(super) fn reserve_referenced_scratch( compressed_payload_bytes: usize, retained_classic_workspace_dimensions: Option<(u32, u32)>, retained_ht_workspace_dimensions: Option<(u32, u32)>, + classic_tasks: Option, scratch: &mut J2kDirectCpuScratch, ) -> Result<()> { let component_count = plan.component_count(); @@ -117,5 +122,6 @@ pub(super) fn reserve_referenced_scratch( if let Some((width, height)) = retained_ht_workspace_dimensions { scratch.ht_workspace.prepare(width, height)?; } - super::super::reserve_ht_task_workspaces(retained_ht_workspace_dimensions, scratch) + super::super::reserve_ht_task_workspaces(retained_ht_workspace_dimensions, scratch)?; + super::super::reserve_classic_task_workspaces(classic_tasks, scratch) } diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs index 113467b95..7846d3989 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs @@ -205,7 +205,54 @@ pub(super) fn referenced_temporary_workspace_bytes( } } let ht_dispatch = referenced_parallel_ht_dispatch_bytes(plan, parallel)?; - Ok(classic.max(ht).max(idwt).max(ht_dispatch)) + let classic_dispatch = referenced_parallel_classic_dispatch_bytes(plan, parallel)?; + Ok(classic + .max(ht) + .max(idwt) + .max(ht_dispatch) + .max(classic_dispatch)) +} + +/// Row-dispatch bytes of parallel referenced classic execution. Classic +/// blocks inside HTJ2K plans always decode serially. +#[cfg(feature = "parallel")] +fn referenced_parallel_classic_dispatch_bytes( + plan: ReferencedPlanView<'_>, + parallel: bool, +) -> Result { + let worker_count = rayon::current_num_threads(); + if !parallel || worker_count <= 1 || !matches!(plan, ReferencedPlanView::Classic(_)) { + return Ok(0); + } + let mut maximum_jobs = 0usize; + for tile_index in 0..plan.tile_count() { + for component_index in 0..plan.component_count() { + for step in &plan.component_plan(tile_index, component_index)?.steps { + if let J2kDirectGrayscaleStep::ClassicSubBand(sub_band) = step { + maximum_jobs = maximum_jobs.max(sub_band.jobs.len()); + } + } + } + } + if maximum_jobs == 0 { + return Ok(0); + } + super::super::super::referenced::rows::row_dispatch_transient_bytes::< + crate::J2kOwnedCodeBlockBatchJob, + crate::J2kCodeBlockDecodeWorkspace, + >(maximum_jobs, maximum_jobs, worker_count) +} + +#[cfg(not(feature = "parallel"))] +#[expect( + clippy::unnecessary_wraps, + reason = "the no-parallel stub preserves the fallible parallel planning signature" +)] +fn referenced_parallel_classic_dispatch_bytes( + _plan: ReferencedPlanView<'_>, + _parallel: bool, +) -> Result { + Ok(0) } #[cfg(feature = "parallel")] @@ -283,6 +330,37 @@ pub(in super::super::super) fn max_referenced_classic_dimensions( max_referenced_dimensions(ReferencedPlanView::Classic(plan), false) } +/// Most runs of code-block rows (consecutive jobs sharing `output_y`) in any +/// classic sub-band. Runs are never fewer than the distinct rows one +/// parallel dispatch decodes at once; an unreadable plan is unbounded. +pub(super) fn max_referenced_classic_rows(plan: &J2kReferencedClassicPlan) -> usize { + let view = ReferencedPlanView::Classic(plan); + let mut maximum = 0usize; + for tile_index in 0..view.tile_count() { + for component_index in 0..view.component_count() { + let Ok(component) = view.component_plan(tile_index, component_index) else { + return usize::MAX; + }; + for step in &component.steps { + if let J2kDirectGrayscaleStep::ClassicSubBand(sub_band) = step { + let breaks = sub_band + .jobs + .windows(2) + .filter(|pair| pair[0].output_y != pair[1].output_y) + .count(); + let rows = if sub_band.jobs.is_empty() { + 0 + } else { + breaks + 1 + }; + maximum = maximum.max(rows); + } + } + } + } + maximum +} + pub(in super::super::super) fn max_referenced_ht_dimensions( plan: &J2kReferencedHtj2kPlan, ) -> Option<(u32, u32)> { diff --git a/crates/j2k-native/src/direct_cpu/referenced.rs b/crates/j2k-native/src/direct_cpu/referenced.rs index becc469c5..d2b90ea5c 100644 --- a/crates/j2k-native/src/direct_cpu/referenced.rs +++ b/crates/j2k-native/src/direct_cpu/referenced.rs @@ -4,6 +4,8 @@ mod component; #[cfg(feature = "parallel")] +pub(super) mod rows; +#[cfg(feature = "parallel")] pub(super) use component::parallel_ht_decode_transient_bytes; mod output; mod payload; @@ -92,6 +94,7 @@ fn execute_referenced_htj2k_plan_with_payloads<'scratch>( classic_workspace, ht_workspace, ht_task_workspaces, + classic_task_workspaces: _, parallel, staged_state: _, } = scratch; diff --git a/crates/j2k-native/src/direct_cpu/referenced/component.rs b/crates/j2k-native/src/direct_cpu/referenced/component.rs index c154a8fa7..e33293a51 100644 --- a/crates/j2k-native/src/direct_cpu/referenced/component.rs +++ b/crates/j2k-native/src/direct_cpu/referenced/component.rs @@ -1,7 +1,5 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -#[cfg(feature = "parallel")] -use crate::error::ValidationError; use crate::error::{bail, DecodingError, Result}; use crate::{ decode_ht_code_block_scalar_with_workspace, @@ -207,15 +205,8 @@ fn execute_ht_sub_band_referenced( Ok(()) } -/// One code block of a parallel sub-band decode with its borrowed payload and -/// output range in the sub-band. #[cfg(feature = "parallel")] -struct RowJob<'p, 'i> { - job: &'p crate::HtOwnedCodeBlockBatchJob, - data: &'i [u8], - start: usize, - end: usize, -} +type RowJob<'p, 'i> = super::rows::RowJob<'p, 'i, crate::HtOwnedCodeBlockBatchJob>; /// Bytes owned together while a referenced HT sub-band is dispatched in /// parallel. Planning and execution share this helper so the private dispatch @@ -226,21 +217,10 @@ pub(in crate::direct_cpu) fn parallel_ht_decode_transient_bytes( row_capacity: usize, workspace_capacity: usize, ) -> Result { - entry_capacity - .checked_mul(core::mem::size_of::>()) - .and_then(|bytes| { - row_capacity - .checked_mul(core::mem::size_of::< - CodeBlockRow<'static, 'static, 'static, 'static>, - >()) - .and_then(|rows| bytes.checked_add(rows)) - }) - .and_then(|bytes| { - workspace_capacity - .checked_mul(core::mem::size_of::<&mut HtCodeBlockDecodeWorkspace>()) - .and_then(|workspaces| bytes.checked_add(workspaces)) - }) - .ok_or_else(|| ValidationError::ImageTooLarge.into()) + super::rows::row_dispatch_transient_bytes::< + crate::HtOwnedCodeBlockBatchJob, + HtCodeBlockDecodeWorkspace, + >(entry_capacity, row_capacity, workspace_capacity) } /// Decode a sub-band's rows of code blocks in parallel, each writing its own @@ -264,12 +244,48 @@ fn decode_ht_sub_band_rows_parallel( }; entries.sort_unstable_by_key(|entry| (entry.job.output_y, entry.job.output_x)); let entry_capacity = entries.capacity(); - let Some(rows) = carve_rows(&entries, output, sub_band_width, entry_capacity, ht.budget)? + let Some(rows) = super::rows::carve_rows::<_, HtCodeBlockDecodeWorkspace>( + &entries, + output, + sub_band_width, + entry_capacity, + ht.budget, + )? else { payloads.rewind(checkpoint); return Ok(false); }; - decode_rows(rows, entry_capacity, plan.irreversible_midpoint, ht)?; + let irreversible_midpoint = plan.irreversible_midpoint; + super::rows::decode_rows( + rows, + entry_capacity, + &mut *ht.main, + ht.tasks, + ht.budget, + |entry, output, workspace| { + let job = entry.job; + let code_block = HtCodeBlockDecodeJob { + data: entry.data, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + roi_shift: job.roi_shift, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + }; + if irreversible_midpoint { + decode_ht_code_block_scalar_with_workspace_midpoint(code_block, output, workspace) + } else { + decode_ht_code_block_scalar_with_workspace(code_block, output, workspace) + } + }, + )?; Ok(true) } @@ -318,130 +334,3 @@ fn collect_row_jobs<'p, 'i>( } Ok(Some(entries)) } - -/// A row of code blocks: its output rows, its jobs, and its first sample. -#[cfg(feature = "parallel")] -type CodeBlockRow<'o, 'j, 'p, 'i> = (&'o mut [f32], &'j [RowJob<'p, 'i>], usize); - -/// Split `output` into one slice per row of code blocks, or `None` when the -/// sorted jobs do not form disjoint rows of equal-height blocks. -#[cfg(feature = "parallel")] -fn carve_rows<'o, 'j, 'p, 'i>( - entries: &'j [RowJob<'p, 'i>], - output: &'o mut [f32], - sub_band_width: usize, - entry_capacity: usize, - budget: DirectWorkspaceBudget, -) -> Result>>> { - let mut rows = alloc::vec::Vec::new(); - crate::try_reserve_decode_elements(&mut rows, entries.len())?; - budget.validate_workspace(parallel_ht_decode_transient_bytes( - entry_capacity, - rows.capacity(), - 0, - )?)?; - let mut rest = output; - let mut consumed = 0usize; - let mut index = 0usize; - while index < entries.len() { - let first = &entries[index]; - let (y, height) = (first.job.output_y as usize, first.job.height as usize); - let mut end = index; - let mut next_x = 0u32; - while end < entries.len() && entries[end].job.output_y as usize == y { - let job = entries[end].job; - if job.height as usize != height || job.output_x < next_x { - return Ok(None); - } - next_x = job.output_x + job.width; - end += 1; - } - let row_start = y * sub_band_width; - let row_end = (y + height) * sub_band_width; - if row_start < consumed || row_end > consumed + rest.len() { - return Ok(None); - } - let tail = core::mem::take(&mut rest); - let (_, tail) = tail.split_at_mut(row_start - consumed); - let (row, tail) = tail.split_at_mut(row_end - row_start); - rest = tail; - consumed = row_end; - rows.push((row, &entries[index..end], row_start)); - index = end; - } - Ok(Some(rows)) -} - -/// Decode rows of code blocks on the Rayon pool, each with a pooled workspace. -#[cfg(feature = "parallel")] -fn decode_rows( - rows: alloc::vec::Vec>, - entry_capacity: usize, - irreversible_midpoint: bool, - ht: &mut HtWorkspaces<'_>, -) -> Result<()> { - use rayon::prelude::*; - use std::sync::Mutex; - - let budget = ht.budget; - let row_capacity = rows.capacity(); - let workspace_count = ht - .tasks - .len() - .checked_add(1) - .ok_or(ValidationError::ImageTooLarge)?; - let mut workspaces = alloc::vec::Vec::new(); - crate::try_reserve_decode_elements(&mut workspaces, workspace_count)?; - workspaces.push(&mut *ht.main); - workspaces.extend(ht.tasks.iter_mut()); - budget.validate_workspace(parallel_ht_decode_transient_bytes( - entry_capacity, - row_capacity, - workspaces.capacity(), - )?)?; - let pool = Mutex::new(workspaces); - rows.into_par_iter() - .with_max_len(1) - .try_for_each(|(row, jobs, row_start)| -> Result<()> { - // Bind the popped workspace first so the guard drops before the - // workspace is returned. - let pooled = pool - .lock() - .map_err(|_| DecodingError::CodeBlockDecodeFailure)? - .pop(); - let mut decode_row = |workspace: &mut HtCodeBlockDecodeWorkspace| -> Result<()> { - jobs.iter().try_for_each(|entry| { - let job = entry.job; - let code_block = HtCodeBlockDecodeJob { - data: entry.data, - cleanup_length: job.cleanup_length, - refinement_length: job.refinement_length, - width: job.width, - height: job.height, - output_stride: job.output_stride, - missing_bit_planes: job.missing_bit_planes, - number_of_coding_passes: job.number_of_coding_passes, - num_bitplanes: job.num_bitplanes, - roi_shift: job.roi_shift, - stripe_causal: job.stripe_causal, - strict: job.strict, - dequantization_step: job.dequantization_step, - }; - let output = &mut row[entry.start - row_start..entry.end - row_start]; - if irreversible_midpoint { - decode_ht_code_block_scalar_with_workspace_midpoint( - code_block, output, workspace, - ) - } else { - decode_ht_code_block_scalar_with_workspace(code_block, output, workspace) - } - }) - }; - let workspace = pooled.ok_or(DecodingError::CodeBlockDecodeFailure)?; - let result = decode_row(workspace); - pool.lock() - .map_err(|_| DecodingError::CodeBlockDecodeFailure)? - .push(workspace); - result - }) -} diff --git a/crates/j2k-native/src/direct_cpu/referenced/rows.rs b/crates/j2k-native/src/direct_cpu/referenced/rows.rs new file mode 100644 index 000000000..7910af285 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced/rows.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Row-parallel code-block dispatch shared by referenced HT and classic +//! execution: borrowed jobs are grouped into disjoint rows of one sub-band +//! and decoded on the Rayon pool, each row with a pooled workspace. + +use crate::error::{DecodingError, Result, ValidationError}; +use crate::{HtOwnedCodeBlockBatchJob, J2kOwnedCodeBlockBatchJob}; + +use super::super::DirectWorkspaceBudget; + +/// Output placement of one owned code-block job inside its sub-band. +pub(in crate::direct_cpu) trait CodeBlockRowGeometry: Sync { + fn output_x(&self) -> u32; + fn output_y(&self) -> u32; + fn width(&self) -> u32; + fn height(&self) -> u32; +} + +impl CodeBlockRowGeometry for HtOwnedCodeBlockBatchJob { + fn output_x(&self) -> u32 { + self.output_x + } + fn output_y(&self) -> u32 { + self.output_y + } + fn width(&self) -> u32 { + self.width + } + fn height(&self) -> u32 { + self.height + } +} + +impl CodeBlockRowGeometry for J2kOwnedCodeBlockBatchJob { + fn output_x(&self) -> u32 { + self.output_x + } + fn output_y(&self) -> u32 { + self.output_y + } + fn width(&self) -> u32 { + self.width + } + fn height(&self) -> u32 { + self.height + } +} + +/// One code block of a parallel sub-band decode with its borrowed payload and +/// output range in the sub-band. +pub(in crate::direct_cpu) struct RowJob<'p, 'i, J> { + pub(in crate::direct_cpu) job: &'p J, + pub(in crate::direct_cpu) data: &'i [u8], + pub(in crate::direct_cpu) start: usize, + pub(in crate::direct_cpu) end: usize, +} + +/// A row of code blocks: its output rows, its jobs, and its first sample. +pub(in crate::direct_cpu) type CodeBlockRow<'o, 'j, 'p, 'i, J> = + (&'o mut [f32], &'j [RowJob<'p, 'i, J>], usize); + +/// Bytes owned together while one sub-band is dispatched in parallel. +/// Planning and execution share this helper so the private dispatch types +/// cannot drift from their accounting. +pub(in crate::direct_cpu) fn row_dispatch_transient_bytes( + entry_capacity: usize, + row_capacity: usize, + workspace_capacity: usize, +) -> Result { + entry_capacity + .checked_mul(core::mem::size_of::>()) + .and_then(|bytes| { + row_capacity + .checked_mul(core::mem::size_of::>()) + .and_then(|rows| bytes.checked_add(rows)) + }) + .and_then(|bytes| { + workspace_capacity + .checked_mul(core::mem::size_of::<&mut W>()) + .and_then(|workspaces| bytes.checked_add(workspaces)) + }) + .ok_or_else(|| ValidationError::ImageTooLarge.into()) +} + +/// Split `output` into one slice per row of code blocks, or `None` when the +/// jobs, sorted by position, do not form disjoint rows of equal-height blocks. +pub(in crate::direct_cpu) fn carve_rows<'o, 'j, 'p, 'i, J: CodeBlockRowGeometry, W>( + entries: &'j [RowJob<'p, 'i, J>], + output: &'o mut [f32], + sub_band_width: usize, + entry_capacity: usize, + budget: DirectWorkspaceBudget, +) -> Result>>> { + let mut rows = alloc::vec::Vec::new(); + crate::try_reserve_decode_elements(&mut rows, entries.len())?; + budget.validate_workspace(row_dispatch_transient_bytes::( + entry_capacity, + rows.capacity(), + 0, + )?)?; + let mut rest = output; + let mut consumed = 0usize; + let mut index = 0usize; + while index < entries.len() { + let first = &entries[index]; + let (y, height) = (first.job.output_y() as usize, first.job.height() as usize); + let mut end = index; + let mut next_x = 0u32; + while end < entries.len() && entries[end].job.output_y() as usize == y { + let job = entries[end].job; + if job.height() as usize != height || job.output_x() < next_x { + return Ok(None); + } + next_x = job.output_x() + job.width(); + end += 1; + } + let row_start = y * sub_band_width; + let row_end = (y + height) * sub_band_width; + if row_start < consumed || row_end > consumed + rest.len() { + return Ok(None); + } + let tail = core::mem::take(&mut rest); + let (_, tail) = tail.split_at_mut(row_start - consumed); + let (row, tail) = tail.split_at_mut(row_end - row_start); + rest = tail; + consumed = row_end; + rows.push((row, &entries[index..end], row_start)); + index = end; + } + Ok(Some(rows)) +} + +/// Decode rows of code blocks on the Rayon pool, each with a pooled +/// workspace taken from `main` and `tasks`. +pub(in crate::direct_cpu) fn decode_rows( + rows: alloc::vec::Vec>, + entry_capacity: usize, + main: &mut W, + tasks: &mut [W], + budget: DirectWorkspaceBudget, + decode: F, +) -> Result<()> +where + J: Sync, + W: Send, + F: Fn(&RowJob<'_, '_, J>, &mut [f32], &mut W) -> Result<()> + Sync, +{ + use rayon::prelude::*; + use std::sync::Mutex; + + let row_capacity = rows.capacity(); + let workspace_count = tasks + .len() + .checked_add(1) + .ok_or(ValidationError::ImageTooLarge)?; + let mut workspaces = alloc::vec::Vec::new(); + crate::try_reserve_decode_elements(&mut workspaces, workspace_count)?; + workspaces.push(main); + workspaces.extend(tasks.iter_mut()); + budget.validate_workspace(row_dispatch_transient_bytes::( + entry_capacity, + row_capacity, + workspaces.capacity(), + )?)?; + let pool = Mutex::new(workspaces); + rows.into_par_iter() + .with_max_len(1) + .try_for_each(|(row, jobs, row_start)| -> Result<()> { + // Bind the popped workspace first so the guard drops before the + // workspace is returned. + let pooled = pool + .lock() + .map_err(|_| DecodingError::CodeBlockDecodeFailure)? + .pop(); + let workspace = pooled.ok_or(DecodingError::CodeBlockDecodeFailure)?; + let result = jobs.iter().try_for_each(|entry| { + let output = &mut row[entry.start - row_start..entry.end - row_start]; + decode(entry, output, workspace) + }); + pool.lock() + .map_err(|_| DecodingError::CodeBlockDecodeFailure)? + .push(workspace); + result + }) +} diff --git a/crates/j2k-native/src/direct_cpu/referenced_classic.rs b/crates/j2k-native/src/direct_cpu/referenced_classic.rs index 435d34430..d0d2928da 100644 --- a/crates/j2k-native/src/direct_cpu/referenced_classic.rs +++ b/crates/j2k-native/src/direct_cpu/referenced_classic.rs @@ -15,7 +15,8 @@ use super::referenced::{decoded_color_components, decoded_plane, payload_slice}; use super::{ apply_inverse_mct_region, checked_sub_band_job_output_range, execute_idwt_step, prepare_sub_band_output, rounds_at_store, store_component, DirectComponentBandScratch, - DirectComponentPlane, J2kDirectCpuScratch, J2kDirectDecodedComponents, SubBandJobOutputRange, + DirectComponentPlane, DirectWorkspaceBudget, J2kDirectCpuScratch, J2kDirectDecodedComponents, + SubBandJobOutputRange, }; /// Execute retained per-tile Gray/RGB/RGBA classic JPEG 2000 geometry @@ -76,7 +77,7 @@ fn execute_referenced_classic_plan_with_payloads<'scratch>( bail!(DecodingError::CodeBlockDecodeFailure); } validate_payload_ranges(encoded_input, payload_descriptors, payload_ranges)?; - prepare_referenced_classic_scratch(plan, scratch)?; + let workspace_budget = prepare_referenced_classic_scratch(plan, scratch)?; { let J2kDirectCpuScratch { component_band_sets, @@ -85,9 +86,16 @@ fn execute_referenced_classic_plan_with_payloads<'scratch>( classic_workspace, ht_workspace: _, ht_task_workspaces: _, - parallel: _, + classic_task_workspaces, + parallel, staged_state: _, } = scratch; + let classic_workspace = &mut ClassicWorkspaces { + main: classic_workspace, + tasks: classic_task_workspaces, + parallel: *parallel, + budget: workspace_budget, + }; let mut payloads = ClassicPayloadCursor::new(payload_descriptors, payload_ranges, encoded_input); let mut output_initialized = [false; 4]; @@ -171,7 +179,7 @@ fn execute_color_components_referenced( reconstructed_planes: &mut [DirectComponentPlane], payloads: &mut ClassicPayloadCursor<'_>, compressed_payload: &mut alloc::vec::Vec, - classic_workspace: &mut J2kCodeBlockDecodeWorkspace, + classic_workspace: &mut ClassicWorkspaces<'_>, output_initialized: &mut [bool; 4], destination: crate::J2kRect, ) -> Result<()> { @@ -220,7 +228,7 @@ fn execute_component_plan_referenced( output: &mut DirectComponentPlane, payloads: &mut ClassicPayloadCursor<'_>, compressed_payload: &mut alloc::vec::Vec, - classic_workspace: &mut J2kCodeBlockDecodeWorkspace, + classic_workspace: &mut ClassicWorkspaces<'_>, output_initialized: &mut bool, round_irreversible_output: bool, ) -> Result<()> { @@ -229,7 +237,7 @@ fn execute_component_plan_referenced( for step in &plan.steps { match step { J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { - execute_classic_sub_band_referenced( + execute_classic_sub_band_with_workspaces( sub_band, bands, payloads, @@ -242,7 +250,9 @@ fn execute_component_plan_referenced( "referenced classic CPU plan encountered HT code blocks" )); } - J2kDirectGrayscaleStep::Idwt(step) => execute_idwt_step(step, bands, false)?, + J2kDirectGrayscaleStep::Idwt(step) => { + execute_idwt_step(step, bands, classic_workspace.parallel)?; + } J2kDirectGrayscaleStep::Store(store) => { store_component( store, @@ -262,6 +272,59 @@ fn execute_component_plan_referenced( } } +/// Classic code-block workspaces for one referenced classic execution: the +/// retained main workspace, plus one per extra Rayon worker when `parallel` +/// is set. +struct ClassicWorkspaces<'a> { + main: &'a mut J2kCodeBlockDecodeWorkspace, + #[cfg_attr( + not(feature = "parallel"), + expect(dead_code, reason = "extra workspaces serve parallel execution only") + )] + tasks: &'a mut [J2kCodeBlockDecodeWorkspace], + parallel: bool, + #[cfg_attr( + not(feature = "parallel"), + expect( + dead_code, + reason = "the workspace budget validates parallel dispatch owners" + ) + )] + budget: DirectWorkspaceBudget, +} + +fn execute_classic_sub_band_with_workspaces( + plan: &J2kOwnedSubBandPlan, + bands: &mut DirectComponentBandScratch, + payloads: &mut ClassicPayloadCursor<'_>, + compressed_payload: &mut alloc::vec::Vec, + workspaces: &mut ClassicWorkspaces<'_>, +) -> Result<()> { + let (output, sub_band_width) = + prepare_sub_band_output(bands, plan.band_id, plan.rect, plan.width, plan.height)?; + #[cfg(feature = "parallel")] + if workspaces.parallel + && !workspaces.tasks.is_empty() + && decode_classic_sub_band_rows_parallel( + plan, + output, + sub_band_width, + payloads, + workspaces, + )? + { + return Ok(()); + } + decode_classic_sub_band_jobs( + plan, + output, + sub_band_width, + payloads, + compressed_payload, + workspaces.main, + ) +} + pub(super) fn execute_classic_sub_band_referenced( plan: &J2kOwnedSubBandPlan, bands: &mut DirectComponentBandScratch, @@ -271,6 +334,24 @@ pub(super) fn execute_classic_sub_band_referenced( ) -> Result<()> { let (output, sub_band_width) = prepare_sub_band_output(bands, plan.band_id, plan.rect, plan.width, plan.height)?; + decode_classic_sub_band_jobs( + plan, + output, + sub_band_width, + payloads, + compressed_payload, + workspace, + ) +} + +fn decode_classic_sub_band_jobs( + plan: &J2kOwnedSubBandPlan, + output: &mut [f32], + sub_band_width: usize, + payloads: &mut ClassicPayloadCursor<'_>, + compressed_payload: &mut alloc::vec::Vec, + workspace: &mut J2kCodeBlockDecodeWorkspace, +) -> Result<()> { for job in &plan.jobs { if !job.data.is_empty() { bail!(DecodingError::CodeBlockDecodeFailure); @@ -312,6 +393,131 @@ pub(super) fn execute_classic_sub_band_referenced( Ok(()) } +#[cfg(feature = "parallel")] +type ClassicRowJob<'p, 'i> = + super::referenced::rows::RowJob<'p, 'i, crate::J2kOwnedCodeBlockBatchJob>; + +/// Decode a classic sub-band's rows of code blocks in parallel, each writing +/// its own rows of `output`. Returns `false` without consuming payloads when a +/// block's bytes are not one contiguous input slice or the blocks do not tile +/// the sub-band in rows; the serial path then decodes the sub-band. Every +/// block decodes exactly as on the serial path, so the samples are identical. +#[cfg(feature = "parallel")] +fn decode_classic_sub_band_rows_parallel( + plan: &J2kOwnedSubBandPlan, + output: &mut [f32], + sub_band_width: usize, + payloads: &mut ClassicPayloadCursor<'_>, + workspaces: &mut ClassicWorkspaces<'_>, +) -> Result { + use super::referenced::rows; + + let checkpoint = payloads.position(); + let Some(mut entries) = collect_classic_row_jobs( + plan, + output.len(), + sub_band_width, + payloads, + workspaces.budget, + )? + else { + payloads.rewind(checkpoint); + return Ok(false); + }; + entries.sort_unstable_by_key(|entry| (entry.job.output_y, entry.job.output_x)); + let entry_capacity = entries.capacity(); + let Some(rows) = rows::carve_rows::<_, J2kCodeBlockDecodeWorkspace>( + &entries, + output, + sub_band_width, + entry_capacity, + workspaces.budget, + )? + else { + payloads.rewind(checkpoint); + return Ok(false); + }; + let irreversible_midpoint = plan.irreversible_midpoint; + rows::decode_rows( + rows, + entry_capacity, + &mut *workspaces.main, + workspaces.tasks, + workspaces.budget, + |entry, output, workspace| { + let job = entry.job; + let code_block = J2kCodeBlockDecodeJob { + data: entry.data, + segments: &job.segments, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + total_bitplanes: job.total_bitplanes, + roi_shift: job.roi_shift, + sub_band_type: job.sub_band_type, + style: job.style, + strict: job.strict, + dequantization_step: job.dequantization_step, + }; + if irreversible_midpoint { + decode_j2k_code_block_scalar_with_workspace_midpoint(code_block, output, workspace) + } else { + decode_j2k_code_block_scalar_with_workspace(code_block, output, workspace) + } + }, + )?; + Ok(true) +} + +/// Validate every job's output range and borrow its payload, or `None` when +/// a payload is split over several input fragments. +#[cfg(feature = "parallel")] +fn collect_classic_row_jobs<'p, 'i>( + plan: &'p J2kOwnedSubBandPlan, + output_len: usize, + sub_band_width: usize, + payloads: &mut ClassicPayloadCursor<'i>, + budget: DirectWorkspaceBudget, +) -> Result>>> { + let mut entries = alloc::vec::Vec::new(); + try_reserve_decode_elements(&mut entries, plan.jobs.len())?; + budget.validate_workspace(super::referenced::rows::row_dispatch_transient_bytes::< + crate::J2kOwnedCodeBlockBatchJob, + J2kCodeBlockDecodeWorkspace, + >(entries.capacity(), 0, 0)?)?; + for job in &plan.jobs { + if !job.data.is_empty() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let range = checked_sub_band_job_output_range(&SubBandJobOutputRange { + output_x: job.output_x, + output_y: job.output_y, + output_stride: job.output_stride, + width: job.width, + height: job.height, + sub_band_width, + plan_width: plan.width, + plan_height: plan.height, + output_len, + })?; + if job.output_stride != sub_band_width { + return Ok(None); + } + let Some(data) = payloads.next_contiguous()? else { + return Ok(None); + }; + entries.push(ClassicRowJob { + job, + data, + start: range.start, + end: range.end, + }); + } + Ok(Some(entries)) +} + pub(super) fn validate_payload_ranges( encoded_input: &[u8], payloads: &[crate::J2kClassicCodeBlockPayload], @@ -405,6 +611,47 @@ impl<'plan> ClassicPayloadCursor<'plan> { Ok(combined) } + /// Position for [`Self::rewind`] when a parallel attempt falls back. + #[cfg(feature = "parallel")] + const fn position(&self) -> usize { + self.next + } + + #[cfg(feature = "parallel")] + fn rewind(&mut self, position: usize) { + self.next = position; + } + + /// The next payload borrowed from the input when it is one contiguous + /// fragment (or empty), else `None` without consuming it. + #[cfg(feature = "parallel")] + fn next_contiguous(&mut self) -> Result> { + let payload = self + .payloads + .get(self.next) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let end_range = payload + .end_range() + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let fragments = self + .ranges + .get(payload.first_range..end_range) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let data = match fragments { + [] if payload.combined_length == 0 => &[][..], + [range] if range.length == payload.combined_length => { + payload_slice(self.encoded_input, *range)? + } + [_] => bail!(DecodingError::CodeBlockDecodeFailure), + _ => return Ok(None), + }; + self.next = self + .next + .checked_add(1) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + Ok(Some(data)) + } + pub(super) fn ensure_exhausted(&self) -> Result<()> { if self.next == self.payloads.len() { Ok(()) @@ -486,3 +733,6 @@ pub(super) fn decoded_components<'scratch>( } } } + +#[cfg(test)] +mod tests; diff --git a/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs b/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs new file mode 100644 index 000000000..fd0839217 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use alloc::vec::Vec; + +use crate::{ + encode, execute_referenced_classic_plan, DecodeSettings, DecoderContext, EncodeOptions, Image, + J2kDirectCpuScratch, +}; + +fn rgb_fixture(width: u32, height: u32, reversible: bool) -> Vec { + let samples = (0..width * height * 3) + .map(|index| { + let (pixel, channel) = (index / 3, index % 3); + let (x, y) = (pixel % width, pixel / width); + u8::try_from((x * 5 + y * 3 + channel * 41 + (x ^ y) % 37) & 0xff) + .expect("masked sample") + }) + .collect::>(); + let options = EncodeOptions { + reversible, + num_decomposition_levels: 4, + ..EncodeOptions::default() + }; + encode(&samples, width, height, 3, 8, false, &options).expect("encode classic RGB fixture") +} + +/// Every reconstructed plane as raw bits, plus the retained classic +/// workspace bytes after the execution. +fn execute_whole_plan(bytes: &[u8], parallel: bool) -> (Vec>, usize) { + let image = Image::new(bytes, &DecodeSettings::default()).expect("classic image"); + let region = (0, 0, image.width(), image.height()); + let plan = image + .build_referenced_classic_plan_region_with_context(&mut DecoderContext::default(), region) + .expect("classic whole-image plan"); + let mut scratch = J2kDirectCpuScratch::new(); + scratch.set_parallel_decode(parallel); + let decoded = execute_referenced_classic_plan(&plan, bytes, false, &mut scratch) + .expect("execute referenced classic plan"); + let planes = (0..3) + .map(|component| { + decoded + .plane(component) + .expect("RGB plane") + .samples() + .iter() + .map(|sample| sample.to_bits()) + .collect() + }) + .collect(); + (planes, scratch.retained_classic_workspace_bytes()) +} + +#[cfg(feature = "parallel")] +#[test] +fn parallel_referenced_classic_execution_matches_serial_bits() { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .expect("four-thread pool"); + for reversible in [true, false] { + // Several rows of 64 x 64 code blocks per high-resolution sub-band. + let bytes = rgb_fixture(300, 260, reversible); + let (serial, serial_workspace_bytes) = execute_whole_plan(&bytes, false); + let (parallel, parallel_workspace_bytes) = + pool.install(|| execute_whole_plan(&bytes, true)); + assert_eq!(serial, parallel, "reversible={reversible}"); + // The tallest sub-band (130 rows) holds three rows of 64-high + // blocks, so of the four workers only two extra ones can take a row + // and keep their own prepared workspace. + assert_eq!( + parallel_workspace_bytes, + serial_workspace_bytes * 3, + "reversible={reversible}" + ); + } +} + +#[test] +fn parallel_request_on_the_global_pool_matches_serial_bits() { + let bytes = rgb_fixture(200, 136, false); + let (serial, _) = execute_whole_plan(&bytes, false); + let (parallel, _) = execute_whole_plan(&bytes, true); + assert_eq!(serial, parallel); +} diff --git a/crates/j2k/src/batch/worker/owned.rs b/crates/j2k/src/batch/worker/owned.rs index ce1dac106..3f239838a 100644 --- a/crates/j2k/src/batch/worker/owned.rs +++ b/crates/j2k/src/batch/worker/owned.rs @@ -136,6 +136,8 @@ impl BatchWorker { signed: bool, ) -> j2k_native::Result> { self.prepared_plan_decode_calls = self.prepared_plan_decode_calls.saturating_add(1); + self.prepared_plan_scratch + .set_parallel_decode(self.owned_inner_parallelism == CpuDecodeParallelism::Auto); j2k_native::execute_referenced_classic_plan( plan, encoded_input, From ad19472a256a76cc5fd8d105e36b31894b96c701 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 21:23:23 -0400 Subject: [PATCH 50/53] fix(j2k): check prepared batch memory during prepare and admit decodes by plan Prepared batches had no byte admission, and an offset plan whose execution exceeds the per-decode bound (4096 x 4096 RGB, 6656 x 6656 gray, the 5661 x 8287 library scan) only failed once decode had started; an image whose own output exceeds the per-allocation cap aborted the whole decode call. Preparation now plans each offset plan's execution peak with the accounting its execution enforces (new hidden j2k-native planners, no allocation) and records a CPU-only choice on the prepared image; device decoders see the same codec plan as before. CpuBatchDecoder::prepare rejects, as indexed prepare errors, images whose serial execution exceeds the bound and images whose own output exceeds the cap; batches prepared for any decoder report the same error at decode before any decode work. decode_prepared admits every execution under the batch execution cap with one claim per worker at the largest plan it ran. Bytes and decode times unchanged where decodes succeeded before; oversized items fail in prepare instead of decode. --- crates/j2k-native/src/direct_cpu.rs | 38 ++++- .../j2k-native/src/direct_cpu/allocation.rs | 1 + .../src/direct_cpu/allocation/referenced.rs | 57 ++++++- crates/j2k-native/src/lib.rs | 7 +- crates/j2k/src/batch.rs | 2 +- crates/j2k/src/batch/admission.rs | 25 ++- crates/j2k/src/batch/admission/tests.rs | 14 ++ crates/j2k/src/batch/allocation.rs | 2 +- crates/j2k/src/owned_batch.rs | 4 +- crates/j2k/src/owned_batch/cpu.rs | 13 +- crates/j2k/src/owned_batch/cpu_execute.rs | 15 +- crates/j2k/src/owned_batch/cpu_materialize.rs | 13 +- crates/j2k/src/owned_batch/prepare.rs | 86 ++++++++++- .../prepare/cpu_preflight_tests.rs | 144 ++++++++++++++++++ crates/j2k/src/owned_batch/prepare/image.rs | 58 ++++++- crates/j2k/src/owned_batch/prepared.rs | 35 +++++ ...able-api-1.0.implementation-public-api.txt | 2 + 17 files changed, 485 insertions(+), 31 deletions(-) create mode 100644 crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs diff --git a/crates/j2k-native/src/direct_cpu.rs b/crates/j2k-native/src/direct_cpu.rs index 18d287cb0..f3b8fd142 100644 --- a/crates/j2k-native/src/direct_cpu.rs +++ b/crates/j2k-native/src/direct_cpu.rs @@ -16,7 +16,10 @@ use crate::{ }; mod allocation; -use allocation::{prepare_direct_scratch, DirectWorkspaceBudget}; +use allocation::{ + planned_referenced_classic_execution_bytes, planned_referenced_htj2k_execution_bytes, + prepare_direct_scratch, DirectWorkspaceBudget, +}; mod color; use color::apply_inverse_mct_region; pub use color::{execute_direct_color_plan_rgb8_into, execute_direct_color_plan_rgba8_into}; @@ -46,6 +49,39 @@ pub use referenced_classic::{ execute_referenced_classic_plan, execute_referenced_classic_plan_from_payloads, }; +/// Planned peak bytes one execution of `plan` allocates into empty +/// [`J2kDirectCpuScratch`], including the retained plan, computed without +/// allocating. `parallel` selects the extra owners of parallel execution +/// ([`J2kDirectCpuScratch::set_parallel_decode`]). +/// +/// # Errors +/// +/// Returns [`ValidationError::ImageTooLarge`](crate::ValidationError) when +/// the execution exceeds the per-decode memory bound execution enforces, or +/// another error when the plan is malformed. +#[doc(hidden)] +pub fn referenced_htj2k_plan_execution_bytes( + plan: &crate::J2kReferencedHtj2kPlan, + parallel: bool, +) -> Result { + planned_referenced_htj2k_execution_bytes(plan, parallel) +} + +/// Classic counterpart of [`referenced_htj2k_plan_execution_bytes`]. +/// +/// # Errors +/// +/// Returns [`ValidationError::ImageTooLarge`](crate::ValidationError) when +/// the execution exceeds the per-decode memory bound execution enforces, or +/// another error when the plan is malformed. +#[doc(hidden)] +pub fn referenced_classic_plan_execution_bytes( + plan: &crate::J2kReferencedClassicPlan, + parallel: bool, +) -> Result { + planned_referenced_classic_execution_bytes(plan, parallel) +} + /// Adapter reusable scratch for executing direct J2K RGB plans on the CPU. #[derive(Debug, Default)] pub struct J2kDirectCpuScratch { diff --git a/crates/j2k-native/src/direct_cpu/allocation.rs b/crates/j2k-native/src/direct_cpu/allocation.rs index 141c71a19..6f7c22fa8 100644 --- a/crates/j2k-native/src/direct_cpu/allocation.rs +++ b/crates/j2k-native/src/direct_cpu/allocation.rs @@ -19,6 +19,7 @@ use core::mem::size_of; mod referenced; pub(super) use self::referenced::{ max_referenced_classic_dimensions, max_referenced_ht_dimensions, + planned_referenced_classic_execution_bytes, planned_referenced_htj2k_execution_bytes, prepare_referenced_classic_scratch, prepare_referenced_classic_staged_scratch, prepare_referenced_direct_scratch, prepare_referenced_htj2k_staged_scratch, }; diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs index 9743bbc2d..c3f88c9ea 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs @@ -37,6 +37,58 @@ pub(in super::super) fn prepare_referenced_direct_scratch( ) } +/// Planned peak bytes of one referenced HTJ2K execution into empty scratch, +/// from the accounting execution enforces, without allocating. +pub(in super::super) fn planned_referenced_htj2k_execution_bytes( + plan: &J2kReferencedHtj2kPlan, + parallel: bool, +) -> Result { + let view = ReferencedPlanView::Htj2k(plan); + let mut scratch = J2kDirectCpuScratch::new(); + scratch.parallel = parallel; + validate_referenced_aggregate_plan( + view, + parallel, + plan.retained_allocation_bytes()?, + max_referenced_payload_bytes(plan)?, + max_referenced_dimensions(view, false), + max_referenced_dimensions(view, true), + None, + &scratch, + ) + .map(DirectWorkspaceBudget::peak_bytes) +} + +/// Planned peak bytes of one referenced classic execution into empty +/// scratch, from the accounting execution enforces, without allocating. +pub(in super::super) fn planned_referenced_classic_execution_bytes( + plan: &J2kReferencedClassicPlan, + parallel: bool, +) -> Result { + let mut scratch = J2kDirectCpuScratch::new(); + scratch.parallel = parallel; + let classic_dimensions = max_referenced_classic_dimensions(plan); + validate_referenced_aggregate_plan( + ReferencedPlanView::Classic(plan), + parallel, + plan.retained_allocation_bytes()?, + max_referenced_classic_payload_bytes(plan), + classic_dimensions, + None, + classic_task_shape(plan), + &scratch, + ) + .map(DirectWorkspaceBudget::peak_bytes) +} + +/// Shape of the extra classic workspaces parallel execution of `plan` uses. +fn classic_task_shape(plan: &J2kReferencedClassicPlan) -> Option { + Some(ClassicTaskShape { + dimensions: max_referenced_classic_dimensions(plan)?, + rows: max_referenced_classic_rows(plan), + }) +} + pub(in super::super) fn prepare_referenced_classic_scratch( plan: &J2kReferencedClassicPlan, scratch: &mut J2kDirectCpuScratch, @@ -44,10 +96,7 @@ pub(in super::super) fn prepare_referenced_classic_scratch( let view = ReferencedPlanView::Classic(plan); let parallel = scratch.parallel; let classic_dimensions = max_referenced_classic_dimensions(plan); - let classic_tasks = classic_dimensions.map(|dimensions| ClassicTaskShape { - dimensions, - rows: max_referenced_classic_rows(plan), - }); + let classic_tasks = classic_task_shape(plan); prepare_referenced_component_scratch( view, parallel, diff --git a/crates/j2k-native/src/lib.rs b/crates/j2k-native/src/lib.rs index d385338aa..92a2bb40e 100644 --- a/crates/j2k-native/src/lib.rs +++ b/crates/j2k-native/src/lib.rs @@ -187,9 +187,10 @@ pub use direct_cpu::{ finish_referenced_htj2k_staged, finish_referenced_htj2k_tile_staged, prepare_referenced_classic_entropy_workspace, prepare_referenced_classic_staged, prepare_referenced_classic_tile_staged, prepare_referenced_htj2k_entropy_workspace, - prepare_referenced_htj2k_staged, prepare_referenced_htj2k_tile_staged, J2kDirectCodeBlockIndex, - J2kDirectCpuEntropyWorkspace, J2kDirectCpuScratch, J2kDirectDecodedComponents, - J2kDirectDecodedPlane, + prepare_referenced_htj2k_staged, prepare_referenced_htj2k_tile_staged, + referenced_classic_plan_execution_bytes, referenced_htj2k_plan_execution_bytes, + J2kDirectCodeBlockIndex, J2kDirectCpuEntropyWorkspace, J2kDirectCpuScratch, + J2kDirectDecodedComponents, J2kDirectDecodedPlane, }; #[doc(hidden)] pub use direct_plan::{ diff --git a/crates/j2k/src/batch.rs b/crates/j2k/src/batch.rs index ef8a42b95..9b89eff78 100644 --- a/crates/j2k/src/batch.rs +++ b/crates/j2k/src/batch.rs @@ -10,7 +10,7 @@ use j2k_core::{BatchResultSlot, DecodeOutcome, PixelFormat, Rect, TileBatchDecod use crate::{J2kCodec, J2kContext, J2kDecodeWarning, J2kError, J2kScratchPool}; -mod admission; +pub(crate) mod admission; pub(crate) mod allocation; mod direct; mod host_memory; diff --git a/crates/j2k/src/batch/admission.rs b/crates/j2k/src/batch/admission.rs index f875d296f..beec297f2 100644 --- a/crates/j2k/src/batch/admission.rs +++ b/crates/j2k/src/batch/admission.rs @@ -8,7 +8,7 @@ use j2k_core::BatchInfrastructureError; use super::allocation::batch_execution_cap_bytes; -pub(super) struct BatchAllocationBudget { +pub(crate) struct BatchAllocationBudget { cap: usize, /// Bytes charged for the budget's whole lifetime (a shared plan). No /// claim ever releases them, so a claim above `cap - baseline` can never @@ -26,7 +26,7 @@ struct AdmissionState { } impl BatchAllocationBudget { - pub(super) fn with_baseline(baseline: usize) -> Result, BatchInfrastructureError> { + pub(crate) fn with_baseline(baseline: usize) -> Result, BatchInfrastructureError> { let cap = batch_execution_cap_bytes(); if baseline > cap { return Err(BatchInfrastructureError::AllocationTooLarge { @@ -48,7 +48,7 @@ impl BatchAllocationBudget { })) } - pub(super) fn claim( + pub(crate) fn claim( self: &Arc, bytes: usize, ) -> Result { @@ -125,7 +125,7 @@ impl BatchAllocationBudget { } } -pub(super) struct BatchAllocationClaim { +pub(crate) struct BatchAllocationClaim { budget: Arc, bytes: usize, } @@ -158,6 +158,23 @@ impl BatchAllocationClaim { } } +/// Keep `held` at least `bytes`: one owner's claim at the largest planned +/// execution it has run, because its retained scratch keeps that peak while +/// later executions reuse it. A smaller claim is released before waiting for +/// the larger one, so an owner never holds two. +pub(crate) fn hold_at_least( + budget: &Arc, + held: &mut Option, + bytes: usize, +) -> Result<(), BatchInfrastructureError> { + if held.as_ref().is_some_and(|claim| claim.bytes >= bytes) { + return Ok(()); + } + *held = None; + *held = Some(budget.claim(bytes)?); + Ok(()) +} + impl Drop for BatchAllocationClaim { fn drop(&mut self) { let mut state = self.budget.lock_state(); diff --git a/crates/j2k/src/batch/admission/tests.rs b/crates/j2k/src/batch/admission/tests.rs index d12df4b6c..f3c3f2586 100644 --- a/crates/j2k/src/batch/admission/tests.rs +++ b/crates/j2k/src/batch/admission/tests.rs @@ -143,3 +143,17 @@ fn poisoned_admission_state_is_a_typed_infrastructure_error() { Err(BatchInfrastructureError::SchedulerPoisoned) )); } + +#[test] +fn held_claim_grows_to_the_largest_execution_and_never_doubles() { + let budget = BatchAllocationBudget::with_baseline(0).expect("budget"); + let mut held = None; + hold_at_least(&budget, &mut held, 100).expect("first claim"); + assert_eq!(budget.lock_state().live, 100); + hold_at_least(&budget, &mut held, 40).expect("smaller execution reuses the claim"); + assert_eq!(budget.lock_state().live, 100); + hold_at_least(&budget, &mut held, 300).expect("larger execution grows the claim"); + assert_eq!(budget.lock_state().live, 300); + drop(held); + assert_eq!(budget.lock_state().live, 0); +} diff --git a/crates/j2k/src/batch/allocation.rs b/crates/j2k/src/batch/allocation.rs index f14b99543..13817929c 100644 --- a/crates/j2k/src/batch/allocation.rs +++ b/crates/j2k/src/batch/allocation.rs @@ -25,7 +25,7 @@ const HOST_MEMORY_SHARE_DIVISOR: u64 = 2; pub(crate) const J2K_BATCH_METADATA_ALLOWANCE_BYTES: usize = 64 * 1024 * 1024; /// Authoritative worst-case claim for one generic native decoder worker. -pub(super) const GENERIC_WORKER_CLAIM_BYTES: usize = j2k_native::DEFAULT_MAX_DECODE_BYTES; +pub(crate) const GENERIC_WORKER_CLAIM_BYTES: usize = j2k_native::DEFAULT_MAX_DECODE_BYTES; /// Largest claim count whose execution cap plus metadata still fits `usize`. const MAX_SCALED_GENERIC_BATCH_WORKERS: usize = diff --git a/crates/j2k/src/owned_batch.rs b/crates/j2k/src/owned_batch.rs index a2d68e850..593c02bf4 100644 --- a/crates/j2k/src/owned_batch.rs +++ b/crates/j2k/src/owned_batch.rs @@ -37,7 +37,7 @@ pub use self::errors::{ mod prepared; pub use self::prepared::{BatchDecoder, PreparedBatch, PreparedBatchGroup, PreparedImage}; -use self::prepared::{BatchExecutionShape, PreparedCodecPlan, PreparedImageInner}; +use self::prepared::{BatchExecutionShape, CpuExecution, PreparedCodecPlan, PreparedImageInner}; mod prepared_plan; pub use self::prepared_plan::{ @@ -63,10 +63,10 @@ pub use self::cpu::{CpuBatchDecodeResult, CpuBatchDecoder, CpuBatchGroup, CpuBat mod cpu_diagnostics; pub use self::cpu_diagnostics::CpuBatchWorkspaceStats; mod prepare; -use self::prepare::prepare_batch_with_workers; #[cfg(test)] use self::prepare::prepare_staging_bytes; pub use self::prepare::{prepare_batch, prepare_batch_from_images}; +use self::prepare::{prepare_batch_from_images_for, prepare_batch_with_workers, PrepareFor}; #[cfg(test)] mod tests { diff --git a/crates/j2k/src/owned_batch/cpu.rs b/crates/j2k/src/owned_batch/cpu.rs index 67aeec9ea..9a48065f8 100644 --- a/crates/j2k/src/owned_batch/cpu.rs +++ b/crates/j2k/src/owned_batch/cpu.rs @@ -3,12 +3,13 @@ //! Persistent CPU batch session and public native output owners. use super::{ - decode_cpu_group, fmt, generic_batch_worker_limit, prepare_batch_from_images, + decode_cpu_group, fmt, generic_batch_worker_limit, prepare_batch_from_images_for, prepare_batch_with_workers, BatchDecodeOptions, BatchDecoder, BatchGroupInfo, BatchInfrastructureError, BatchWorker, CpuBatchWorkspaceStats, CpuGroupFastWorkspace, CpuStagedWorkspace, EncodedImage, IndexedBatchError, J2kDecodeWarning, Mutex, NativeSampleType, - NonZeroUsize, PreparedBatch, PreparedImage, Rect, Vec, + NonZeroUsize, PrepareFor, PreparedBatch, PreparedImage, Rect, Vec, }; +use crate::batch::admission::BatchAllocationBudget; use crate::batch::allocation::try_vec_with_capacity; use crate::batch::scheduler::decode_retained_tiles; use crate::{DecodeOutcome, PixelFormat, TileBatchError, TileBatchOptions, TileDecodeJob}; @@ -327,7 +328,7 @@ impl CpuBatchDecoder { .workers .lock() .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)?; - prepare_batch_with_workers(inputs, self.options, &mut workers) + prepare_batch_with_workers(inputs, self.options, &mut workers, PrepareFor::CpuDecoder) } /// Regroup caller-supplied prepared images under this session's output policy. @@ -335,7 +336,7 @@ impl CpuBatchDecoder { &self, images: Vec, ) -> Result { - prepare_batch_from_images(images, self.options) + prepare_batch_from_images_for(images, self.options, PrepareFor::CpuDecoder) } /// Prepare and decode one batch. @@ -380,6 +381,9 @@ impl CpuBatchDecoder { try_vec_with_capacity(prepared.groups().len(), "J2K owned batch result groups")?; let mut errors = try_vec_with_capacity(input_count, "J2K owned batch indexed errors")?; errors.extend_from_slice(prepared.errors()); + // Every execution in flight holds a claim sized from its prepared + // plan, under the same aggregate cap as the tile-batch facade. + let admission = BatchAllocationBudget::with_baseline(0)?; for group in prepared.groups() { if let Some(decoded) = decode_cpu_group( workers, @@ -388,6 +392,7 @@ impl CpuBatchDecoder { group, prepared.options, self.options.workers, + &admission, &mut errors, )? { groups.push(decoded); diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index 6685400da..a87a14af0 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -14,11 +14,13 @@ use super::{ CpuStagedWorkspace, IndexedBatchError, J2kError, NativeSampleType, NonZeroUsize, PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, }; +use crate::batch::admission::{hold_at_least, BatchAllocationBudget}; use crate::batch::allocation::try_vec_with_capacity; #[expect( clippy::too_many_lines, - reason = "the typed output allocation and one shared decode/compaction boundary remain together so every native sample type follows the same group-level ownership rules" + clippy::too_many_arguments, + reason = "the typed output allocation and one shared decode/compaction boundary remain together so every native sample type follows the same group-level ownership rules; retained workspaces and admission stay explicit" )] pub(super) fn decode_cpu_group( workers: &mut [BatchWorker], @@ -27,6 +29,7 @@ pub(super) fn decode_cpu_group( group: &PreparedBatchGroup, options: BatchDecodeOptions, workers_option: Option, + admission: &Arc, errors: &mut Vec, ) -> Result, BatchInfrastructureError> { let tile_options = TileBatchOptions::new(workers_option); @@ -67,6 +70,7 @@ pub(super) fn decode_cpu_group( &mut values, has_flattened_payloads.then_some(&mut *fast_workspace), staged_workspace, + admission, decode_image_u8, convert_u8, errors, @@ -88,6 +92,7 @@ pub(super) fn decode_cpu_group( &mut values, has_flattened_payloads.then_some(&mut *fast_workspace), staged_workspace, + admission, decode_image_u16, convert_u16, errors, @@ -109,6 +114,7 @@ pub(super) fn decode_cpu_group( &mut values, has_flattened_payloads.then_some(&mut *fast_workspace), staged_workspace, + admission, decode_image_i16, round_signed, errors, @@ -180,6 +186,7 @@ fn run_typed_group( output: &mut Vec, flattened_payloads: Option<&mut CpuGroupFastWorkspace>, staged_workspace: &mut CpuStagedWorkspace, + admission: &Arc, decode: CpuDecodeFn, convert: fn(f32, u8) -> T, errors: &mut Vec, @@ -220,7 +227,13 @@ fn run_typed_group( let parallelism = worker.prepare_owned_decode(); let workspace = worker.take_native_workspace(); let mut context = j2k_native::DecoderContext::from_workspace(workspace); + let mut claim = None; for (job, slot) in jobs.iter_mut().zip(results) { + hold_at_least( + admission, + &mut claim, + job.image.cpu_execution().claim_bytes(), + )?; *slot = Some(decode( job.image, options, diff --git a/crates/j2k/src/owned_batch/cpu_materialize.rs b/crates/j2k/src/owned_batch/cpu_materialize.rs index b7ea9eb12..f9dda639d 100644 --- a/crates/j2k/src/owned_batch/cpu_materialize.rs +++ b/crates/j2k/src/owned_batch/cpu_materialize.rs @@ -4,8 +4,8 @@ use super::{ backend, decode_prepared_plan_samples, size_of, BatchDecodeOptions, BatchGroupInfo, - BatchInfrastructureError, BatchLayout, BatchWorker, CpuDecodeParallelism, J2kError, - NativeSampleType, PreparedImage, Rect, Vec, + BatchInfrastructureError, BatchLayout, BatchWorker, CpuDecodeParallelism, CpuExecution, + J2kError, NativeSampleType, PreparedImage, Rect, Vec, }; pub(super) fn ensure_batch_output_within_cap( @@ -91,6 +91,9 @@ pub(super) fn decode_image_u8<'image>( worker: &mut BatchWorker, out: &mut [u8], ) -> Result<(), J2kError> { + if image.cpu_execution() == CpuExecution::ExceedsDecodeBound { + return Err(CpuExecution::decode_bound_error()); + } if decode_prepared_plan_samples(image, info, worker, out, |sample, precision| { u8::try_from(round_unsigned(sample, precision)).unwrap_or(u8::MAX) })? { @@ -113,6 +116,9 @@ pub(super) fn decode_image_u16<'image>( worker: &mut BatchWorker, out: &mut [u16], ) -> Result<(), J2kError> { + if image.cpu_execution() == CpuExecution::ExceedsDecodeBound { + return Err(CpuExecution::decode_bound_error()); + } if decode_prepared_plan_samples(image, info, worker, out, |sample, precision| { u16::try_from(round_unsigned(sample, precision)).unwrap_or(u16::MAX) })? { @@ -135,6 +141,9 @@ pub(super) fn decode_image_i16<'image>( worker: &mut BatchWorker, out: &mut [i16], ) -> Result<(), J2kError> { + if image.cpu_execution() == CpuExecution::ExceedsDecodeBound { + return Err(CpuExecution::decode_bound_error()); + } if decode_prepared_plan_samples(image, info, worker, out, round_signed)? { return Ok(()); } diff --git a/crates/j2k/src/owned_batch/prepare.rs b/crates/j2k/src/owned_batch/prepare.rs index 3f5a40f3f..84ce312c9 100644 --- a/crates/j2k/src/owned_batch/prepare.rs +++ b/crates/j2k/src/owned_batch/prepare.rs @@ -4,16 +4,28 @@ use super::{ generic_batch_worker_limit, run_retained_chunks, size_of, Arc, BatchDecodeOptions, - BatchInfrastructureError, BatchItemError, BatchWorker, EncodedImage, IndexedBatchError, - NonZeroUsize, PrepareImageResult, PrepareJob, PreparedBatch, PreparedBatchGroup, PreparedImage, + BatchErrorStage, BatchGroupInfo, BatchInfrastructureError, BatchItemError, BatchWorker, + CpuExecution, EncodedImage, IndexedBatchError, J2kError, NativeSampleType, NonZeroUsize, + PrepareImageResult, PrepareJob, PreparedBatch, PreparedBatchGroup, PreparedImage, TileBatchOptions, Vec, J2K_BATCH_METADATA_ALLOWANCE_BYTES, }; +#[cfg(test)] +mod cpu_preflight_tests; mod image; use self::image::{ batch_execution_shape, batch_group_info, prepare_image, reconcile_codec_plan_metadata, }; +/// Which decoder a preparation serves. Shared preparation keeps every image +/// a device decoder may execute; the CPU decoder also rejects, per image and +/// before any decode work, what its own execution cannot hold. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PrepareFor { + AnyDecoder, + CpuDecoder, +} + /// Inspect, validate, and group owned encoded images without decoding pixels. /// /// Per-image failures are retained in [`PreparedBatch::errors`]. The outer @@ -33,13 +45,14 @@ pub fn prepare_batch( for _ in 0..worker_count { workers.push(BatchWorker::new_owned(input_count.max(2))); } - prepare_batch_with_workers(inputs, options, &mut workers) + prepare_batch_with_workers(inputs, options, &mut workers, PrepareFor::AnyDecoder) } pub(super) fn prepare_batch_with_workers( inputs: Vec, options: BatchDecodeOptions, workers: &mut [BatchWorker], + prepare_for: PrepareFor, ) -> Result { prepare_staging_bytes(inputs.len())?; let mut groups: Vec = Vec::new(); @@ -81,7 +94,14 @@ pub(super) fn prepare_batch_with_workers( index: source_index, }); }; - push_prepared_result(&mut groups, &mut errors, source_index, result, options)?; + push_prepared_result( + &mut groups, + &mut errors, + source_index, + result, + options, + prepare_for, + )?; } Ok(PreparedBatch { @@ -103,13 +123,28 @@ pub(super) fn prepare_batch_with_workers( pub fn prepare_batch_from_images( images: Vec, options: BatchDecodeOptions, +) -> Result { + prepare_batch_from_images_for(images, options, PrepareFor::AnyDecoder) +} + +pub(super) fn prepare_batch_from_images_for( + images: Vec, + options: BatchDecodeOptions, + prepare_for: PrepareFor, ) -> Result { prepare_staging_bytes(images.len())?; let mut groups = Vec::new(); let mut errors = Vec::new(); for (source_index, image) in images.into_iter().enumerate() { let result = regroup_prepared_image(image, options); - push_prepared_result(&mut groups, &mut errors, source_index, result, options)?; + push_prepared_result( + &mut groups, + &mut errors, + source_index, + result, + options, + prepare_for, + )?; } Ok(PreparedBatch { groups: Arc::from(groups), @@ -138,7 +173,15 @@ fn push_prepared_result( source_index: usize, result: PrepareImageResult, options: BatchDecodeOptions, + prepare_for: PrepareFor, ) -> Result<(), BatchInfrastructureError> { + let result = match (prepare_for, result) { + (PrepareFor::CpuDecoder, Ok((image, info, execution_shape))) => { + cpu_preflight_error(&image, &info) + .map_or_else(|| Ok((image, info, execution_shape)), Err) + } + (_, result) => result, + }; match result { Ok((image, info, execution_shape)) => { if let Some(group) = groups @@ -178,6 +221,39 @@ fn push_prepared_result( Ok(()) } +/// The CPU decoder's per-image checks, from planned bytes only: an offset +/// plan whose execution exceeds the per-decode bound, or one image whose own +/// output exceeds the per-allocation output cap, can never decode, so each +/// fails here as an indexed error instead of at decode time. +fn cpu_preflight_error(image: &PreparedImage, info: &BatchGroupInfo) -> Option { + let source = if image.cpu_execution() == CpuExecution::ExceedsDecodeBound { + CpuExecution::decode_bound_error() + } else { + let sample_bytes = match info.sample_type { + NativeSampleType::U8 => 1, + NativeSampleType::U16 | NativeSampleType::I16 => 2, + // Decode reports an unsupported sample width itself. + _ => return None, + }; + let output_bytes = info + .samples_per_image() + .and_then(|samples| samples.checked_mul(sample_bytes)) + .unwrap_or(usize::MAX); + if output_bytes <= j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES { + return None; + } + J2kError::from_native_decode_error(j2k_native::DecodeError::AllocationTooLarge { + what: "J2K owned batch output of one image", + requested: output_bytes, + cap: j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES, + }) + }; + Some(BatchItemError::Codec { + stage: BatchErrorStage::Prepare, + source: Arc::new(source), + }) +} + pub(super) fn prepare_staging_bytes(input_count: usize) -> Result { let execution_item_bytes = size_of::() .checked_add(size_of::>()) diff --git a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs new file mode 100644 index 000000000..82fb982d9 --- /dev/null +++ b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use alloc::{sync::Arc, vec, vec::Vec}; +use core::num::NonZeroUsize; + +use super::super::{ + prepare_batch, BatchDecodeOptions, BatchErrorStage, BatchItemError, CpuBatchDecoder, + CpuBatchSamples, CpuExecution, EncodedImage, IndexedBatchError, PreparedBatch, +}; +use super::image::PLANNED_DECODE_BOUND; + +fn classic_rgb8(width: u32, height: u32) -> Arc<[u8]> { + let samples = (0..width * height * 3) + .map(|index| u8::try_from((index * 7 + index / 97) & 0xff).expect("masked sample")) + .collect::>(); + let options = j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + ..j2k_native::EncodeOptions::default() + }; + Arc::from( + j2k_native::encode(&samples, width, height, 3, 8, false, &options) + .expect("encode classic RGB8 fixture"), + ) +} + +/// One worker keeps preparation and decode on this thread, where the +/// planned-bound override applies. +fn single_worker_options() -> BatchDecodeOptions { + BatchDecodeOptions { + workers: NonZeroUsize::new(1), + ..BatchDecodeOptions::default() + } +} + +struct PlannedBound; + +impl PlannedBound { + fn hold(bytes: usize) -> Self { + PLANNED_DECODE_BOUND.with(|bound| bound.set(Some(bytes))); + Self + } +} + +impl Drop for PlannedBound { + fn drop(&mut self) { + PLANNED_DECODE_BOUND.with(|bound| bound.set(None)); + } +} + +fn planned_bytes(prepared: &PreparedBatch) -> Vec<(usize, CpuExecution)> { + let mut planned = prepared + .groups() + .iter() + .flat_map(|group| group.source_indices().iter().zip(group.images())) + .map(|(&index, image)| (index, image.cpu_execution())) + .collect::>(); + planned.sort_by_key(|(index, _)| *index); + planned +} + +fn u8_samples(groups: &[super::super::CpuBatchGroup], source_index: usize) -> Vec { + let group = groups + .iter() + .find(|group| group.source_indices() == [source_index]) + .expect("decoded group"); + let CpuBatchSamples::U8(samples) = group.samples() else { + panic!("RGB8 decodes to u8 samples"); + }; + samples.clone() +} + +fn is_decode_bound_error(error: &IndexedBatchError, index: usize, stage: BatchErrorStage) -> bool { + error.index == index + && matches!( + &error.source, + BatchItemError::Codec { stage: actual, source } + if *actual == stage && **source == CpuExecution::decode_bound_error() + ) +} + +#[test] +fn cpu_prepare_rejects_plans_over_the_decode_bound_before_decoding() { + let small = classic_rgb8(16, 16); + let large = classic_rgb8(96, 64); + let inputs = || { + vec![ + EncodedImage::full(Arc::clone(&small)), + EncodedImage::full(Arc::clone(&large)), + ] + }; + let mut decoder = CpuBatchDecoder::new(single_worker_options()); + let unbounded = decoder.prepare(inputs()).expect("prepare without a bound"); + let planned = planned_bytes(&unbounded); + let [(0, CpuExecution::OffsetPlan { bytes: small_bytes }), (1, CpuExecution::OffsetPlan { bytes: large_bytes })] = + planned[..] + else { + panic!("both images keep offset plans: {planned:?}"); + }; + assert!(small_bytes < large_bytes, "{small_bytes} < {large_bytes}"); + let reference = decoder + .decode_prepared(&unbounded) + .expect("decode without a bound"); + assert_eq!(reference.errors(), &[] as &[IndexedBatchError]); + + let _bound = PlannedBound::hold(small_bytes); + let rejected = decoder.prepare(inputs()).expect("prepare under the bound"); + assert_eq!(rejected.errors().len(), 1); + assert!(is_decode_bound_error( + &rejected.errors()[0], + 1, + BatchErrorStage::Prepare + )); + assert_eq!(planned_bytes(&rejected).len(), 1); + let decoded = decoder + .decode_prepared(&rejected) + .expect("decode the admitted image"); + assert_eq!( + u8_samples(decoded.groups(), 0), + u8_samples(reference.groups(), 0) + ); + + // Shared preparation keeps the image for device decoders; the CPU + // decoder still rejects it before decoding. + let shared = prepare_batch(inputs(), single_worker_options()).expect("shared prepare"); + assert_eq!(shared.errors(), &[] as &[IndexedBatchError]); + assert_eq!( + planned_bytes(&shared)[1], + (1, CpuExecution::ExceedsDecodeBound) + ); + let decoded = decoder + .decode_prepared(&shared) + .expect("decode the shared batch"); + assert_eq!(decoded.errors().len(), 1); + assert!(is_decode_bound_error( + &decoded.errors()[0], + 1, + BatchErrorStage::Decode + )); + assert_eq!( + u8_samples(decoded.groups(), 0), + u8_samples(reference.groups(), 0) + ); +} diff --git a/crates/j2k/src/owned_batch/prepare/image.rs b/crates/j2k/src/owned_batch/prepare/image.rs index 02aab69e6..3bb05b393 100644 --- a/crates/j2k/src/owned_batch/prepare/image.rs +++ b/crates/j2k/src/owned_batch/prepare/image.rs @@ -6,9 +6,9 @@ use super::super::{ extract_j2k_codestream_payload, parse_image_info, Arc, BatchAlpha, BatchCodecRoute, BatchColor, BatchDecodeOptions, BatchErrorStage, BatchExecutionShape, BatchGroupInfo, BatchItemError, BatchLayout, BatchWaveletTransform, BatchWorker, Colorspace, CompressedTransferSyntax, - DeviceDecodePlan, EncodedImage, J2kCodestreamRange, J2kError, J2kSupportInfo, NativeSampleType, - NonRepresentableReason, PreparationDepth, PrepareImageResult, PreparedClassicPlan, - PreparedCodecPlan, PreparedHtj2kPlan, PreparedImage, PreparedImageInner, + CpuExecution, DeviceDecodePlan, EncodedImage, J2kCodestreamRange, J2kError, J2kSupportInfo, + NativeSampleType, NonRepresentableReason, PreparationDepth, PrepareImageResult, + PreparedClassicPlan, PreparedCodecPlan, PreparedHtj2kPlan, PreparedImage, PreparedImageInner, }; pub(super) fn prepare_image( @@ -54,6 +54,7 @@ pub(super) fn prepare_image( .map_or(PreparedCodecPlan::MetadataOnly, PreparedCodecPlan::Htj2k), }; reconcile_codec_plan_metadata(&mut info, &codec_plan)?; + let cpu_execution = choose_cpu_execution(&codec_plan); let preparation_depth = codec_plan.preparation_depth(); let execution_shape = batch_execution_shape(&support, plan, preparation_depth); let image = PreparedImage { @@ -67,11 +68,62 @@ pub(super) fn prepare_image( plan, codestream_range, codec_plan, + cpu_execution, }), }; Ok((image, info, execution_shape)) } +/// Choose the CPU execution from the planned working memory, before any +/// decode work, with the same accounting the offset plan's execution +/// enforces. Only an image whose serial execution already exceeds the +/// per-decode bound is [`CpuExecution::ExceedsDecodeBound`], so the choice +/// rejects nothing execution could decode; the claim is the parallel peak. +pub(super) fn choose_cpu_execution(codec_plan: &PreparedCodecPlan) -> CpuExecution { + let planned = |parallel| match codec_plan { + PreparedCodecPlan::Classic(plan) => { + j2k_native::referenced_classic_plan_execution_bytes(plan.cpu_plan(), parallel) + } + PreparedCodecPlan::Htj2k(plan) => { + j2k_native::referenced_htj2k_plan_execution_bytes(plan.cpu_plan(), parallel) + } + PreparedCodecPlan::MetadataOnly => Ok(0), + }; + if matches!(codec_plan, PreparedCodecPlan::MetadataOnly) { + return CpuExecution::ComponentPlanes; + } + match (planned(false), planned(true)) { + ( + Err(j2k_native::DecodeError::Validation(j2k_native::ValidationError::ImageTooLarge)), + _, + ) => CpuExecution::ExceedsDecodeBound, + (Ok(serial), _) if serial > planned_decode_bound() => CpuExecution::ExceedsDecodeBound, + (Ok(_), Ok(bytes)) => CpuExecution::OffsetPlan { bytes }, + // A plan that fits only serially, or a malformed plan, keeps the + // worst-case claim and reports any error from execution, as before. + _ => CpuExecution::OffsetPlan { + bytes: crate::batch::allocation::GENERIC_WORKER_CLAIM_BYTES, + }, + } +} + +#[cfg(test)] +std::thread_local! { + /// Test-only lower bound for [`choose_cpu_execution`] on this thread. + pub(super) static PLANNED_DECODE_BOUND: core::cell::Cell> = + const { core::cell::Cell::new(None) }; +} + +/// The per-decode bound planned executions are held to. Native planning +/// already fails above the default, so this only lowers it in tests. +fn planned_decode_bound() -> usize { + #[cfg(test)] + if let Some(bound) = PLANNED_DECODE_BOUND.with(core::cell::Cell::get) { + return bound; + } + j2k_native::DEFAULT_MAX_DECODE_BYTES +} + pub(super) fn batch_execution_shape( support: &J2kSupportInfo, plan: DeviceDecodePlan, diff --git a/crates/j2k/src/owned_batch/prepared.rs b/crates/j2k/src/owned_batch/prepared.rs index 89391885c..cb97967f2 100644 --- a/crates/j2k/src/owned_batch/prepared.rs +++ b/crates/j2k/src/owned_batch/prepared.rs @@ -34,6 +34,36 @@ impl PreparedCodecPlan { } } +/// CPU execution chosen during preparation from the planned working memory. +/// Device decoders ignore it: the retained codec plan is unchanged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CpuExecution { + /// Execute the retained offset plan; `bytes` is its planned peak. + OffsetPlan { bytes: usize }, + /// Reparse and decode whole component planes; no offset plan exists. + ComponentPlanes, + /// The planned offset-plan execution exceeds the per-decode memory + /// bound, so the CPU decoder rejects the image before any decode work. + ExceedsDecodeBound, +} + +impl CpuExecution { + /// Admission claim for one execution: the planned peak of an offset + /// plan, or one worst-case native decode. + pub(super) const fn claim_bytes(self) -> usize { + match self { + Self::OffsetPlan { bytes } => bytes, + Self::ComponentPlanes => crate::batch::allocation::GENERIC_WORKER_CLAIM_BYTES, + Self::ExceedsDecodeBound => 0, + } + } + + /// The error the offset-plan execution would report, raised up front. + pub(super) fn decode_bound_error() -> crate::J2kError { + crate::J2kError::from_native_decode_error(j2k_native::ValidationError::ImageTooLarge.into()) + } +} + #[derive(Debug)] pub(super) struct PreparedImageInner { pub(super) bytes: Arc<[u8]>, @@ -45,6 +75,7 @@ pub(super) struct PreparedImageInner { pub(super) plan: DeviceDecodePlan, pub(super) codestream_range: J2kCodestreamRange, pub(super) codec_plan: PreparedCodecPlan, + pub(super) cpu_execution: CpuExecution, } impl PreparedImage { @@ -128,6 +159,10 @@ impl PreparedImage { pub(super) fn codec_plan(&self) -> &PreparedCodecPlan { &self.inner.codec_plan } + + pub(super) fn cpu_execution(&self) -> CpuExecution { + self.inner.cpu_execution + } } /// One homogeneous set of prepared images. diff --git a/xtask/api/stable-api-1.0.implementation-public-api.txt b/xtask/api/stable-api-1.0.implementation-public-api.txt index 319468601..303139432 100644 --- a/xtask/api/stable-api-1.0.implementation-public-api.txt +++ b/xtask/api/stable-api-1.0.implementation-public-api.txt @@ -2643,6 +2643,8 @@ pub fn j2k_native::prepare_referenced_htj2k_staged(&j2k_types::decode_plan::refe pub fn j2k_native::prepare_referenced_htj2k_tile_staged(&j2k_types::decode_plan::referenced::J2kReferencedHtj2kPlan, usize, &mut j2k_native::J2kDirectCpuScratch) -> j2k_native::Result<()> pub fn j2k_native::quantize_reversible_reference(&[f32], u16, u16, u8, bool) -> alloc::vec::Vec pub fn j2k_native::quantize_subband_reference(&[f32], u16, u16, u8, bool) -> alloc::vec::Vec +pub fn j2k_native::referenced_classic_plan_execution_bytes(&j2k_types::decode_plan::referenced::J2kReferencedClassicPlan, bool) -> j2k_native::Result +pub fn j2k_native::referenced_htj2k_plan_execution_bytes(&j2k_types::decode_plan::referenced::J2kReferencedHtj2kPlan, bool) -> j2k_native::Result pub fn j2k_native::try_deinterleave_reference(&[u8], usize, u16, u8, bool) -> j2k_native::Result>> pub fn j2k_native::window_referenced_classic_plan_to_stores(&j2k_types::decode_plan::referenced::J2kReferencedClassicPlan) -> j2k_native::Result> pub fn j2k_native::window_referenced_htj2k_plan_to_stores(&j2k_types::decode_plan::referenced::J2kReferencedHtj2kPlan) -> j2k_native::Result> From 5a58a307dd2da9207abba004e887329dd9cda355 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 21:24:43 -0400 Subject: [PATCH 51/53] fix(j2k): admit staged batch windows by their planned executions The staged CPU route keeps one whole-image scratch per window image until the window is materialized, eight images at a time, without any byte admission: eight 3584^2 RGB images held 4.3 GB of scratch. Each window now claims the sum of its images' planned execution peaks and ends early when the next image would push that sum past the batch execution cap, so in-flight staged scratch stays within the cap on hosts where it binds (four worst-case decodes on small hosts). Windows are not narrowed further: per-worker windows cut staged heap 75-80% but cost 10-33% decode time even with one worker. Unchanged on a 64 GB machine (cap 32 GiB). --- crates/j2k/src/batch/allocation.rs | 2 +- crates/j2k/src/owned_batch/cpu_execute.rs | 1 + .../j2k/src/owned_batch/cpu_staged_execute.rs | 70 +++++++++++++++++-- 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/crates/j2k/src/batch/allocation.rs b/crates/j2k/src/batch/allocation.rs index 13817929c..c5640e4fe 100644 --- a/crates/j2k/src/batch/allocation.rs +++ b/crates/j2k/src/batch/allocation.rs @@ -70,7 +70,7 @@ pub(super) fn admitted_batch_worker_limit() -> usize { /// /// A claim is conservative accounting for native ownership that cannot be /// estimated soundly before packet parsing. -pub(super) fn batch_execution_cap_bytes() -> usize { +pub(crate) fn batch_execution_cap_bytes() -> usize { // Cannot saturate: the worker limit is at most MAX_SCALED_GENERIC_BATCH_WORKERS. generic_batch_worker_limit().saturating_mul(GENERIC_WORKER_CLAIM_BYTES) } diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index a87a14af0..e16baa0a3 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -215,6 +215,7 @@ fn run_typed_group( &mut results, flattened, staged_workspace, + admission, convert, )?; } else { diff --git a/crates/j2k/src/owned_batch/cpu_staged_execute.rs b/crates/j2k/src/owned_batch/cpu_staged_execute.rs index 67be25677..a280aaab3 100644 --- a/crates/j2k/src/owned_batch/cpu_staged_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_staged_execute.rs @@ -10,6 +10,9 @@ use super::{ CpuGroupFastWorkspace, CpuStagedWorkspace, J2kError, Ordering, PreparedBatchGroup, Range, TileBatchOptions, CPU_ENTROPY_IMAGES_PER_WINDOW, }; +use crate::batch::admission::BatchAllocationBudget; +use crate::batch::allocation::batch_execution_cap_bytes; +use alloc::sync::Arc; struct StagedWindowContext<'a> { group: &'a PreparedBatchGroup, @@ -30,15 +33,27 @@ pub(super) fn run_staged_typed_group( image_results: &mut [Option>], flattened: &mut CpuGroupFastWorkspace, staged: &mut CpuStagedWorkspace, + admission: &Arc, convert: fn(f32, u8) -> T, ) -> Result<(), BatchInfrastructureError> { let route = flattened .route() .ok_or(BatchInfrastructureError::MissingResult { index: 0 })?; - for window_start in (0..image_jobs.len()).step_by(CPU_ENTROPY_IMAGES_PER_WINDOW) { - let window_end = window_start - .saturating_add(CPU_ENTROPY_IMAGES_PER_WINDOW) - .min(image_jobs.len()); + // Every image of a window keeps its own whole-image scratch until the + // window is materialized, so the window claims the sum of its images' + // planned executions. Windows stay as wide as the admission cap allows: + // narrower windows cut throughput even with one worker. + let window_budget = batch_execution_cap_bytes(); + let mut window_start = 0; + while window_start < image_jobs.len() { + let (window_end, window_bytes) = staged_window_len( + image_jobs[window_start..] + .iter() + .map(|job| job.image.cpu_execution().claim_bytes()), + window_budget, + ); + let window_end = window_start + window_end; + let _window_claim = admission.claim(window_bytes)?; let window = StagedWindowContext { group, route, @@ -66,10 +81,29 @@ pub(super) fn run_staged_typed_group( image_results, )?; materialize_staged_window(workers, staged, &window, image_jobs, image_results, convert)?; + window_start = window_end; } Ok(()) } +/// Images in the next staged window (at least one, at most +/// [`CPU_ENTROPY_IMAGES_PER_WINDOW`]) whose planned execution claims together +/// fit `budget`, and their summed claim, never above `budget`: one image's +/// claim is at most one worst-case decode, which the cap always admits. +fn staged_window_len(claims: impl IntoIterator, budget: usize) -> (usize, usize) { + let mut count = 0; + let mut bytes = 0usize; + for claim in claims.into_iter().take(CPU_ENTROPY_IMAGES_PER_WINDOW) { + let next = bytes.saturating_add(claim); + if count > 0 && next > budget { + break; + } + count += 1; + bytes = next; + } + (count, bytes.min(budget)) +} + fn prepare_staged_window( workers: &mut [BatchWorker], staged: &mut CpuStagedWorkspace, @@ -471,3 +505,31 @@ fn finish_staged_tile_window( }, ) } + +#[cfg(test)] +mod tests { + use super::{staged_window_len, CPU_ENTROPY_IMAGES_PER_WINDOW}; + + #[test] + fn staged_windows_fit_their_summed_plans_in_the_cap() { + const MIB: usize = 1024 * 1024; + // The smallest cap (four worst-case decodes) holds six images + // planned at 330 MiB. + assert_eq!( + staged_window_len([330 * MIB; 8], 4 * 512 * MIB), + (6, 6 * 330 * MIB) + ); + // A larger cap keeps full windows. + assert_eq!( + staged_window_len([330 * MIB; 8], 32 * 512 * MIB), + (CPU_ENTROPY_IMAGES_PER_WINDOW, 8 * 330 * MIB) + ); + assert_eq!( + staged_window_len([3 * MIB; 64], 4 * 512 * MIB), + (CPU_ENTROPY_IMAGES_PER_WINDOW, 8 * 3 * MIB) + ); + // A window always makes progress, and its claim stays in budget. + assert_eq!(staged_window_len([600 * MIB], 512 * MIB), (1, 512 * MIB)); + assert_eq!(staged_window_len([], 512 * MIB), (0, 0)); + } +} From 0ea0007a0e93fa21319bf0c4a9d0b0c0cca2d693 Mon Sep 17 00:00:00 2001 From: GF Date: Fri, 30 Oct 2026 21:46:29 -0400 Subject: [PATCH 52/53] fix(j2k-native): keep no parallel classic workspaces for fragmented payloads Parallel classic dispatch borrows every payload as one input slice and falls back to serial decode when a code block's bytes are split over fragments, as in multi-layer codestreams. Such plans still reserved the extra classic workspaces, which then sat unused: IIIF 512 tiles at 1/2 of a 5661x8287 8-layer scan grew peak heap 2.7%. Plans with a fragmented payload now plan and reserve no extra classic workspaces and no dispatch bytes; their wavelet steps stay parallel. Peak heap of that batch back to where it was before the parallel plans (399.7 MB), bytes unchanged. --- .../src/direct_cpu/allocation/referenced.rs | 12 +++++-- .../direct_cpu/allocation/referenced/view.rs | 14 +++++++- .../direct_cpu/referenced_classic/tests.rs | 32 +++++++++++++++++++ crates/j2k/src/owned_batch/cpu_execute.rs | 3 +- .../prepare/cpu_preflight_tests.rs | 12 +++---- 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs index c3f88c9ea..491a767ca 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced.rs @@ -15,8 +15,9 @@ use super::{ use budget::validate_referenced_aggregate_plan; use storage::{normalize_referenced_scratch, reserve_referenced_scratch}; use view::{ - max_referenced_classic_payload_bytes, max_referenced_classic_rows, max_referenced_dimensions, - max_referenced_payload_bytes, ReferencedPlanView, + classic_plan_has_fragmented_payloads, max_referenced_classic_payload_bytes, + max_referenced_classic_rows, max_referenced_dimensions, max_referenced_payload_bytes, + ReferencedPlanView, }; pub(in super::super) fn prepare_referenced_direct_scratch( @@ -81,8 +82,13 @@ pub(in super::super) fn planned_referenced_classic_execution_bytes( .map(DirectWorkspaceBudget::peak_bytes) } -/// Shape of the extra classic workspaces parallel execution of `plan` uses. +/// Shape of the extra classic workspaces parallel execution of `plan` uses: +/// none for plans whose payloads are split over fragments, which parallel +/// dispatch cannot borrow as one slice. fn classic_task_shape(plan: &J2kReferencedClassicPlan) -> Option { + if classic_plan_has_fragmented_payloads(plan) { + return None; + } Some(ClassicTaskShape { dimensions: max_referenced_classic_dimensions(plan)?, rows: max_referenced_classic_rows(plan), diff --git a/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs b/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs index 7846d3989..73e1425a2 100644 --- a/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs +++ b/crates/j2k-native/src/direct_cpu/allocation/referenced/view.rs @@ -221,7 +221,10 @@ fn referenced_parallel_classic_dispatch_bytes( parallel: bool, ) -> Result { let worker_count = rayon::current_num_threads(); - if !parallel || worker_count <= 1 || !matches!(plan, ReferencedPlanView::Classic(_)) { + let ReferencedPlanView::Classic(classic) = plan else { + return Ok(0); + }; + if !parallel || worker_count <= 1 || classic_plan_has_fragmented_payloads(classic) { return Ok(0); } let mut maximum_jobs = 0usize; @@ -330,6 +333,15 @@ pub(in super::super::super) fn max_referenced_classic_dimensions( max_referenced_dimensions(ReferencedPlanView::Classic(plan), false) } +/// Whether some classic payload is split over several input fragments (a +/// multi-layer codestream). Parallel classic dispatch borrows each payload +/// as one input slice, so such plans decode their blocks serially. +pub(super) fn classic_plan_has_fragmented_payloads(plan: &J2kReferencedClassicPlan) -> bool { + plan.payloads() + .iter() + .any(|payload| payload.range_count > 1) +} + /// Most runs of code-block rows (consecutive jobs sharing `output_y`) in any /// classic sub-band. Runs are never fewer than the distinct rows one /// parallel dispatch decodes at once; an unreadable plan is unbounded. diff --git a/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs b/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs index fd0839217..823825f70 100644 --- a/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs +++ b/crates/j2k-native/src/direct_cpu/referenced_classic/tests.rs @@ -8,6 +8,10 @@ use crate::{ }; fn rgb_fixture(width: u32, height: u32, reversible: bool) -> Vec { + rgb_fixture_with_layers(width, height, reversible, 1) +} + +fn rgb_fixture_with_layers(width: u32, height: u32, reversible: bool, num_layers: u8) -> Vec { let samples = (0..width * height * 3) .map(|index| { let (pixel, channel) = (index / 3, index % 3); @@ -19,6 +23,7 @@ fn rgb_fixture(width: u32, height: u32, reversible: bool) -> Vec { let options = EncodeOptions { reversible, num_decomposition_levels: 4, + num_layers, ..EncodeOptions::default() }; encode(&samples, width, height, 3, 8, false, &options).expect("encode classic RGB fixture") @@ -75,6 +80,33 @@ fn parallel_referenced_classic_execution_matches_serial_bits() { } } +#[cfg(feature = "parallel")] +#[test] +fn fragmented_payloads_decode_serially_without_extra_workspaces() { + let bytes = rgb_fixture_with_layers(300, 260, false, 4); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("classic image"); + let plan = image + .build_referenced_classic_plan_region_with_context( + &mut DecoderContext::default(), + (0, 0, image.width(), image.height()), + ) + .expect("classic whole-image plan"); + assert!( + plan.payloads() + .iter() + .any(|payload| payload.range_count > 1), + "four quality layers split code-block payloads over fragments" + ); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .expect("four-thread pool"); + let (serial, serial_workspace_bytes) = execute_whole_plan(&bytes, false); + let (parallel, parallel_workspace_bytes) = pool.install(|| execute_whole_plan(&bytes, true)); + assert_eq!(serial, parallel); + assert_eq!(parallel_workspace_bytes, serial_workspace_bytes); +} + #[test] fn parallel_request_on_the_global_pool_matches_serial_bits() { let bytes = rgb_fixture(200, 136, false); diff --git a/crates/j2k/src/owned_batch/cpu_execute.rs b/crates/j2k/src/owned_batch/cpu_execute.rs index e16baa0a3..9674a0b1d 100644 --- a/crates/j2k/src/owned_batch/cpu_execute.rs +++ b/crates/j2k/src/owned_batch/cpu_execute.rs @@ -175,7 +175,8 @@ type CpuDecodeFn = for<'image> fn( #[expect( clippy::too_many_arguments, - reason = "the retained-worker boundary keeps output ownership, decode policy, and indexed error collection explicit" + clippy::too_many_lines, + reason = "the retained-worker boundary keeps output ownership, decode policy, and indexed error collection explicit, with the job slots, the worker loop, and the result collection in one place" )] fn run_typed_group( workers: &mut [BatchWorker], diff --git a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs index 82fb982d9..0395bb08d 100644 --- a/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs +++ b/crates/j2k/src/owned_batch/prepare/cpu_preflight_tests.rs @@ -112,11 +112,11 @@ fn cpu_prepare_rejects_plans_over_the_decode_bound_before_decoding() { BatchErrorStage::Prepare )); assert_eq!(planned_bytes(&rejected).len(), 1); - let decoded = decoder + let output = decoder .decode_prepared(&rejected) .expect("decode the admitted image"); assert_eq!( - u8_samples(decoded.groups(), 0), + u8_samples(output.groups(), 0), u8_samples(reference.groups(), 0) ); @@ -128,17 +128,17 @@ fn cpu_prepare_rejects_plans_over_the_decode_bound_before_decoding() { planned_bytes(&shared)[1], (1, CpuExecution::ExceedsDecodeBound) ); - let decoded = decoder + let output = decoder .decode_prepared(&shared) .expect("decode the shared batch"); - assert_eq!(decoded.errors().len(), 1); + assert_eq!(output.errors().len(), 1); assert!(is_decode_bound_error( - &decoded.errors()[0], + &output.errors()[0], 1, BatchErrorStage::Decode )); assert_eq!( - u8_samples(decoded.groups(), 0), + u8_samples(output.groups(), 0), u8_samples(reference.groups(), 0) ); } From a46ae11f6847e77337024db297a0d7a074b68580 Mon Sep 17 00:00:00 2001 From: GF Date: Sun, 1 Nov 2026 15:11:01 -0500 Subject: [PATCH 53/53] chore(api): stage 0.13.2 API evidence and preserve error discriminants --- CHANGELOG.md | 34 +- Cargo.lock | 60 ++-- Cargo.toml | 2 +- crates/j2k-cli/Cargo.toml | 6 +- crates/j2k-codec-math/Cargo.toml | 2 +- crates/j2k-compare/Cargo.toml | 8 +- crates/j2k-cuda-j2k-engine/Cargo.toml | 10 +- crates/j2k-cuda-jpeg-engine/Cargo.toml | 10 +- crates/j2k-cuda-runtime/Cargo.toml | 6 +- crates/j2k-cuda-transcode-engine/Cargo.toml | 6 +- crates/j2k-cuda/Cargo.toml | 12 +- crates/j2k-jpeg-cuda/Cargo.toml | 10 +- crates/j2k-jpeg-metal/Cargo.toml | 8 +- crates/j2k-jpeg/Cargo.toml | 6 +- crates/j2k-jpeg/fuzz/Cargo.lock | 10 +- crates/j2k-metal-support/Cargo.toml | 2 +- crates/j2k-metal/Cargo.toml | 14 +- crates/j2k-ml/Cargo.toml | 16 +- crates/j2k-mpsgraph/Cargo.toml | 12 +- crates/j2k-native/Cargo.toml | 6 +- crates/j2k-native/fuzz/Cargo.lock | 8 +- crates/j2k-native/src/error.rs | 4 +- crates/j2k-t803/Cargo.toml | 14 +- crates/j2k-t803/fuzz/Cargo.lock | 16 +- crates/j2k-test-support/Cargo.toml | 4 +- crates/j2k-tilecodec/Cargo.toml | 2 +- crates/j2k-tilecodec/fuzz/Cargo.lock | 4 +- crates/j2k-transcode-cuda/Cargo.toml | 14 +- crates/j2k-transcode-metal/Cargo.toml | 18 +- crates/j2k-transcode-test-support/Cargo.toml | 6 +- crates/j2k-transcode/Cargo.toml | 12 +- crates/j2k-transcode/fuzz/Cargo.lock | 16 +- crates/j2k/Cargo.toml | 8 +- crates/j2k/fuzz/Cargo.lock | 14 +- crates/j2k/src/error.rs | 32 +- crates/j2k/src/owned_batch/cpu.rs | 7 + docs/stable-api-1.0.md | 9 +- xtask/RELEASING.md | 7 +- .../public-api/public-api-review-0.13.2.yml | 174 +++++++++ .../reviewed-public-api-diff-0.13.2.md | 336 ++++++++++++++++++ xtask/src/semver.rs | 10 +- xtask/src/semver/tests.rs | 6 +- xtask/src/semver/tests/api_planning.rs | 2 +- xtask/src/semver/tests/command_boundaries.rs | 6 +- xtask/tests/command_orchestration.rs | 4 +- xtask/tests/command_orchestration/support.rs | 2 +- 46 files changed, 760 insertions(+), 215 deletions(-) create mode 100644 xtask/release-evidence/public-api/public-api-review-0.13.2.yml create mode 100644 xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.2.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 647c51c75..9bc7efd72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,17 +5,19 @@ and stale roadmap entries have been removed from the public documentation set. ## [Unreleased] +Staged workspace version: `0.13.2`. + - Classic code-blocks with BYPASS, per-pass termination, context resets or segmentation symbols use the packed-column passes, including raw segments. Vertically causal blocks retain the per-coefficient passes. Clean and corrupt code-blocks preserve the same samples and errors in strict and lenient modes. -- Decodes into a caller buffer whose stride is wider than a row and which - ends right after the last row's pixels, the length `strided_output_len` - reports and buffer validation accepts, now write that last row. Full-image +- Decodes into a validated caller buffer whose stride is wider than a row + and which ends before a full stride after the last row now write that + last row, including buffers of exactly `strided_output_len` bytes. Full-image and scaled 8-bit decodes at such a stride and every 16-bit decode at one, regions included, left it unwritten and still returned `Ok`. 8-bit region - decodes, and buffers with any bytes after the last row, were not affected. - The other rows are unchanged. + decodes, and buffers padded to a full stride after the last row, were not + affected. The other rows are unchanged. - Decodes on the worker pool that reconstruct the two finest wavelet levels of a component in one pass now charge the decode budget for what that pass holds: each worker's rows of the coarser level, which were not charged, @@ -28,14 +30,28 @@ and stale roadmap entries have been removed from the public documentation set. `J2kError::ResourceLimit { requested, cap, source }` instead of the generic `J2kError::NativeDecode`, on every decode route: `J2kDecoder`, the tile batch functions (inside `TileBatchError::Tile`) and the owned batch API - (inside `BatchItemError::Codec`). `requested` and `cap` are the live bytes - and the cap when the native decoder knows them (`AllocationTooLarge`), and + (inside `BatchItemError::Codec`), and JP2/JPH metadata inspection. + `requested` and `cap` are the live bytes and the cap when the native decoder + knows them (`AllocationTooLarge`), and `None` for `ImageTooLarge`. Callers can tell these apart from invalid input and unsupported features, and retry with a smaller request such as a region - or a reduced resolution. The display text and the source chain keep the - native error. `j2k_native::DecodeError::classify` reports the same class as + or a reduced resolution. The display prefix becomes `decode resource limit + exceeded:`; the display text and source chain keep the native error. + `j2k_native::DecodeError::classify` reports the same class as `DecodeErrorClass::ResourceLimit`. Both enums are `#[non_exhaustive]`; host allocator failures keep their previous classification. +- Prepared classic CPU batch plans decode in parallel. Batches prepared by + `CpuBatchDecoder` reject individual outputs or planned executions that + exceed their memory limits during preparation, with the input's index, + and exclude those items from the prepared groups. Backend-neutral + `prepare_batch` and `prepare_batch_from_images` keep their existing + behavior: decoding them on the CPU can still fail the whole call when one + image's output exceeds the allocation cap. On either preparation path, a + group whose combined output exceeds that cap still fails the whole decode + call. + Staged windows are admitted by planned execution memory, and an admission + claim that cannot fit beside the shared plan fails instead of waiting + indefinitely. Successful output is unchanged. - x86-64 RGB8 decodes of 9/7 images round and pack the inverse ICT output with the runtime SIMD level (AVX2 where present) instead of a scalar library call per sample: the centered rounding of the inverse MCT runs eight diff --git a/Cargo.lock b/Cargo.lock index 347c680ec..f03e74a51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2496,7 +2496,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "j2k" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "j2k-core", @@ -2511,7 +2511,7 @@ dependencies = [ [[package]] name = "j2k-alloc-probe" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-native", "j2k-profile", @@ -2521,7 +2521,7 @@ dependencies = [ [[package]] name = "j2k-cli" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k", "j2k-jpeg", @@ -2531,7 +2531,7 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-types", ] @@ -2551,7 +2551,7 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.13.1" +version = "0.13.2" dependencies = [ "libc", "thiserror 2.0.18", @@ -2560,7 +2560,7 @@ dependencies = [ [[package]] name = "j2k-cuda" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "j2k", @@ -2577,11 +2577,11 @@ dependencies = [ [[package]] name = "j2k-cuda-build-support" -version = "0.13.1" +version = "0.13.2" [[package]] name = "j2k-cuda-j2k-engine" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "j2k-codec-math", @@ -2595,7 +2595,7 @@ dependencies = [ [[package]] name = "j2k-cuda-jpeg-engine" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-codec-math", "j2k-core", @@ -2606,7 +2606,7 @@ dependencies = [ [[package]] name = "j2k-cuda-runtime" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-codec-math", "j2k-core", @@ -2618,7 +2618,7 @@ dependencies = [ [[package]] name = "j2k-cuda-transcode-engine" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-core", "j2k-cuda-build-support", @@ -2628,7 +2628,7 @@ dependencies = [ [[package]] name = "j2k-jpeg" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "fearless_simd", @@ -2647,7 +2647,7 @@ dependencies = [ [[package]] name = "j2k-jpeg-cuda" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "j2k-core", @@ -2663,7 +2663,7 @@ dependencies = [ [[package]] name = "j2k-jpeg-metal" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "dispatch2", @@ -2682,7 +2682,7 @@ dependencies = [ [[package]] name = "j2k-metal" -version = "0.13.1" +version = "0.13.2" dependencies = [ "cc", "criterion", @@ -2703,7 +2703,7 @@ dependencies = [ [[package]] name = "j2k-metal-support" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-core", "j2k-test-support", @@ -2715,7 +2715,7 @@ dependencies = [ [[package]] name = "j2k-ml" -version = "0.13.1" +version = "0.13.2" dependencies = [ "burn-core", "burn-cubecl", @@ -2740,7 +2740,7 @@ dependencies = [ [[package]] name = "j2k-mpsgraph" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k", "j2k-core", @@ -2759,7 +2759,7 @@ dependencies = [ [[package]] name = "j2k-mpsgraph-support" -version = "0.13.1" +version = "0.13.2" dependencies = [ "block2", "objc2", @@ -2770,7 +2770,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "fearless_simd", @@ -2784,11 +2784,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.13.1" +version = "0.13.2" [[package]] name = "j2k-python" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k", "j2k-cuda", @@ -2805,7 +2805,7 @@ dependencies = [ [[package]] name = "j2k-t803" -version = "0.13.1" +version = "0.13.2" dependencies = [ "image", "j2k", @@ -2828,7 +2828,7 @@ dependencies = [ [[package]] name = "j2k-test-support" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-core", "j2k-native", @@ -2839,7 +2839,7 @@ dependencies = [ [[package]] name = "j2k-tilecodec" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "flate2", @@ -2851,7 +2851,7 @@ dependencies = [ [[package]] name = "j2k-transcode" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "j2k", @@ -2868,7 +2868,7 @@ dependencies = [ [[package]] name = "j2k-transcode-cuda" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "j2k-core", @@ -2885,7 +2885,7 @@ dependencies = [ [[package]] name = "j2k-transcode-metal" -version = "0.13.1" +version = "0.13.2" dependencies = [ "criterion", "j2k-codec-math", @@ -2907,7 +2907,7 @@ dependencies = [ [[package]] name = "j2k-transcode-test-support" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-native", "j2k-transcode", @@ -2916,7 +2916,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.13.1" +version = "0.13.2" [[package]] name = "jni-sys" diff --git a/Cargo.toml b/Cargo.toml index 0f9f33749..61574394d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ members = [ "xtask", ] [workspace.package] -version = "0.13.1" +version = "0.13.2" edition = "2021" rust-version = "1.99.0" license = "MIT OR Apache-2.0" diff --git a/crates/j2k-cli/Cargo.toml b/crates/j2k-cli/Cargo.toml index d419ddd9a..254c90fe8 100644 --- a/crates/j2k-cli/Cargo.toml +++ b/crates/j2k-cli/Cargo.toml @@ -18,9 +18,9 @@ path = "src/main.rs" doc = false [dependencies] -j2k = { path = "../j2k", version = "=0.13.1" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.1" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.13.1" } +j2k = { path = "../j2k", version = "=0.13.2" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.2" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.13.2" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-codec-math/Cargo.toml b/crates/j2k-codec-math/Cargo.toml index e8b0cb397..473b39d14 100644 --- a/crates/j2k-codec-math/Cargo.toml +++ b/crates/j2k-codec-math/Cargo.toml @@ -21,7 +21,7 @@ name = "j2k_codec_math" path = "src/lib.rs" [dependencies] -j2k-types = { path = "../j2k-types", version = "=0.13.1" } +j2k-types = { path = "../j2k-types", version = "=0.13.2" } [lints] workspace = true diff --git a/crates/j2k-compare/Cargo.toml b/crates/j2k-compare/Cargo.toml index 084eecbd3..5574cc4fc 100644 --- a/crates/j2k-compare/Cargo.toml +++ b/crates/j2k-compare/Cargo.toml @@ -13,10 +13,10 @@ publish = false ignored = ["cc"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-native = { path = "../j2k-native", version = "=0.13.1" } -j2k = { path = "../j2k", version = "=0.13.1" } -j2k-test-support = { path = "../j2k-test-support", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } +j2k = { path = "../j2k", version = "=0.13.2" } +j2k-test-support = { path = "../j2k-test-support", version = "=0.13.2" } image = { workspace = true } openjpeg-sys = { workspace = true } diff --git a/crates/j2k-cuda-j2k-engine/Cargo.toml b/crates/j2k-cuda-j2k-engine/Cargo.toml index 48c0a4c22..96dad60b0 100644 --- a/crates/j2k-cuda-j2k-engine/Cargo.toml +++ b/crates/j2k-cuda-j2k-engine/Cargo.toml @@ -33,13 +33,13 @@ cuda-oxide-htj2k-decode = [] cuda-oxide-htj2k-encode = [] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1" } -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.1" } -j2k-types = { path = "../j2k-types", version = "=0.13.1" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.2" } +j2k-types = { path = "../j2k-types", version = "=0.13.2" } [build-dependencies] -j2k-cuda-build-support = { path = "../j2k-cuda-build-support", version = "=0.13.1" } +j2k-cuda-build-support = { path = "../j2k-cuda-build-support", version = "=0.13.2" } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/j2k-cuda-jpeg-engine/Cargo.toml b/crates/j2k-cuda-jpeg-engine/Cargo.toml index c5705c3c0..907d495e6 100644 --- a/crates/j2k-cuda-jpeg-engine/Cargo.toml +++ b/crates/j2k-cuda-jpeg-engine/Cargo.toml @@ -29,13 +29,13 @@ cuda-oxide-jpeg-encode = [] cuda-oxide-jpeg-lossless-encode = [] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1" } -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.1" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.2" } [build-dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1" } -j2k-cuda-build-support = { path = "../j2k-cuda-build-support", version = "=0.13.1" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2" } +j2k-cuda-build-support = { path = "../j2k-cuda-build-support", version = "=0.13.2" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-cuda-runtime/Cargo.toml b/crates/j2k-cuda-runtime/Cargo.toml index 4bd6a77e1..29728908a 100644 --- a/crates/j2k-cuda-runtime/Cargo.toml +++ b/crates/j2k-cuda-runtime/Cargo.toml @@ -30,13 +30,13 @@ cuda-oxide = [ cuda-oxide-copy-u8 = [] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } libloading = { workspace = true } thiserror = { workspace = true } [build-dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1" } -j2k-cuda-build-support = { path = "../j2k-cuda-build-support", version = "=0.13.1" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2" } +j2k-cuda-build-support = { path = "../j2k-cuda-build-support", version = "=0.13.2" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-cuda-transcode-engine/Cargo.toml b/crates/j2k-cuda-transcode-engine/Cargo.toml index c286d254d..a637151c6 100644 --- a/crates/j2k-cuda-transcode-engine/Cargo.toml +++ b/crates/j2k-cuda-transcode-engine/Cargo.toml @@ -27,11 +27,11 @@ cuda-profiling = ["j2k-cuda-runtime/cuda-profiling"] cuda-oxide-transcode = [] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.2" } [build-dependencies] -j2k-cuda-build-support = { path = "../j2k-cuda-build-support", version = "=0.13.1" } +j2k-cuda-build-support = { path = "../j2k-cuda-build-support", version = "=0.13.2" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-cuda/Cargo.toml b/crates/j2k-cuda/Cargo.toml index 4c14627c2..a524fa23b 100644 --- a/crates/j2k-cuda/Cargo.toml +++ b/crates/j2k-cuda/Cargo.toml @@ -36,12 +36,12 @@ cuda-runtime = [ cuda-profiling = ["cuda-runtime", "j2k-cuda-j2k-engine/cuda-profiling"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-cuda-j2k-engine = { path = "../j2k-cuda-j2k-engine", version = "=0.13.1", optional = true } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.1", optional = true } -j2k = { path = "../j2k", version = "=0.13.1" } -j2k-native = { path = "../j2k-native", version = "=0.13.1" } -j2k-profile = { path = "../j2k-profile", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-cuda-j2k-engine = { path = "../j2k-cuda-j2k-engine", version = "=0.13.2", optional = true } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.2", optional = true } +j2k = { path = "../j2k", version = "=0.13.2" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } +j2k-profile = { path = "../j2k-profile", version = "=0.13.2" } rayon = { workspace = true, optional = true } thiserror = { workspace = true } diff --git a/crates/j2k-jpeg-cuda/Cargo.toml b/crates/j2k-jpeg-cuda/Cargo.toml index 1609adcbd..6447c77e4 100644 --- a/crates/j2k-jpeg-cuda/Cargo.toml +++ b/crates/j2k-jpeg-cuda/Cargo.toml @@ -29,11 +29,11 @@ cuda-runtime = [ ] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-cuda-jpeg-engine = { path = "../j2k-cuda-jpeg-engine", version = "=0.13.1", optional = true } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.1", optional = true } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.1" } -j2k-profile = { path = "../j2k-profile", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-cuda-jpeg-engine = { path = "../j2k-cuda-jpeg-engine", version = "=0.13.2", optional = true } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.2", optional = true } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.2" } +j2k-profile = { path = "../j2k-profile", version = "=0.13.2" } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/j2k-jpeg-metal/Cargo.toml b/crates/j2k-jpeg-metal/Cargo.toml index 6b8af8753..f892a470a 100644 --- a/crates/j2k-jpeg-metal/Cargo.toml +++ b/crates/j2k-jpeg-metal/Cargo.toml @@ -19,10 +19,10 @@ name = "j2k_jpeg_metal" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.1" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.1" } -j2k-profile = { path = "../j2k-profile", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.2" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.2" } +j2k-profile = { path = "../j2k-profile", version = "=0.13.2" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-jpeg/Cargo.toml b/crates/j2k-jpeg/Cargo.toml index 0b28a0c0e..67c20a8e5 100644 --- a/crates/j2k-jpeg/Cargo.toml +++ b/crates/j2k-jpeg/Cargo.toml @@ -28,9 +28,9 @@ bench-libjpeg-turbo = [] bench-internals = [] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1" } -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-profile = { path = "../j2k-profile", version = "=0.13.1" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-profile = { path = "../j2k-profile", version = "=0.13.2" } thiserror = { workspace = true } memchr = { workspace = true } rayon = { workspace = true } diff --git a/crates/j2k-jpeg/fuzz/Cargo.lock b/crates/j2k-jpeg/fuzz/Cargo.lock index 6619399e5..42142eb80 100644 --- a/crates/j2k-jpeg/fuzz/Cargo.lock +++ b/crates/j2k-jpeg/fuzz/Cargo.lock @@ -83,14 +83,14 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-types", ] [[package]] name = "j2k-core" -version = "0.13.1" +version = "0.13.2" dependencies = [ "libc", "thiserror", @@ -99,7 +99,7 @@ dependencies = [ [[package]] name = "j2k-jpeg" -version = "0.13.1" +version = "0.13.2" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -120,11 +120,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.13.1" +version = "0.13.2" [[package]] name = "j2k-types" -version = "0.13.1" +version = "0.13.2" [[package]] name = "jobserver" diff --git a/crates/j2k-metal-support/Cargo.toml b/crates/j2k-metal-support/Cargo.toml index 81572d9e7..ff3df7a6b 100644 --- a/crates/j2k-metal-support/Cargo.toml +++ b/crates/j2k-metal-support/Cargo.toml @@ -19,7 +19,7 @@ name = "j2k_metal_support" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } log = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-metal/Cargo.toml b/crates/j2k-metal/Cargo.toml index e8e6c5c77..8d1d120ad 100644 --- a/crates/j2k-metal/Cargo.toml +++ b/crates/j2k-metal/Cargo.toml @@ -24,13 +24,13 @@ name = "j2k_metal" path = "src/lib.rs" [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1" } -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k = { path = "../j2k", version = "=0.13.1" } -j2k-native = { path = "../j2k-native", version = "=0.13.1" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.1" } -j2k-profile = { path = "../j2k-profile", version = "=0.13.1" } -j2k-types = { path = "../j2k-types", version = "=0.13.1" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k = { path = "../j2k", version = "=0.13.2" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.2" } +j2k-profile = { path = "../j2k-profile", version = "=0.13.2" } +j2k-types = { path = "../j2k-types", version = "=0.13.2" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-ml/Cargo.toml b/crates/j2k-ml/Cargo.toml index 9f50bacec..170d59fe9 100644 --- a/crates/j2k-ml/Cargo.toml +++ b/crates/j2k-ml/Cargo.toml @@ -37,24 +37,24 @@ metal = [ ] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } burn-core = { workspace = true } burn-cubecl = { workspace = true, optional = true } burn-cuda = { workspace = true, optional = true } burn-fusion = { workspace = true, optional = true } burn-ir = { workspace = true, optional = true } burn-wgpu = { workspace = true, optional = true } -j2k = { path = "../j2k", version = "=0.13.1" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.1", optional = true } -j2k-cuda = { path = "../j2k-cuda", version = "=0.13.1", features = ["cuda-runtime"], optional = true } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.1", optional = true } -j2k-metal = { path = "../j2k-metal", version = "=0.13.1", optional = true } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.1", optional = true } +j2k = { path = "../j2k", version = "=0.13.2" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.2", optional = true } +j2k-cuda = { path = "../j2k-cuda", version = "=0.13.2", features = ["cuda-runtime"], optional = true } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.2", optional = true } +j2k-metal = { path = "../j2k-metal", version = "=0.13.2", optional = true } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.2", optional = true } thiserror = { workspace = true } [dev-dependencies] criterion = { workspace = true } -j2k-native = { path = "../j2k-native", version = "=0.13.1" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [target.'cfg(all(target_arch = "aarch64", target_os = "linux"))'.dev-dependencies] diff --git a/crates/j2k-mpsgraph/Cargo.toml b/crates/j2k-mpsgraph/Cargo.toml index 6457d36a6..86538fec2 100644 --- a/crates/j2k-mpsgraph/Cargo.toml +++ b/crates/j2k-mpsgraph/Cargo.toml @@ -17,14 +17,14 @@ all-features = true targets = [] [dependencies] -j2k = { path = "../j2k", version = "=0.13.1" } -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-metal = { path = "../j2k-metal", version = "=0.13.1" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.1" } +j2k = { path = "../j2k", version = "=0.13.2" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-metal = { path = "../j2k-metal", version = "=0.13.2" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.2" } thiserror = { workspace = true } [target.'cfg(all(target_arch = "aarch64", target_os = "macos"))'.dependencies] -j2k-mpsgraph-support = { path = "../j2k-mpsgraph-support", version = "=0.13.1" } +j2k-mpsgraph-support = { path = "../j2k-mpsgraph-support", version = "=0.13.2" } objc2 = { workspace = true } objc2-foundation = { workspace = true, features = [ "NSArray", @@ -37,7 +37,7 @@ objc2-metal-performance-shaders = { workspace = true } objc2-metal-performance-shaders-graph = { workspace = true } [dev-dependencies] -j2k-native = { path = "../j2k-native", version = "=0.13.1" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [lints.rust] diff --git a/crates/j2k-native/Cargo.toml b/crates/j2k-native/Cargo.toml index 8bd6cc63e..89bc3796c 100644 --- a/crates/j2k-native/Cargo.toml +++ b/crates/j2k-native/Cargo.toml @@ -30,9 +30,9 @@ fearless_simd = { workspace = true, optional = true } libm = { workspace = true } log = { workspace = true, optional = true } rayon = { workspace = true, optional = true } -j2k-profile = { path = "../j2k-profile", version = "=0.13.1", default-features = false } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1" } -j2k-types = { path = "../j2k-types", version = "=0.13.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.13.2", default-features = false } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2" } +j2k-types = { path = "../j2k-types", version = "=0.13.2" } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/j2k-native/fuzz/Cargo.lock b/crates/j2k-native/fuzz/Cargo.lock index b2af73172..d33e25af8 100644 --- a/crates/j2k-native/fuzz/Cargo.lock +++ b/crates/j2k-native/fuzz/Cargo.lock @@ -82,14 +82,14 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-types", ] [[package]] name = "j2k-native" -version = "0.13.1" +version = "0.13.2" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -109,11 +109,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.13.1" +version = "0.13.2" [[package]] name = "j2k-types" -version = "0.13.1" +version = "0.13.2" [[package]] name = "jobserver" diff --git a/crates/j2k-native/src/error.rs b/crates/j2k-native/src/error.rs index adf74d0b5..5b6c82420 100644 --- a/crates/j2k-native/src/error.rs +++ b/crates/j2k-native/src/error.rs @@ -64,6 +64,8 @@ pub enum DecodeErrorClass { /// Stable user-facing feature label. what: &'static str, }, + /// All other native decoder failures. + Backend, /// The decode would exceed a resource limit: its live working memory /// would pass the decode cap, or the image is too large to size. /// @@ -75,8 +77,6 @@ pub enum DecodeErrorClass { /// The cap the request exceeded, when known. cap: Option, }, - /// All other native decoder failures. - Backend, } /// Error returned by native JPEG 2000 and HTJ2K encode operations. diff --git a/crates/j2k-t803/Cargo.toml b/crates/j2k-t803/Cargo.toml index 90470286c..0027abbe0 100644 --- a/crates/j2k-t803/Cargo.toml +++ b/crates/j2k-t803/Cargo.toml @@ -29,14 +29,14 @@ metal-runner = ["runner", "dep:j2k-metal", "dep:objc2-metal"] [dependencies] image = { workspace = true, optional = true } -j2k = { path = "../j2k", version = "=0.13.1", optional = true } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1", optional = true } +j2k = { path = "../j2k", version = "=0.13.2", optional = true } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2", optional = true } j2k-compare = { path = "../j2k-compare", version = "=0.2.0", optional = true } -j2k-core = { path = "../j2k-core", version = "=0.13.1", optional = true } -j2k-native = { path = "../j2k-native", version = "=0.13.1", optional = true } -j2k-cuda = { path = "../j2k-cuda", version = "=0.13.1", optional = true } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.1", optional = true } -j2k-metal = { path = "../j2k-metal", version = "=0.13.1", optional = true } +j2k-core = { path = "../j2k-core", version = "=0.13.2", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.13.2", optional = true } +j2k-cuda = { path = "../j2k-cuda", version = "=0.13.2", optional = true } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.2", optional = true } +j2k-metal = { path = "../j2k-metal", version = "=0.13.2", optional = true } serde = { workspace = true } serde_json = { workspace = true, features = ["float_roundtrip"] } sha2 = { workspace = true } diff --git a/crates/j2k-t803/fuzz/Cargo.lock b/crates/j2k-t803/fuzz/Cargo.lock index 92a3db007..0b46cb9d6 100644 --- a/crates/j2k-t803/fuzz/Cargo.lock +++ b/crates/j2k-t803/fuzz/Cargo.lock @@ -262,7 +262,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "j2k" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-core", "j2k-native", @@ -273,7 +273,7 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-types", ] @@ -293,7 +293,7 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.13.1" +version = "0.13.2" dependencies = [ "libc", "thiserror", @@ -302,7 +302,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.13.1" +version = "0.13.2" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -314,11 +314,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.13.1" +version = "0.13.2" [[package]] name = "j2k-t803" -version = "0.13.1" +version = "0.13.2" dependencies = [ "image", "j2k", @@ -344,7 +344,7 @@ dependencies = [ [[package]] name = "j2k-test-support" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-core", "serde", @@ -354,7 +354,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.13.1" +version = "0.13.2" [[package]] name = "jobserver" diff --git a/crates/j2k-test-support/Cargo.toml b/crates/j2k-test-support/Cargo.toml index 65d018956..3c656dd24 100644 --- a/crates/j2k-test-support/Cargo.toml +++ b/crates/j2k-test-support/Cargo.toml @@ -16,8 +16,8 @@ path = "src/lib.rs" j2k-native-fixtures = ["dep:j2k-native"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-native = { path = "../j2k-native", version = "=0.13.1", optional = true } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-native = { path = "../j2k-native", version = "=0.13.2", optional = true } serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } diff --git a/crates/j2k-tilecodec/Cargo.toml b/crates/j2k-tilecodec/Cargo.toml index fe004598d..06f5703f3 100644 --- a/crates/j2k-tilecodec/Cargo.toml +++ b/crates/j2k-tilecodec/Cargo.toml @@ -19,7 +19,7 @@ name = "j2k_tilecodec" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } thiserror = { workspace = true } flate2 = { workspace = true } zstd = { workspace = true } diff --git a/crates/j2k-tilecodec/fuzz/Cargo.lock b/crates/j2k-tilecodec/fuzz/Cargo.lock index fe39e7860..2a48c495b 100644 --- a/crates/j2k-tilecodec/fuzz/Cargo.lock +++ b/crates/j2k-tilecodec/fuzz/Cargo.lock @@ -72,7 +72,7 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.13.1" +version = "0.13.2" dependencies = [ "libc", "thiserror", @@ -81,7 +81,7 @@ dependencies = [ [[package]] name = "j2k-tilecodec" -version = "0.13.1" +version = "0.13.2" dependencies = [ "flate2", "j2k-core", diff --git a/crates/j2k-transcode-cuda/Cargo.toml b/crates/j2k-transcode-cuda/Cargo.toml index 5bd9bc417..951a413d6 100644 --- a/crates/j2k-transcode-cuda/Cargo.toml +++ b/crates/j2k-transcode-cuda/Cargo.toml @@ -31,16 +31,16 @@ cuda-runtime = [ cuda-profiling = ["cuda-runtime", "j2k-cuda-runtime/cuda-profiling", "j2k-cuda-transcode-engine/cuda-profiling"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.13.1" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.1", optional = true } -j2k-cuda-transcode-engine = { path = "../j2k-cuda-transcode-engine", version = "=0.13.1", optional = true } -j2k-cuda-j2k-engine = { path = "../j2k-cuda-j2k-engine", version = "=0.13.1", optional = true } -j2k-native = { path = "../j2k-native", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.13.2" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.13.2", optional = true } +j2k-cuda-transcode-engine = { path = "../j2k-cuda-transcode-engine", version = "=0.13.2", optional = true } +j2k-cuda-j2k-engine = { path = "../j2k-cuda-j2k-engine", version = "=0.13.2", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } [dev-dependencies] criterion = { workspace = true } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.1" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.2" } j2k-test-support = { path = "../j2k-test-support" } j2k-transcode-test-support = { path = "../j2k-transcode-test-support" } sha2 = { workspace = true } diff --git a/crates/j2k-transcode-metal/Cargo.toml b/crates/j2k-transcode-metal/Cargo.toml index 5c98f040c..f31f3c225 100644 --- a/crates/j2k-transcode-metal/Cargo.toml +++ b/crates/j2k-transcode-metal/Cargo.toml @@ -26,11 +26,11 @@ default = [] bench-internals = ["j2k-transcode/dev-support"] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1" } -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.1" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.13.1" } -j2k-types = { path = "../j2k-types", version = "=0.13.1" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.13.2" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.13.2" } +j2k-types = { path = "../j2k-types", version = "=0.13.2" } [target.'cfg(target_os = "macos")'.dependencies] objc2 = { workspace = true } @@ -40,10 +40,10 @@ objc2-metal = { workspace = true } [dev-dependencies] criterion = { workspace = true } rayon = { workspace = true } -j2k-metal = { path = "../j2k-metal", version = "=0.13.1" } -j2k-native = { path = "../j2k-native", version = "=0.13.1" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.1" } -j2k-profile = { path = "../j2k-profile", version = "=0.13.1" } +j2k-metal = { path = "../j2k-metal", version = "=0.13.2" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.2" } +j2k-profile = { path = "../j2k-profile", version = "=0.13.2" } j2k-test-support = { path = "../j2k-test-support" } j2k-transcode-test-support = { path = "../j2k-transcode-test-support" } diff --git a/crates/j2k-transcode-test-support/Cargo.toml b/crates/j2k-transcode-test-support/Cargo.toml index e31dc94ac..e0f83a462 100644 --- a/crates/j2k-transcode-test-support/Cargo.toml +++ b/crates/j2k-transcode-test-support/Cargo.toml @@ -13,11 +13,11 @@ name = "j2k_transcode_test_support" path = "src/lib.rs" [dependencies] -j2k-transcode = { path = "../j2k-transcode", version = "=0.13.1", features = ["dev-support"] } -j2k-types = { path = "../j2k-types", version = "=0.13.1" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.13.2", features = ["dev-support"] } +j2k-types = { path = "../j2k-types", version = "=0.13.2" } [dev-dependencies] -j2k-native = { path = "../j2k-native", version = "=0.13.1" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } [lints.rust] unsafe_code = "forbid" diff --git a/crates/j2k-transcode/Cargo.toml b/crates/j2k-transcode/Cargo.toml index 4b11452bf..92683c216 100644 --- a/crates/j2k-transcode/Cargo.toml +++ b/crates/j2k-transcode/Cargo.toml @@ -23,12 +23,12 @@ dev-support = [] [dependencies] rayon = { workspace = true } -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.1" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.1" } -j2k = { path = "../j2k", version = "=0.13.1" } -j2k-native = { path = "../j2k-native", version = "=0.13.1" } -j2k-profile = { path = "../j2k-profile", version = "=0.13.1", default-features = false } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.13.2" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.13.2" } +j2k = { path = "../j2k", version = "=0.13.2" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } +j2k-profile = { path = "../j2k-profile", version = "=0.13.2", default-features = false } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/j2k-transcode/fuzz/Cargo.lock b/crates/j2k-transcode/fuzz/Cargo.lock index ddbaa2db5..2e0f9a28c 100644 --- a/crates/j2k-transcode/fuzz/Cargo.lock +++ b/crates/j2k-transcode/fuzz/Cargo.lock @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "j2k" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-core", "j2k-native", @@ -100,14 +100,14 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-types", ] [[package]] name = "j2k-core" -version = "0.13.1" +version = "0.13.2" dependencies = [ "libc", "thiserror", @@ -116,7 +116,7 @@ dependencies = [ [[package]] name = "j2k-jpeg" -version = "0.13.1" +version = "0.13.2" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -129,7 +129,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.13.1" +version = "0.13.2" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -141,11 +141,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.13.1" +version = "0.13.2" [[package]] name = "j2k-transcode" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k", "j2k-codec-math", @@ -167,7 +167,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.13.1" +version = "0.13.2" [[package]] name = "jobserver" diff --git a/crates/j2k/Cargo.toml b/crates/j2k/Cargo.toml index 4f4c16fca..6905aac67 100644 --- a/crates/j2k/Cargo.toml +++ b/crates/j2k/Cargo.toml @@ -20,9 +20,9 @@ name = "j2k" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.13.1" } -j2k-native = { path = "../j2k-native", version = "=0.13.1" } -j2k-types = { path = "../j2k-types", version = "=0.13.1" } +j2k-core = { path = "../j2k-core", version = "=0.13.2" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } +j2k-types = { path = "../j2k-types", version = "=0.13.2" } moxcms = { workspace = true } thiserror = { workspace = true } @@ -30,7 +30,7 @@ thiserror = { workspace = true } proptest = { workspace = true } criterion = { workspace = true } rayon = { workspace = true } -j2k-native = { path = "../j2k-native", version = "=0.13.1" } +j2k-native = { path = "../j2k-native", version = "=0.13.2" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [[bench]] diff --git a/crates/j2k/fuzz/Cargo.lock b/crates/j2k/fuzz/Cargo.lock index 704e15438..e69b44b86 100644 --- a/crates/j2k/fuzz/Cargo.lock +++ b/crates/j2k/fuzz/Cargo.lock @@ -143,7 +143,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "j2k" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-core", "j2k-native", @@ -154,14 +154,14 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-types", ] [[package]] name = "j2k-core" -version = "0.13.1" +version = "0.13.2" dependencies = [ "libc", "thiserror", @@ -179,7 +179,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.13.1" +version = "0.13.2" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -191,11 +191,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.13.1" +version = "0.13.2" [[package]] name = "j2k-test-support" -version = "0.13.1" +version = "0.13.2" dependencies = [ "j2k-core", "j2k-native", @@ -206,7 +206,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.13.1" +version = "0.13.2" [[package]] name = "jobserver" diff --git a/crates/j2k/src/error.rs b/crates/j2k/src/error.rs index 6b97e6157..cf5c73ed0 100644 --- a/crates/j2k/src/error.rs +++ b/crates/j2k/src/error.rs @@ -125,22 +125,6 @@ pub enum J2kError { source: NativeBackendError, }, - /// Native decode would exceed a resource limit: its live working memory - /// would pass the decode cap, or the image is too large to size. - /// - /// Unlike invalid or unsupported input, the same codestream may decode - /// through a smaller request, such as a region or a reduced resolution. - #[error("decode resource limit exceeded: {source}")] - ResourceLimit { - /// Requested live bytes at the rejected boundary, when known. - requested: Option, - /// The cap the request exceeded, when known. - cap: Option, - /// Structured native decoder failure. - #[source] - source: NativeBackendError, - }, - /// Native decoder failure while validating generated encoder output. /// /// The structured source is retained for diagnostics, but this phase is @@ -300,6 +284,22 @@ pub enum J2kError { /// Image height. height: u32, }, + + /// Native decode would exceed a resource limit: its live working memory + /// would pass the decode cap, or the image is too large to size. + /// + /// Unlike invalid or unsupported input, the same codestream may decode + /// through a smaller request, such as a region or a reduced resolution. + #[error("decode resource limit exceeded: {source}")] + ResourceLimit { + /// Requested live bytes at the rejected boundary, when known. + requested: Option, + /// The cap the request exceeded, when known. + cap: Option, + /// Structured native decoder failure. + #[source] + source: NativeBackendError, + }, } impl J2kError { diff --git a/crates/j2k/src/owned_batch/cpu.rs b/crates/j2k/src/owned_batch/cpu.rs index 9a48065f8..81115c491 100644 --- a/crates/j2k/src/owned_batch/cpu.rs +++ b/crates/j2k/src/owned_batch/cpu.rs @@ -320,6 +320,11 @@ impl CpuBatchDecoder { } /// Inspect and group owned encoded inputs for repeated decode. + /// + /// Inputs whose output allocation or planned execution exceeds a memory + /// limit are recorded as indexed preparation errors in the returned batch. + /// Preparation checks individual images; a group's combined output + /// allocation is checked during decoding. pub fn prepare( &self, inputs: Vec, @@ -332,6 +337,8 @@ impl CpuBatchDecoder { } /// Regroup caller-supplied prepared images under this session's output policy. + /// + /// This applies the same per-image memory checks as [`Self::prepare`]. pub fn prepare_prepared_images( &self, images: Vec, diff --git a/docs/stable-api-1.0.md b/docs/stable-api-1.0.md index 7fcb75492..336a5a5cc 100644 --- a/docs/stable-api-1.0.md +++ b/docs/stable-api-1.0.md @@ -35,7 +35,14 @@ consumers to retain pooled surfaces until their GPU work completes. `0.8.0` could only be compared against `v0.7.5`, and `0.9.0` only against `v0.8.1`. `0.10.0` was compared directly with `v0.9.0` under a one-time exception for the crate split, which is now disabled. The current semver -baseline is `v0.13.0`. The `0.13.1` release is reviewed in the +baseline is `v0.13.1`. Development for `0.13.2` is reviewed in the +[0.13.2 API review](../xtask/release-evidence/public-api/public-api-review-0.13.2.yml). +It adds resource-limit error variants to existing non-exhaustive enums and +two rustdoc-hidden batch execution-size helpers. No existing API item is +removed or changed. Resource-limit failures now have their own error class, +and `CpuBatchDecoder` preparation checks planned execution memory before decoding; +the review records these error-behavior corrections. +The `0.13.1` release is reviewed in the [0.13.1 API review](../xtask/release-evidence/public-api/public-api-review-0.13.1.yml); it changes no ordinary API item and only adds rustdoc-hidden `j2k-native` items. The `0.13.0` release was compared with `v0.12.0` in the diff --git a/xtask/RELEASING.md b/xtask/RELEASING.md index 79effd217..20ef7c098 100644 --- a/xtask/RELEASING.md +++ b/xtask/RELEASING.md @@ -8,6 +8,10 @@ used only for shapes where it was tested and measured faster. ## Release status +Development is staged for `0.13.2`, with the public API compared against +`v0.13.1` in its [API review](release-evidence/public-api/public-api-review-0.13.2.yml). +It has not been released; the published release remains `0.13.1`. + Version `0.13.1` is published. Its [API report](release-evidence/public-api/reviewed-public-api-diff-0.13.1.md) compares with `v0.13.0`; the @@ -502,7 +506,7 @@ other item. Every `cargo xtask semver` run generates both passes, compares them with the committed files, and checks the added/removed fingerprints and the hidden-item count against -`xtask/release-evidence/public-api/public-api-review-0.13.1.yml`. A package with +`xtask/release-evidence/public-api/public-api-review-0.13.2.yml`. A package with hidden items must give a reason for them in that file. Any removed item must be listed in the review file with its package, a summary, @@ -524,6 +528,7 @@ item list into the docs. | Release | Compared with | Report | | --- | --- | --- | +| 0.13.2 (unreleased) | 0.13.1 | [API diff](release-evidence/public-api/reviewed-public-api-diff-0.13.2.md), [review](release-evidence/public-api/public-api-review-0.13.2.yml) | | 0.13.1 | 0.13.0 | [API diff](release-evidence/public-api/reviewed-public-api-diff-0.13.1.md), [review](release-evidence/public-api/public-api-review-0.13.1.yml) | | 0.13.0 | 0.12.0 | [API diff](release-evidence/public-api/reviewed-public-api-diff-0.13.0.md), [review](release-evidence/public-api/public-api-review-0.13.0.yml) | | 0.12.0 | 0.11.3 | [API diff](release-evidence/public-api/reviewed-public-api-diff-0.12.0.md), [review](release-evidence/public-api/public-api-review-0.12.0.yml) | diff --git a/xtask/release-evidence/public-api/public-api-review-0.13.2.yml b/xtask/release-evidence/public-api/public-api-review-0.13.2.yml new file mode 100644 index 000000000..b636c35a1 --- /dev/null +++ b/xtask/release-evidence/public-api/public-api-review-0.13.2.yml @@ -0,0 +1,174 @@ +version: 3 +baseline_tag: v0.13.1 +baseline_version: 0.13.1 +candidate_version: "0.13.2" +break_ledger: [] +reviews: + j2k-core: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 226 + hidden_fingerprint: "fnv1a64:487523cc4e7f504e" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-profile: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 117 + hidden_fingerprint: "fnv1a64:a1546f56ae38f2e3" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-types: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 111 + hidden_fingerprint: "fnv1a64:6eaf63f068893d17" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-codec-math: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-cuda-build-support: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-cuda-runtime: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 255 + hidden_fingerprint: "fnv1a64:57c61a4568990ce6" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-cuda-j2k-engine: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 692 + hidden_fingerprint: "fnv1a64:684fbcddde38e1f1" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-cuda-jpeg-engine: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 205 + hidden_fingerprint: "fnv1a64:ad37da6966da1cfd" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-cuda-transcode-engine: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 84 + hidden_fingerprint: "fnv1a64:b2fcaa9b0171aba7" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-metal-support: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 11 + hidden_fingerprint: "fnv1a64:02139606f1202938" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-mpsgraph-support: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-native: + removed_fingerprint: "none" + added_fingerprint: "fnv1a64:22b943651a07318e" + hidden_count: 586 + hidden_fingerprint: "fnv1a64:733f3aa607999371" + rationale: "Compared the complete ordinary API inventory against v0.13.1. Adds DecodeErrorClass::ResourceLimit { requested, cap } to an already non-exhaustive enum; no existing item changes and existing variant discriminants are preserved. AllocationTooLarge and ImageTooLarge now classify as ResourceLimit instead of the generic Backend class. This is a compatible refinement of the extensible error taxonomy: invalid-input and allocator-failure classes stay unchanged. Callers that want to handle memory limits separately can match the new variant; existing wildcard matches remain valid." + hidden_rationale: "Adds referenced_classic_plan_execution_bytes and referenced_htj2k_plan_execution_bytes for sibling CPU batch adapters to check planned live execution memory before allocation. Both helpers return Result and do not alter existing signatures. Every other hidden item is unchanged from v0.13.1." + j2k-jpeg: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 701 + hidden_fingerprint: "fnv1a64:bd0cf738a0f96592" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-tilecodec: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 32 + hidden_fingerprint: "fnv1a64:eb677842559d9b0e" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k: + removed_fingerprint: "none" + added_fingerprint: "fnv1a64:363048555efa46de" + hidden_count: 108 + hidden_fingerprint: "fnv1a64:01e74ad6c3ff416e" + rationale: "Compared the complete ordinary API inventory against v0.13.1. Adds J2kError::ResourceLimit { requested, cap, source } to an already non-exhaustive enum; no existing item changes and existing variant discriminants are preserved. Budget and image-size failures now use this variant instead of NativeDecode, retaining the structured native cause. Callers specifically handling budget failures through NativeDecode should also handle ResourceLimit. Batches prepared by CpuBatchDecoder report items whose output allocation or planned execution cannot fit as indexed preparation errors and exclude those items from the prepared groups. Backend-neutral prepare_batch and prepare_batch_from_images retain whole-call output-cap failures for a single oversized image when subsequently decoded on the CPU. On either preparation path, a group whose combined output exceeds the allocation cap still fails the whole decode call. These correct admission timing within the CPU-owned preparation paths and preserve the existing per-item failure model; successful decode output and signatures are unchanged. The public error documentation and changelog describe the refinements." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-transcode: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 458 + hidden_fingerprint: "fnv1a64:449a76476549c1e2" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-transcode-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 20 + hidden_fingerprint: "fnv1a64:be22ca1d5e12b043" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-jpeg-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 83 + hidden_fingerprint: "fnv1a64:79397ffc221464c3" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 321 + hidden_fingerprint: "fnv1a64:41513c1e93413909" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-transcode-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 49 + hidden_fingerprint: "fnv1a64:b4a8b53d5e10bb00" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-jpeg-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 133 + hidden_fingerprint: "fnv1a64:9aa724bde7af1618" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 193 + hidden_fingerprint: "fnv1a64:8a9b0ac701f08d00" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-ml: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." + j2k-mpsgraph: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Compared the complete ordinary API inventory against v0.13.1; no signatures were added, removed, or changed." + hidden_rationale: "Compared the complete hidden API inventory against v0.13.1; it is unchanged." diff --git a/xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.2.md b/xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.2.md new file mode 100644 index 000000000..66b391428 --- /dev/null +++ b/xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.2.md @@ -0,0 +1,336 @@ +# Reviewed public API diff for j2k 0.13.2 + +This report is generated by `cargo xtask semver --write-report`. Normal `cargo xtask semver` regenerates it in memory and fails if this committed file is stale. Every ordinary added/removed fingerprint and every full rustdoc-hidden candidate-inventory fingerprint requires an exact reviewed entry in `xtask/release-evidence/public-api/public-api-review-0.13.2.yml`; report regeneration never updates that review config. + +- Baseline registry version: `0.13.1` +- Baseline source snapshot: `v0.13.1` peeled to `0c6b6e474d2d96e911df94727787b9b0fb0db300` +- Candidate version: `0.13.2` +- Tool pins: Rust `1.99.0`, `cargo-semver-checks 0.51.0`, `cargo-public-api 0.52.0`, rustdoc `nightly-2026-08-13`, target `aarch64-apple-darwin` + +## Summary + +| Package | Baseline | Candidate | Computed release type | Added | Removed/changed | Removed fingerprint | Added fingerprint | Rustdoc-hidden items | Hidden inventory fingerprint | +| --- | --- | --- | --- | ---: | ---: | --- | --- | ---: | --- | +| `j2k-core` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 226 | `fnv1a64:487523cc4e7f504e` | +| `j2k-profile` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 117 | `fnv1a64:a1546f56ae38f2e3` | +| `j2k-types` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 111 | `fnv1a64:6eaf63f068893d17` | +| `j2k-codec-math` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | +| `j2k-cuda-build-support` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | +| `j2k-cuda-runtime` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 255 | `fnv1a64:57c61a4568990ce6` | +| `j2k-cuda-j2k-engine` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 692 | `fnv1a64:684fbcddde38e1f1` | +| `j2k-cuda-jpeg-engine` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 205 | `fnv1a64:ad37da6966da1cfd` | +| `j2k-cuda-transcode-engine` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 84 | `fnv1a64:b2fcaa9b0171aba7` | +| `j2k-metal-support` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 11 | `fnv1a64:02139606f1202938` | +| `j2k-mpsgraph-support` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | +| `j2k-native` | `0.13.1` | `0.13.2` | `minor` | 3 | 0 | `none` | `fnv1a64:22b943651a07318e` | 586 | `fnv1a64:733f3aa607999371` | +| `j2k-jpeg` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 701 | `fnv1a64:bd0cf738a0f96592` | +| `j2k-tilecodec` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 32 | `fnv1a64:eb677842559d9b0e` | +| `j2k` | `0.13.1` | `0.13.2` | `minor` | 4 | 0 | `none` | `fnv1a64:363048555efa46de` | 108 | `fnv1a64:01e74ad6c3ff416e` | +| `j2k-transcode` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 458 | `fnv1a64:449a76476549c1e2` | +| `j2k-transcode-cuda` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 20 | `fnv1a64:be22ca1d5e12b043` | +| `j2k-jpeg-metal` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 83 | `fnv1a64:79397ffc221464c3` | +| `j2k-metal` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 321 | `fnv1a64:41513c1e93413909` | +| `j2k-transcode-metal` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 49 | `fnv1a64:b4a8b53d5e10bb00` | +| `j2k-jpeg-cuda` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 133 | `fnv1a64:9aa724bde7af1618` | +| `j2k-cuda` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 193 | `fnv1a64:8a9b0ac701f08d00` | +| `j2k-ml` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | +| `j2k-mpsgraph` | `0.13.1` | `0.13.2` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | + +## Published-package details + +### `j2k-core` + +Baseline items: 526. Candidate items: 526. Computed release type: `minor`. Rustdoc-hidden candidate items: 226. Full hidden-inventory fingerprint: `fnv1a64:487523cc4e7f504e`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-profile` + +Baseline items: 1. Candidate items: 1. Computed release type: `minor`. Rustdoc-hidden candidate items: 117. Full hidden-inventory fingerprint: `fnv1a64:a1546f56ae38f2e3`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-types` + +Baseline items: 638. Candidate items: 638. Computed release type: `minor`. Rustdoc-hidden candidate items: 111. Full hidden-inventory fingerprint: `fnv1a64:6eaf63f068893d17`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-codec-math` + +Baseline items: 124. Candidate items: 124. Computed release type: `minor`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda-build-support` + +Baseline items: 10. Candidate items: 10. Computed release type: `minor`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda-runtime` + +Baseline items: 129. Candidate items: 129. Computed release type: `minor`. Rustdoc-hidden candidate items: 255. Full hidden-inventory fingerprint: `fnv1a64:57c61a4568990ce6`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda-j2k-engine` + +Baseline items: 5. Candidate items: 5. Computed release type: `minor`. Rustdoc-hidden candidate items: 692. Full hidden-inventory fingerprint: `fnv1a64:684fbcddde38e1f1`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda-jpeg-engine` + +Baseline items: 14. Candidate items: 14. Computed release type: `minor`. Rustdoc-hidden candidate items: 205. Full hidden-inventory fingerprint: `fnv1a64:ad37da6966da1cfd`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda-transcode-engine` + +Baseline items: 6. Candidate items: 6. Computed release type: `minor`. Rustdoc-hidden candidate items: 84. Full hidden-inventory fingerprint: `fnv1a64:b2fcaa9b0171aba7`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-metal-support` + +Baseline items: 180. Candidate items: 180. Computed release type: `minor`. Rustdoc-hidden candidate items: 11. Full hidden-inventory fingerprint: `fnv1a64:02139606f1202938`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-mpsgraph-support` + +Baseline items: 19. Candidate items: 19. Computed release type: `minor`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-native` + +Baseline items: 422. Candidate items: 425. Computed release type: `minor`. Rustdoc-hidden candidate items: 586. Full hidden-inventory fingerprint: `fnv1a64:733f3aa607999371`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +```text +pub j2k_native::DecodeErrorClass::ResourceLimit +pub j2k_native::DecodeErrorClass::ResourceLimit::cap: core::option::Option +pub j2k_native::DecodeErrorClass::ResourceLimit::requested: core::option::Option +``` + +### `j2k-jpeg` + +Baseline items: 710. Candidate items: 710. Computed release type: `minor`. Rustdoc-hidden candidate items: 701. Full hidden-inventory fingerprint: `fnv1a64:bd0cf738a0f96592`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-tilecodec` + +Baseline items: 31. Candidate items: 31. Computed release type: `minor`. Rustdoc-hidden candidate items: 32. Full hidden-inventory fingerprint: `fnv1a64:eb677842559d9b0e`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k` + +Baseline items: 804. Candidate items: 808. Computed release type: `minor`. Rustdoc-hidden candidate items: 108. Full hidden-inventory fingerprint: `fnv1a64:01e74ad6c3ff416e`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +```text +pub j2k::J2kError::ResourceLimit +pub j2k::J2kError::ResourceLimit::cap: core::option::Option +pub j2k::J2kError::ResourceLimit::requested: core::option::Option +pub j2k::J2kError::ResourceLimit::source: j2k::NativeBackendError +``` + +### `j2k-transcode` + +Baseline items: 427. Candidate items: 427. Computed release type: `minor`. Rustdoc-hidden candidate items: 458. Full hidden-inventory fingerprint: `fnv1a64:449a76476549c1e2`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-transcode-cuda` + +Baseline items: 49. Candidate items: 49. Computed release type: `minor`. Rustdoc-hidden candidate items: 20. Full hidden-inventory fingerprint: `fnv1a64:be22ca1d5e12b043`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg-metal` + +Baseline items: 186. Candidate items: 186. Computed release type: `minor`. Rustdoc-hidden candidate items: 83. Full hidden-inventory fingerprint: `fnv1a64:79397ffc221464c3`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-metal` + +Baseline items: 280. Candidate items: 280. Computed release type: `minor`. Rustdoc-hidden candidate items: 321. Full hidden-inventory fingerprint: `fnv1a64:41513c1e93413909`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-transcode-metal` + +Baseline items: 99. Candidate items: 99. Computed release type: `minor`. Rustdoc-hidden candidate items: 49. Full hidden-inventory fingerprint: `fnv1a64:b4a8b53d5e10bb00`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg-cuda` + +Baseline items: 79. Candidate items: 79. Computed release type: `minor`. Rustdoc-hidden candidate items: 133. Full hidden-inventory fingerprint: `fnv1a64:9aa724bde7af1618`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda` + +Baseline items: 247. Candidate items: 247. Computed release type: `minor`. Rustdoc-hidden candidate items: 193. Full hidden-inventory fingerprint: `fnv1a64:8a9b0ac701f08d00`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-ml` + +Baseline items: 133. Candidate items: 133. Computed release type: `minor`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-mpsgraph` + +Baseline items: 89. Candidate items: 89. Computed release type: `minor`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. diff --git a/xtask/src/semver.rs b/xtask/src/semver.rs index 9986b8bb0..e25561859 100644 --- a/xtask/src/semver.rs +++ b/xtask/src/semver.rs @@ -25,12 +25,12 @@ use compatibility::{semver_check_args, semver_check_release_type}; const CARGO_SEMVER_CHECKS_VERSION: &str = "0.51.0"; const SEMVER_TOOLCHAIN: &str = "1.99.0"; -const SEMVER_BASELINE_VERSION: &str = "0.13.0"; -const SEMVER_BASELINE_TAG: &str = "v0.13.0"; -const SEMVER_BASELINE_COMMIT: &str = "501242621b7e7171fd7f8813a40cd0cb31a5f5a7"; +const SEMVER_BASELINE_VERSION: &str = "0.13.1"; +const SEMVER_BASELINE_TAG: &str = "v0.13.1"; +const SEMVER_BASELINE_COMMIT: &str = "0c6b6e474d2d96e911df94727787b9b0fb0db300"; const API_DIFF_REPORT: &str = - "xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.1.md"; -const API_REVIEW_CONFIG: &str = "xtask/release-evidence/public-api/public-api-review-0.13.1.yml"; + "xtask/release-evidence/public-api/reviewed-public-api-diff-0.13.2.md"; +const API_REVIEW_CONFIG: &str = "xtask/release-evidence/public-api/public-api-review-0.13.2.yml"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct BaselineTransition<'a> { diff --git a/xtask/src/semver/tests.rs b/xtask/src/semver/tests.rs index 4fef094c3..757659098 100644 --- a/xtask/src/semver/tests.rs +++ b/xtask/src/semver/tests.rs @@ -272,7 +272,7 @@ fn report_has_one_published_details_section_without_consumed_transition() { assert_eq!(report.matches("## Published-package details").count(), 1); assert!(report.contains("Rustdoc-hidden candidate items: 1")); assert!(report.contains("Full hidden-inventory fingerprint: `fnv1a64:")); - assert!(report.contains("Baseline registry version: `0.13.0`")); + assert!(report.contains("Baseline registry version: `0.13.1`")); assert!(!report.contains("Active intentional-break transition:")); assert!(!report.contains("Required next semver baseline:")); } @@ -281,8 +281,8 @@ fn report_has_one_published_details_section_without_consumed_transition() { fn parses_review_config_and_rejects_unknown_fields() { let source = "\ version: 3 -baseline_tag: v0.13.0 -baseline_version: 0.13.0 +baseline_tag: v0.13.1 +baseline_version: 0.13.1 candidate_version: 0.11.0 break_ledger: - id: strict-decode-default diff --git a/xtask/src/semver/tests/api_planning.rs b/xtask/src/semver/tests/api_planning.rs index 604b38d30..9ca8a9ba4 100644 --- a/xtask/src/semver/tests/api_planning.rs +++ b/xtask/src/semver/tests/api_planning.rs @@ -89,7 +89,7 @@ fn package_diff_planning_distinguishes_published_and_new_packages() { assert!(diffs[1].removed.is_empty()); let report = render_report("0.7.4", &diffs, "0.52.0"); - assert!(report.contains("## New packages without a 0.13.0 registry baseline")); + assert!(report.contains("## New packages without a 0.13.1 registry baseline")); assert!(report.contains("- `j2k-future` `0.7.4`: 1 ordinary public API items")); assert!(report.contains("### `j2k`")); assert!(report.contains("```text\nremoved\n```")); diff --git a/xtask/src/semver/tests/command_boundaries.rs b/xtask/src/semver/tests/command_boundaries.rs index 0f04c82c4..287c61868 100644 --- a/xtask/src/semver/tests/command_boundaries.rs +++ b/xtask/src/semver/tests/command_boundaries.rs @@ -76,7 +76,7 @@ fn committed_candidate_semver_inputs_match_the_pinned_workspace_contract() { assert!(hidden.starts_with("# J2K 1.0 Rustdoc-Hidden Public API Snapshot")); let versions = workspace_package_versions().expect("workspace package versions"); - assert_eq!(versions.get("j2k").map(String::as_str), Some("0.13.1")); + assert_eq!(versions.get("j2k").map(String::as_str), Some("0.13.2")); assert!(versions.keys().collect::>().len() > 10); } @@ -91,7 +91,7 @@ fn baseline_revision_validation_accepts_only_the_pinned_commit() { #[test] fn committed_review_config_covers_the_exact_release_scope() { let config = load_review_config().expect("load committed API review config"); - assert_eq!(config.candidate_version, "0.13.1"); + assert_eq!(config.candidate_version, "0.13.2"); assert!( config.break_ledger.is_empty(), "patch release must not carry a consumed breaking-change ledger" @@ -157,7 +157,7 @@ fn semver_check_command_uses_the_computed_candidate_release_type() { "--package", "j2k-core", "--baseline-version", - "0.13.0", + "0.13.1", "--release-type", "major", "--color", diff --git a/xtask/tests/command_orchestration.rs b/xtask/tests/command_orchestration.rs index e4d516ac2..1389a711b 100644 --- a/xtask/tests/command_orchestration.rs +++ b/xtask/tests/command_orchestration.rs @@ -119,8 +119,8 @@ fn release_critical_orchestrators_run_from_the_workspace_without_real_cargo() { assert!(log.contains("package -p j2k-cli --no-verify")); #[cfg(target_os = "macos")] { - assert!(log.contains("git rev-parse v0.13.0^{commit}")); - assert!(log.contains("git show v0.13.0:xtask/api/stable-api-1.0.public-api.txt")); + assert!(log.contains("git rev-parse v0.13.1^{commit}")); + assert!(log.contains("git show v0.13.1:xtask/api/stable-api-1.0.public-api.txt")); } } diff --git a/xtask/tests/command_orchestration/support.rs b/xtask/tests/command_orchestration/support.rs index 1770c7bc1..f587d5507 100644 --- a/xtask/tests/command_orchestration/support.rs +++ b/xtask/tests/command_orchestration/support.rs @@ -60,7 +60,7 @@ impl Harness { fs::write( &git, format!( - "#!/bin/sh\nprintf 'git %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = status ]; then exit 0; fi\nif [ \"$1\" = config ] && [ \"$2\" = --get ] && [ \"$3\" = remote.origin.url ]; then printf '%s\\n' 'git@example.invalid:frames-sg/j2k.git'; exit 0; fi\nif [ \"$1\" = rev-parse ] && [ \"$2\" = 'v0.13.0^{{commit}}' ]; then printf '%s\\n' '501242621b7e7171fd7f8813a40cd0cb31a5f5a7'; exit 0; fi\nif [ \"$1\" = show ] && [ \"$2\" = 'v0.13.0:xtask/api/stable-api-1.0.public-api.txt' ]; then exec cat '{}'; fi\nif [ \"$1\" = rev-parse ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release revision: %s\\n' \"$2\" >&2; exit 97; fi\nif [ \"$1\" = show ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release object: %s\\n' \"$2\" >&2; exit 97; fi\nexec \"{}\" \"$@\"\n", + "#!/bin/sh\nprintf 'git %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = status ]; then exit 0; fi\nif [ \"$1\" = config ] && [ \"$2\" = --get ] && [ \"$3\" = remote.origin.url ]; then printf '%s\\n' 'git@example.invalid:frames-sg/j2k.git'; exit 0; fi\nif [ \"$1\" = rev-parse ] && [ \"$2\" = 'v0.13.1^{{commit}}' ]; then printf '%s\\n' '0c6b6e474d2d96e911df94727787b9b0fb0db300'; exit 0; fi\nif [ \"$1\" = show ] && [ \"$2\" = 'v0.13.1:xtask/api/stable-api-1.0.public-api.txt' ]; then exec cat '{}'; fi\nif [ \"$1\" = rev-parse ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release revision: %s\\n' \"$2\" >&2; exit 97; fi\nif [ \"$1\" = show ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release object: %s\\n' \"$2\" >&2; exit 97; fi\nexec \"{}\" \"$@\"\n", log.display(), baseline_snapshot.display(), real_git.display()